[All] Change threading to be global

This commit is contained in:
Matt Nadareski
2017-07-13 17:03:38 -07:00
parent b7b6247ef0
commit 181716fb41
12 changed files with 50 additions and 48 deletions

View File

@@ -337,7 +337,7 @@ namespace RombaSharp
} }
// Finally set all of the fields // Finally set all of the fields
Globals.MaxDegreeOfParallelism = workers; Globals.MaxThreads = workers;
_logdir = logdir; _logdir = logdir;
_tmpdir = tmpdir; _tmpdir = tmpdir;
_webdir = webdir; _webdir = webdir;

View File

@@ -18,6 +18,9 @@ namespace SabreTools.Library.Data
/// <param name="name">The name to be displayed as the program</param>B /// <param name="name">The name to be displayed as the program</param>B
public static void Start(string name) public static void Start(string name)
{ {
// Set the maximum number of threads right off the bat
System.Threading.ThreadPool.SetMaxThreads(Globals.MaxThreads, Globals.MaxThreads);
// Dynamically create the header string, adapted from http://stackoverflow.com/questions/8200661/how-to-align-string-in-fixed-length-string // Dynamically create the header string, adapted from http://stackoverflow.com/questions/8200661/how-to-align-string-in-fixed-length-string
int width = Console.WindowWidth - 3; int width = Console.WindowWidth - 3;
string border = "+" + new string('-', width) + "+"; string border = "+" + new string('-', width) + "+";

View File

@@ -36,9 +36,14 @@ namespace SabreTools.Library.Data
} }
set { _logger = value; } set { _logger = value; }
} }
public static int MaxDegreeOfParallelism public static int MaxThreads
{ {
set { _maxDegreeOfParallelism = value; } get { return _maxDegreeOfParallelism; }
set
{
_maxDegreeOfParallelism = value;
System.Threading.ThreadPool.SetMaxThreads(_maxDegreeOfParallelism, _maxDegreeOfParallelism);
}
} }
public static ParallelOptions ParallelOptions public static ParallelOptions ParallelOptions
{ {

View File

@@ -112,7 +112,7 @@ namespace SabreTools.Library.Dats
Globals.Logger.User("Processing individual DATs"); Globals.Logger.User("Processing individual DATs");
// Parse all of the DATs into their own DatFiles in the array // Parse all of the DATs into their own DatFiles in the array
Parallel.For(0, inputs.Count, Globals.ParallelOptions, i => Parallel.For(0, inputs.Count, i =>
{ {
string input = inputs[i]; string input = inputs[i];
Globals.Logger.User("Adding DAT: " + input.Split('¬')[0]); Globals.Logger.User("Adding DAT: " + input.Split('¬')[0]);
@@ -128,11 +128,11 @@ namespace SabreTools.Library.Dats
Globals.Logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); Globals.Logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
Globals.Logger.User("Populating internal DAT"); Globals.Logger.User("Populating internal DAT");
Parallel.For(0, inputs.Count, Globals.ParallelOptions, i => Parallel.For(0, inputs.Count, i =>
{ {
// Get the list of keys from the DAT // Get the list of keys from the DAT
List<string> keys = datHeaders[i].Keys.ToList(); List<string> keys = datHeaders[i].Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
// Add everything from the key to the internal DAT // Add everything from the key to the internal DAT
AddRange(key, datHeaders[i][key]); AddRange(key, datHeaders[i][key]);
@@ -179,9 +179,7 @@ namespace SabreTools.Library.Dats
Globals.Logger.User("Populating base DAT for comparison..."); Globals.Logger.User("Populating base DAT for comparison...");
List<string> baseFileNames = FileTools.GetOnlyFilesFromInputs(basePaths); List<string> baseFileNames = FileTools.GetOnlyFilesFromInputs(basePaths);
Parallel.ForEach(baseFileNames, Parallel.ForEach(baseFileNames, path =>
Globals.ParallelOptions,
path =>
{ {
Parse(path, 0, 0, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName); Parse(path, 0, 0, keep: true, clean: clean, remUnicode: remUnicode, descAsName: descAsName);
}); });
@@ -193,9 +191,7 @@ namespace SabreTools.Library.Dats
// Now we want to compare each input DAT against the base // Now we want to compare each input DAT against the base
List<string> inputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, appendparent: true); List<string> inputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, appendparent: true);
Parallel.ForEach(inputFileNames, Parallel.ForEach(inputFileNames, path =>
Globals.ParallelOptions,
path =>
{ {
// Get the two halves of the path // Get the two halves of the path
string[] splitpath = path.Split('¬'); string[] splitpath = path.Split('¬');
@@ -211,9 +207,7 @@ namespace SabreTools.Library.Dats
// Then we do a hashwise comparison against the base DAT // Then we do a hashwise comparison against the base DAT
List<string> keys = intDat.Keys.ToList(); List<string> keys = intDat.Keys.ToList();
Parallel.ForEach(keys, Parallel.ForEach(keys, key =>
Globals.ParallelOptions,
key =>
{ {
List<DatItem> datItems = intDat[key]; List<DatItem> datItems = intDat[key];
List<DatItem> keepDatItems = new List<DatItem>(); List<DatItem> keepDatItems = new List<DatItem>();
@@ -277,7 +271,7 @@ namespace SabreTools.Library.Dats
DatFile[] outDatsArray = new DatFile[inputs.Count]; DatFile[] outDatsArray = new DatFile[inputs.Count];
Parallel.For(0, inputs.Count, Globals.ParallelOptions, j => Parallel.For(0, inputs.Count, j =>
{ {
string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)"; string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
DatFile diffData; DatFile diffData;
@@ -307,7 +301,7 @@ namespace SabreTools.Library.Dats
Globals.Logger.User("Populating all output DATs"); Globals.Logger.User("Populating all output DATs");
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = DatItem.Merge(this[key]); List<DatItem> items = DatItem.Merge(this[key]);
@@ -419,7 +413,7 @@ namespace SabreTools.Library.Dats
{ {
DatFile[] outDatsArray = new DatFile[inputs.Count]; DatFile[] outDatsArray = new DatFile[inputs.Count];
Parallel.For(0, inputs.Count, Globals.ParallelOptions, j => Parallel.For(0, inputs.Count, j =>
{ {
string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)"; string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
DatFile diffData = new DatFile(this); DatFile diffData = new DatFile(this);
@@ -439,7 +433,7 @@ namespace SabreTools.Library.Dats
Globals.Logger.User("Populating all output DATs"); Globals.Logger.User("Populating all output DATs");
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = DatItem.Merge(this[key]); List<DatItem> items = DatItem.Merge(this[key]);
@@ -537,7 +531,7 @@ namespace SabreTools.Library.Dats
if (Type == "SuperDAT") if (Type == "SuperDAT")
{ {
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key].ToList(); List<DatItem> items = this[key].ToList();
List<DatItem> newItems = new List<DatItem>(); List<DatItem> newItems = new List<DatItem>();
@@ -586,7 +580,7 @@ namespace SabreTools.Library.Dats
public void Update(List<string> inputFileNames, string outDir, bool inplace, bool clean, bool remUnicode, bool descAsName, public void Update(List<string> inputFileNames, string outDir, bool inplace, bool clean, bool remUnicode, bool descAsName,
Filter filter, SplitType splitType, bool trim, bool single, string root) Filter filter, SplitType splitType, bool trim, bool single, string root)
{ {
Parallel.ForEach(inputFileNames, Globals.ParallelOptions, inputFileName => Parallel.ForEach(inputFileNames, inputFileName =>
{ {
// Clean the input string // Clean the input string
if (inputFileName != "") if (inputFileName != "")
@@ -624,7 +618,7 @@ namespace SabreTools.Library.Dats
} }
List<string> subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList(); List<string> subFiles = Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories).ToList();
Parallel.ForEach(subFiles, Globals.ParallelOptions, file => Parallel.ForEach(subFiles, file =>
{ {
Globals.Logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\""); Globals.Logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\"");
DatFile innerDatdata = new DatFile(this); DatFile innerDatdata = new DatFile(this);

View File

@@ -66,7 +66,7 @@ namespace SabreTools.Library.Dats
// Process the files in the main folder // Process the files in the main folder
List<string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.TopDirectoryOnly).ToList(); List<string> files = Directory.EnumerateFiles(basePath, "*", SearchOption.TopDirectoryOnly).ToList();
Parallel.ForEach(files, Globals.ParallelOptions, item => Parallel.ForEach(files, item =>
{ {
PopulateFromDirCheckFile(item, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType, PopulateFromDirCheckFile(item, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType,
addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst); addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst);
@@ -74,10 +74,10 @@ namespace SabreTools.Library.Dats
// Find all top-level subfolders // Find all top-level subfolders
files = Directory.EnumerateDirectories(basePath, "*", SearchOption.TopDirectoryOnly).ToList(); files = Directory.EnumerateDirectories(basePath, "*", SearchOption.TopDirectoryOnly).ToList();
Parallel.ForEach(files, Globals.ParallelOptions, item => Parallel.ForEach(files, item =>
{ {
List<string> subfiles = Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories).ToList(); List<string> subfiles = Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories).ToList();
Parallel.ForEach(subfiles, Globals.ParallelOptions, subitem => Parallel.ForEach(subfiles, subitem =>
{ {
PopulateFromDirCheckFile(subitem, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType, PopulateFromDirCheckFile(subitem, basePath, omitFromScan, bare, archivesAsFiles, enableGzip, skipFileType,
addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst); addBlanks, addDate, tempDir, copyFiles, headerToCheckAgainst);
@@ -88,7 +88,7 @@ namespace SabreTools.Library.Dats
if (!Romba && addBlanks) if (!Romba && addBlanks)
{ {
List<string> empties = FileTools.GetEmptyDirectories(basePath).ToList(); List<string> empties = FileTools.GetEmptyDirectories(basePath).ToList();
Parallel.ForEach(empties, Globals.ParallelOptions, dir => Parallel.ForEach(empties, dir =>
{ {
// Get the full path for the directory // Get the full path for the directory
string fulldir = Path.GetFullPath(dir); string fulldir = Path.GetFullPath(dir);
@@ -243,7 +243,7 @@ namespace SabreTools.Library.Dats
else else
{ {
// First take care of the found items // First take care of the found items
Parallel.ForEach(extracted, Globals.ParallelOptions, rom => Parallel.ForEach(extracted, rom =>
{ {
PopulateFromDirProcessFileHelper(newItem, PopulateFromDirProcessFileHelper(newItem,
rom, rom,
@@ -255,7 +255,7 @@ namespace SabreTools.Library.Dats
if (addBlanks) if (addBlanks)
{ {
List<string> empties = ArchiveTools.GetEmptyFoldersInArchive(newItem); List<string> empties = ArchiveTools.GetEmptyFoldersInArchive(newItem);
Parallel.ForEach(empties, Globals.ParallelOptions, empty => Parallel.ForEach(empties, empty =>
{ {
Rom emptyRom = new Rom(Path.Combine(empty, "_"), newItem, omitFromScan); Rom emptyRom = new Rom(Path.Combine(empty, "_"), newItem, omitFromScan);
PopulateFromDirProcessFileHelper(newItem, PopulateFromDirProcessFileHelper(newItem,

View File

@@ -52,7 +52,7 @@ namespace SabreTools.Library.Dats
// First do the initial sort of all of the roms // First do the initial sort of all of the roms
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> roms = this[key]; List<DatItem> roms = this[key];
@@ -143,7 +143,7 @@ namespace SabreTools.Library.Dats
// Now go through and sort all of the individual lists // Now go through and sort all of the individual lists
keys = sortable.Keys.ToList(); keys = sortable.Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
// Get the possibly unsorted list // Get the possibly unsorted list
List<DatItem> sortedlist = sortable[key]; List<DatItem> sortedlist = sortable[key];
@@ -183,7 +183,7 @@ namespace SabreTools.Library.Dats
{ {
// Loop over every key in the dictionary // Loop over every key in the dictionary
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
// For every item in the current key // For every item in the current key
List<DatItem> items = this[key]; List<DatItem> items = this[key];
@@ -235,7 +235,7 @@ namespace SabreTools.Library.Dats
// First we want to get a mapping for all games to description // First we want to get a mapping for all games to description
ConcurrentDictionary<string, string> mapping = new ConcurrentDictionary<string, string>(); ConcurrentDictionary<string, string> mapping = new ConcurrentDictionary<string, string>();
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key]; List<DatItem> items = this[key];
foreach (DatItem item in items) foreach (DatItem item in items)
@@ -250,7 +250,7 @@ namespace SabreTools.Library.Dats
// Now we loop through every item and update accordingly // Now we loop through every item and update accordingly
keys = Keys.ToList(); keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key]; List<DatItem> items = this[key];
List<DatItem> newItems = new List<DatItem>(); List<DatItem> newItems = new List<DatItem>();
@@ -305,7 +305,7 @@ namespace SabreTools.Library.Dats
// Now process all of the roms // Now process all of the roms
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key]; List<DatItem> items = this[key];
for (int j = 0; j < items.Count; j++) for (int j = 0; j < items.Count; j++)

View File

@@ -128,7 +128,7 @@ namespace SabreTools.Library.Dats
// Now loop through and get only directories from the input paths // Now loop through and get only directories from the input paths
List<string> directories = new List<string>(); List<string> directories = new List<string>();
Parallel.ForEach(inputs, Globals.ParallelOptions, input => Parallel.ForEach(inputs, input =>
{ {
// Add to the list if the input is a directory // Add to the list if the input is a directory
if (Directory.Exists(input)) if (Directory.Exists(input))

View File

@@ -85,7 +85,7 @@ namespace SabreTools.Library.Dats
// Now separate the roms accordingly // Now separate the roms accordingly
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key]; List<DatItem> items = this[key];
foreach (DatItem item in items) foreach (DatItem item in items)
@@ -307,7 +307,7 @@ namespace SabreTools.Library.Dats
// Now populate each of the DAT objects in turn // Now populate each of the DAT objects in turn
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key]; List<DatItem> items = this[key];
foreach (DatItem item in items) foreach (DatItem item in items)
@@ -418,7 +418,7 @@ namespace SabreTools.Library.Dats
keys.Sort(SplitByLevelSort); keys.Sort(SplitByLevelSort);
// Then, we loop over the games // Then, we loop over the games
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
// Here, the key is the name of the game to be used for comparison // Here, the key is the name of the game to be used for comparison
if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key)) if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key))
@@ -584,7 +584,7 @@ namespace SabreTools.Library.Dats
// Now populate each of the DAT objects in turn // Now populate each of the DAT objects in turn
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key]; List<DatItem> items = this[key];
foreach(DatItem item in items) foreach(DatItem item in items)

View File

@@ -177,7 +177,7 @@ namespace SabreTools.Library.Dats
// Loop through and add // Loop through and add
List<string> keys = Keys.ToList(); List<string> keys = Keys.ToList();
Parallel.ForEach(keys, Globals.ParallelOptions, key => Parallel.ForEach(keys, key =>
{ {
List<DatItem> items = this[key]; List<DatItem> items = this[key];
foreach(DatItem item in items) foreach(DatItem item in items)
@@ -387,7 +387,7 @@ namespace SabreTools.Library.Dats
// Make sure we have all files // Make sure we have all files
List<Tuple<string, string>> newinputs = new List<Tuple<string, string>>(); // item, basepath List<Tuple<string, string>> newinputs = new List<Tuple<string, string>>(); // item, basepath
Parallel.ForEach(inputs, Globals.ParallelOptions, input => Parallel.ForEach(inputs, input =>
{ {
if (File.Exists(input)) if (File.Exists(input))
{ {

View File

@@ -136,7 +136,7 @@ namespace SabreTools.Library.Dats
keys.Sort(new NaturalComparer()); keys.Sort(new NaturalComparer());
// Write out all required formats // Write out all required formats
Parallel.ForEach(outfiles.Keys, Globals.ParallelOptions, datFormat => Parallel.ForEach(outfiles.Keys, datFormat =>
{ {
string outfile = outfiles[datFormat]; string outfile = outfiles[datFormat];

View File

@@ -38,7 +38,7 @@ namespace SabreTools.Library.External
} }
subdirs.Clear(); subdirs.Clear();
Parallel.ForEach(dirs, Globals.ParallelOptions, currentDir => Parallel.ForEach(dirs, currentDir =>
{ {
string[] subDirs = Directory.GetDirectories(currentDir); string[] subDirs = Directory.GetDirectories(currentDir);
@@ -54,7 +54,7 @@ namespace SabreTools.Library.External
try try
{ {
FileInfo[] files = dir.GetFiles("*.*", SearchOption.TopDirectoryOnly); FileInfo[] files = dir.GetFiles("*.*", SearchOption.TopDirectoryOnly);
Parallel.ForEach(files, Globals.ParallelOptions, info => Parallel.ForEach(files, info =>
{ {
action(info); action(info);
}); });

View File

@@ -740,11 +740,11 @@ namespace SabreTools
case "--mt": case "--mt":
if (Int32.TryParse(args[++i], out int mdop)) if (Int32.TryParse(args[++i], out int mdop))
{ {
Globals.MaxDegreeOfParallelism = mdop; Globals.MaxThreads = mdop;
} }
else else
{ {
Globals.MaxDegreeOfParallelism = 4; Globals.MaxThreads = 4;
} }
break; break;
case "-n": case "-n":
@@ -1026,11 +1026,11 @@ namespace SabreTools
case "--mt": case "--mt":
if (Int32.TryParse(split[1], out int odop)) if (Int32.TryParse(split[1], out int odop))
{ {
Globals.MaxDegreeOfParallelism = odop; Globals.MaxThreads = odop;
} }
else else
{ {
Globals.MaxDegreeOfParallelism = 4; Globals.MaxThreads = 4;
} }
break; break;
case "-n": case "-n":