claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DATFromDirParallel] Add limit to parallelism

Browse Source
This commit is contained in:
Matt Nadareski
2016-09-06 17:41:44 -07:00
parent fd931043ac
commit 8f3b8a3362
4 changed files with 27 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
SabreTools.Helper/Data/Build.cs
View File

@@ -144,6 +144,7 @@ namespace SabreTools.Helper
helptext.Add(" -au=, --author= Set the author of the DAT");
helptext.Add(" -sd, --superdat Enable SuperDAT creation");
helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -mt={-1} Amount of threads to use (-1 unlimted)");
helptext.Add(" -es, --ext-split Split a DAT by two file extensions");
helptext.Add(" -exta= First set of extensions (comma-separated)");
helptext.Add(" -extb= Second set of extensions (comma-separated)");

19
SabreTools.Helper/Objects/DATFromDirParallel.cs
View File

@@ -26,6 +26,7 @@ namespace SabreTools
private bool _archivesAsFiles;
private bool _enableGzip;
private bool _addblanks;
private int _maxParallelism;
// Other required variables
private Logger _logger;
@@ -47,9 +48,10 @@ namespace SabreTools
/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory)</param>
/// <param name="nowrite">True if the file should not be written out, false otherwise (default)</param>
/// <param name="maxParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for console and file output</param>
public DATFromDirParallel(string basePath, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger)
public DATFromDirParallel(string basePath, Dat datdata, bool noMD5, bool noSHA1, bool bare,
bool archivesAsFiles, bool enableGzip, string tempDir, int maxParallelism, Logger logger)
{
_basePath = Path.GetFullPath(basePath);
_datdata = datdata;
@@ -61,6 +63,7 @@ namespace SabreTools
_enableGzip = enableGzip;
_addblanks = true;
_tempDir = tempDir;
_maxParallelism = maxParallelism;
_logger = logger;
}
@@ -93,7 +96,9 @@ namespace SabreTools
_logger.Log("Folder found: " + _basePath);
// Process the files in all subfolders
Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.AllDirectories), item =>
Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.AllDirectories),
new ParallelOptions { MaxDegreeOfParallelism = 4 },
item =>
{
ProcessPossibleArchive(item);
});
@@ -101,7 +106,9 @@ namespace SabreTools
// Now find all folders that are empty, if we are supposed to
if (_addblanks)
{
Parallel.ForEach(Directory.EnumerateDirectories(_basePath, "*", SearchOption.AllDirectories), dir =>
Parallel.ForEach(Directory.EnumerateDirectories(_basePath, "*", SearchOption.AllDirectories),
new ParallelOptions { MaxDegreeOfParallelism = 4 },
dir =>
{
if (Directory.EnumerateFiles(dir, "*", SearchOption.TopDirectoryOnly).Count() == 0)
{
@@ -239,7 +246,9 @@ namespace SabreTools
if (!encounteredErrors)
{
_logger.Log(Path.GetFileName(item) + " treated like an archive");
Parallel.ForEach(Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories), entry =>
Parallel.ForEach(Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories),
new ParallelOptions { MaxDegreeOfParallelism = 4 },
entry =>
{
ProcessFile(entry, tempSubDir, Path.GetFileNameWithoutExtension(item), _datdata);
});

8
SabreTools/Partials/SabreTools_Inits.cs
View File

@@ -424,7 +424,7 @@ namespace SabreTools
/// <summary>
/// Wrap creating a DAT file from files or a directory in parallel
/// </summary>
/// <param name="input">List of innput filenames</param>
/// <param name="inputs">List of input filenames</param>
/// <param name="filename">New filename</param>
/// <param name="name">New name</param>
/// <param name="description">New description</param>
@@ -441,6 +441,7 @@ namespace SabreTools
/// <param name="archivesAsFiles">True if archives should be treated as files, false otherwise</param>
/// <param name="enableGzip">True if GZIP archives should be treated as files, false otherwise</param>
/// <param name="tempDir">Name of the directory to create a temp folder in (blank is current directory</param>
/// <param name="maxParallelism">Integer representing the maximum amount of parallelization to be used</param>
private static void InitDatFromDirParallel(List<string> inputs,
string filename,
string name,
@@ -457,7 +458,8 @@ namespace SabreTools
bool bare,
bool archivesAsFiles,
bool enableGzip,
string tempDir)
string tempDir,
int maxParallelism)
{
// Create a new DATFromDir object and process the inputs
Dat datdata = new Dat
@@ -482,7 +484,7 @@ namespace SabreTools
if (Directory.Exists(path))
{
string basePath = Path.GetFullPath(path);
DATFromDirParallel dfd = new DATFromDirParallel(basePath, datdata, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempDir, _logger);
DATFromDirParallel dfd = new DATFromDirParallel(basePath, datdata, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempDir, maxParallelism, _logger);
bool success = dfd.Start();
// For DFDParallel only

8
SabreTools/SabreTools.cs
View File

@@ -126,6 +126,7 @@ namespace SabreTools
nodump = null,
tsv = null;
DiffMode diff = 0x0;
int maxParallelism = -1;
long sgt = -1,
slt = -1,
seq = -1;
@@ -502,6 +503,10 @@ namespace SabreTools
{
md5 = temparg.Split('=')[1];
}
else if (temparg.StartsWith("-mt=") || temparg.StartsWith("--mt="))
{
Int32.TryParse(temparg.Split('=')[1], out maxParallelism);
}
else if (temparg.StartsWith("-n=") || temparg.StartsWith("--name="))
{
name = temparg.Split('=')[1];
@@ -740,7 +745,8 @@ namespace SabreTools
// Create a DAT from a directory or set of directories in parallel
else if (datfromdirparallel)
{
InitDatFromDirParallel(inputs, filename, name, description, category, version, author, forceunpack, old, romba, superdat, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempdir);
InitDatFromDirParallel(inputs, filename, name, description, category, version, author,
forceunpack, old, romba, superdat, noMD5, noSHA1, bare, archivesAsFiles, enableGzip, tempdir, maxParallelism);
}
// If we want to run Offline merging mode