claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DatFile] More parallelization to fix build

Browse Source
This commit is contained in:
Matt Nadareski
2017-03-01 20:36:22 -08:00
parent 98bbbfa5d9
commit 8c4a4ac01a
3 changed files with 40 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
SabreTools.Helper/Dats/Partials/DatFile.Splitters.cs
View File

@@ -26,9 +26,10 @@ namespace SabreTools.Helper.Dats
/// <param name="basepath">Parent path for replacement</param> /// <param name="basepath">Parent path for replacement</param>
/// <param name="extA">List of extensions to split on (first DAT)</param> /// <param name="extA">List of extensions to split on (first DAT)</param>
/// <param name="extB">List of extensions to split on (second DAT)</param> /// <param name="extB">List of extensions to split on (second DAT)</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for console and file writing</param> /// <param name="logger">Logger object for console and file writing</param>
/// <returns>True if split succeeded, false otherwise</returns> /// <returns>True if split succeeded, false otherwise</returns>
public bool SplitByExt(string outDir, string basepath, List<string> extA, List<string> extB, Logger logger) public bool SplitByExt(string outDir, string basepath, List<string> extA, List<string> extB, int maxDegreeOfParallelism, Logger logger)
{ {
// Make sure all of the extensions have a dot at the beginning // Make sure all of the extensions have a dot at the beginning
List<string> newExtA = new List<string>(); List<string> newExtA = new List<string>();
@@ -115,8 +116,8 @@ namespace SabreTools.Helper.Dats
} }
// Then write out both files // Then write out both files
bool success = datdataA.WriteToFile(outDir, logger); bool success = datdataA.WriteToFile(outDir, maxDegreeOfParallelism, logger);
success &= datdataB.WriteToFile(outDir, logger); success &= datdataB.WriteToFile(outDir, maxDegreeOfParallelism, logger);
return success; return success;
} }
@@ -126,9 +127,10 @@ namespace SabreTools.Helper.Dats
/// </summary> /// </summary>
/// <param name="outDir">Name of the directory to write the DATs out to</param> /// <param name="outDir">Name of the directory to write the DATs out to</param>
/// <param name="basepath">Parent path for replacement</param> /// <param name="basepath">Parent path for replacement</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for console and file writing</param> /// <param name="logger">Logger object for console and file writing</param>
/// <returns>True if split succeeded, false otherwise</returns> /// <returns>True if split succeeded, false otherwise</returns>
public bool SplitByHash(string outDir, string basepath, Logger logger) public bool SplitByHash(string outDir, string basepath, int maxDegreeOfParallelism, Logger logger)
{ {
// Sanitize the basepath to be more predictable // Sanitize the basepath to be more predictable
basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar); basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
@@ -299,10 +301,10 @@ namespace SabreTools.Helper.Dats
// Now, output all of the files to the output directory // Now, output all of the files to the output directory
logger.User("DAT information created, outputting new files"); logger.User("DAT information created, outputting new files");
bool success = true; bool success = true;
success &= nodump.WriteToFile(outDir, logger); success &= nodump.WriteToFile(outDir, maxDegreeOfParallelism, logger);
success &= sha1.WriteToFile(outDir, logger); success &= sha1.WriteToFile(outDir, maxDegreeOfParallelism, logger);
success &= md5.WriteToFile(outDir, logger); success &= md5.WriteToFile(outDir, maxDegreeOfParallelism, logger);
success &= crc.WriteToFile(outDir, logger); success &= crc.WriteToFile(outDir, maxDegreeOfParallelism, logger);
return success; return success;
} }
@@ -314,15 +316,16 @@ namespace SabreTools.Helper.Dats
/// <param name="basepath">Parent path for replacement</param> /// <param name="basepath">Parent path for replacement</param>
/// <param name="shortname">True if short names should be used, false otherwise</param> /// <param name="shortname">True if short names should be used, false otherwise</param>
/// <param name="basedat">True if original filenames should be used as the base for output filename, false otherwise</param> /// <param name="basedat">True if original filenames should be used as the base for output filename, false otherwise</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for console and file writing</param> /// <param name="logger">Logger object for console and file writing</param>
/// <returns>True if split succeeded, false otherwise</returns> /// <returns>True if split succeeded, false otherwise</returns>
public bool SplitByLevel(string outDir, string basepath, bool shortname, bool basedat, Logger logger) public bool SplitByLevel(string outDir, string basepath, bool shortname, bool basedat, int maxDegreeOfParallelism, Logger logger)
{ {
// Sanitize the basepath to be more predictable // Sanitize the basepath to be more predictable
basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar); basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
// First, organize by games so that we can do the right thing // First, organize by games so that we can do the right thing
BucketBy(SortedBy.Game, false /* mergeroms */, logger, lower: false, norename: true); BucketBy(SortedBy.Game, false /* mergeroms */, maxDegreeOfParallelism, logger, lower: false, norename: true);
// Create a temporary DAT to add things to // Create a temporary DAT to add things to
DatFile tempDat = new DatFile(this) DatFile tempDat = new DatFile(this)
@@ -341,7 +344,7 @@ namespace SabreTools.Helper.Dats
if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key)) if (tempDat.Name != null && tempDat.Name != Style.GetDirectoryName(key))
{ {
// Process and output the DAT // Process and output the DAT
SplitByLevelHelper(tempDat, outDir, shortname, basedat, logger); SplitByLevelHelper(tempDat, outDir, shortname, basedat, maxDegreeOfParallelism, logger);
// Reset the DAT for the next items // Reset the DAT for the next items
tempDat = new DatFile(this) tempDat = new DatFile(this)
@@ -363,7 +366,7 @@ namespace SabreTools.Helper.Dats
} }
// Then we write the last DAT out since it would be skipped otherwise // Then we write the last DAT out since it would be skipped otherwise
SplitByLevelHelper(tempDat, outDir, shortname, basedat, logger); SplitByLevelHelper(tempDat, outDir, shortname, basedat, maxDegreeOfParallelism, logger);
return true; return true;
} }
@@ -394,8 +397,9 @@ namespace SabreTools.Helper.Dats
/// <param name="outDir">Directory to write out to</param> /// <param name="outDir">Directory to write out to</param>
/// <param name="shortname">True if short naming scheme should be used, false otherwise</param> /// <param name="shortname">True if short naming scheme should be used, false otherwise</param>
/// <param name="restore">True if original filenames should be used as the base for output filename, false otherwise</param> /// <param name="restore">True if original filenames should be used as the base for output filename, false otherwise</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for file and console output</param> /// <param name="logger">Logger object for file and console output</param>
private void SplitByLevelHelper(DatFile datFile, string outDir, bool shortname, bool restore, Logger logger) private void SplitByLevelHelper(DatFile datFile, string outDir, bool shortname, bool restore, int maxDegreeOfParallelism, Logger logger)
{ {
// Get the name from the DAT to use separately // Get the name from the DAT to use separately
string name = datFile.Name; string name = datFile.Name;
@@ -420,7 +424,7 @@ namespace SabreTools.Helper.Dats
datFile.Type = null; datFile.Type = null;
// Write out the temporary DAT to the proper directory // Write out the temporary DAT to the proper directory
datFile.WriteToFile(path, logger); datFile.WriteToFile(path, maxDegreeOfParallelism, logger);
} }
/// <summary> /// <summary>
@@ -428,9 +432,10 @@ namespace SabreTools.Helper.Dats
/// </summary> /// </summary>
/// <param name="outDir">Name of the directory to write the DATs out to</param> /// <param name="outDir">Name of the directory to write the DATs out to</param>
/// <param name="basepath">Parent path for replacement</param> /// <param name="basepath">Parent path for replacement</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for console and file writing</param> /// <param name="logger">Logger object for console and file writing</param>
/// <returns>True if split succeeded, false otherwise</returns> /// <returns>True if split succeeded, false otherwise</returns>
public bool SplitByType(string outDir, string basepath, Logger logger) public bool SplitByType(string outDir, string basepath, int maxDegreeOfParallelism, Logger logger)
{ {
// Sanitize the basepath to be more predictable // Sanitize the basepath to be more predictable
basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar); basepath = (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()) ? basepath : basepath + Path.DirectorySeparatorChar);
@@ -539,9 +544,9 @@ namespace SabreTools.Helper.Dats
// Now, output all of the files to the output directory // Now, output all of the files to the output directory
logger.User("DAT information created, outputting new files"); logger.User("DAT information created, outputting new files");
bool success = true; bool success = true;
success &= romdat.WriteToFile(outDir, logger); success &= romdat.WriteToFile(outDir, maxDegreeOfParallelism, logger);
success &= diskdat.WriteToFile(outDir, logger); success &= diskdat.WriteToFile(outDir, maxDegreeOfParallelism, logger);
success &= sampledat.WriteToFile(outDir, logger); success &= sampledat.WriteToFile(outDir, maxDegreeOfParallelism, logger);
return success; return success;
} }

24
SabreTools.Helper/Dats/Partials/DatFile.Statistics.cs
View File

@@ -102,13 +102,14 @@ namespace SabreTools.Helper.Dats
/// </summary> /// </summary>
/// <param name="outputs">Dictionary representing the outputs</param> /// <param name="outputs">Dictionary representing the outputs</param>
/// <param name="statDatFormat">Set the statistics output format to use</param> /// <param name="statDatFormat">Set the statistics output format to use</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for file and console writing</param> /// <param name="logger">Logger object for file and console writing</param>
/// <param name="recalculate">True if numbers should be recalculated for the DAT, false otherwise (default)</param> /// <param name="recalculate">True if numbers should be recalculated for the DAT, false otherwise (default)</param>
/// <param name="game">Number of games to use, -1 means recalculate games (default)</param> /// <param name="game">Number of games to use, -1 means recalculate games (default)</param>
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise (default)</param> /// <param name="baddumpCol">True if baddumps should be included in output, false otherwise (default)</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise (default)</param> /// <param name="nodumpCol">True if nodumps should be included in output, false otherwise (default)</param>
public void OutputStats(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat, Logger logger, bool recalculate = false, public void OutputStats(Dictionary<StatDatFormat, StreamWriter> outputs, StatDatFormat statDatFormat, int maxDegreeOfParallelism, Logger logger,
long game = -1, bool baddumpCol = false, bool nodumpCol = false) bool recalculate = false, long game = -1, bool baddumpCol = false, bool nodumpCol = false)
{ {
// If we're supposed to recalculate the statistics, do so // If we're supposed to recalculate the statistics, do so
if (recalculate) if (recalculate)
@@ -116,7 +117,7 @@ namespace SabreTools.Helper.Dats
RecalculateStats(); RecalculateStats();
} }
BucketBy(SortedBy.Game, false /* mergeroms */, logger, norename: true); BucketBy(SortedBy.Game, false /* mergeroms */, maxDegreeOfParallelism, logger, norename: true);
if (TotalSize < 0) if (TotalSize < 0)
{ {
TotalSize = Int64.MaxValue + TotalSize; TotalSize = Int64.MaxValue + TotalSize;
@@ -262,9 +263,10 @@ namespace SabreTools.Helper.Dats
/// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param> /// <param name="baddumpCol">True if baddumps should be included in output, false otherwise</param>
/// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param> /// <param name="nodumpCol">True if nodumps should be included in output, false otherwise</param>
/// <param name="statDatFormat" > Set the statistics output format to use</param> /// <param name="statDatFormat" > Set the statistics output format to use</param>
/// <param name="maxDegreeOfParallelism">Integer representing the maximum amount of parallelization to be used</param>
/// <param name="logger">Logger object for file and console output</param> /// <param name="logger">Logger object for file and console output</param>
public static void OutputStats(List<string> inputs, string reportName, string outDir, bool single, public static void OutputStats(List<string> inputs, string reportName, string outDir, bool single,
bool baddumpCol, bool nodumpCol, StatDatFormat statDatFormat, Logger logger) bool baddumpCol, bool nodumpCol, StatDatFormat statDatFormat, int maxDegreeOfParallelism, Logger logger)
{ {
// If there's no output format, set the default // If there's no output format, set the default
if (statDatFormat == 0x0) if (statDatFormat == 0x0)
@@ -362,7 +364,8 @@ namespace SabreTools.Helper.Dats
BaddumpCount = dirBaddump, BaddumpCount = dirBaddump,
NodumpCount = dirNodump, NodumpCount = dirNodump,
}; };
lastdirdat.OutputStats(outputs, statDatFormat, logger, game: dirGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol); lastdirdat.OutputStats(outputs, statDatFormat, maxDegreeOfParallelism, logger,
game: dirGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol);
// Write the mid-footer, if any // Write the mid-footer, if any
OutputStatsWriteMidFooter(outputs, statDatFormat, baddumpCol, nodumpCol); OutputStatsWriteMidFooter(outputs, statDatFormat, baddumpCol, nodumpCol);
@@ -387,13 +390,14 @@ namespace SabreTools.Helper.Dats
List<string> games = new List<string>(); List<string> games = new List<string>();
DatFile datdata = new DatFile(); DatFile datdata = new DatFile();
datdata.Parse(filename.Item1, 0, 0, logger); datdata.Parse(filename.Item1, 0, 0, logger);
datdata.BucketBy(SortedBy.Game, false /* mergeroms */, logger, norename: true); datdata.BucketBy(SortedBy.Game, false /* mergeroms */, maxDegreeOfParallelism, logger, norename: true);
// Output single DAT stats (if asked) // Output single DAT stats (if asked)
logger.User("Adding stats for file '" + filename.Item1 + "'\n", false); logger.User("Adding stats for file '" + filename.Item1 + "'\n", false);
if (single) if (single)
{ {
datdata.OutputStats(outputs, statDatFormat, logger, baddumpCol: baddumpCol, nodumpCol: nodumpCol); datdata.OutputStats(outputs, statDatFormat, maxDegreeOfParallelism, logger,
baddumpCol: baddumpCol, nodumpCol: nodumpCol);
} }
// Add single DAT stats to dir // Add single DAT stats to dir
@@ -442,7 +446,8 @@ namespace SabreTools.Helper.Dats
BaddumpCount = dirBaddump, BaddumpCount = dirBaddump,
NodumpCount = dirNodump, NodumpCount = dirNodump,
}; };
dirdat.OutputStats(outputs, statDatFormat, logger, game: dirGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol); dirdat.OutputStats(outputs, statDatFormat, maxDegreeOfParallelism, logger,
game: dirGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol);
} }
// Write the mid-footer, if any // Write the mid-footer, if any
@@ -476,7 +481,8 @@ namespace SabreTools.Helper.Dats
BaddumpCount = totalBaddump, BaddumpCount = totalBaddump,
NodumpCount = totalNodump, NodumpCount = totalNodump,
}; };
totaldata.OutputStats(outputs, statDatFormat, logger, game: totalGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol); totaldata.OutputStats(outputs, statDatFormat, maxDegreeOfParallelism, logger,
game: totalGame, baddumpCol: baddumpCol, nodumpCol: nodumpCol);
// Output footer if needed // Output footer if needed
OutputStatsWriteFooter(outputs, statDatFormat); OutputStatsWriteFooter(outputs, statDatFormat);

3
SabreTools.Helper/Dats/Partials/DatFile.Writers.cs
View File

@@ -107,7 +107,8 @@ namespace SabreTools.Helper.Dats
// Output initial statistics, for kicks // Output initial statistics, for kicks
if (stats) if (stats)
{ {
OutputStats(new Dictionary<StatDatFormat, StreamWriter>(), StatDatFormat.None, logger, recalculate: (RomCount + DiskCount == 0), baddumpCol: true, nodumpCol: true); OutputStats(new Dictionary<StatDatFormat, StreamWriter>(), StatDatFormat.None, maxDegreeOfParallelism, logger,
recalculate: (RomCount + DiskCount == 0), baddumpCol: true, nodumpCol: true);
} }
// Bucket roms by game name and optionally dedupe // Bucket roms by game name and optionally dedupe