claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

[DatFile] Add verify depot

Browse Source
This commit is contained in:
Matt Nadareski
2017-01-31 23:18:41 -08:00
parent 047f7cc31f
commit bceb99d8e4
6 changed files with 160 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
RombaSharp/Partials/RombaSharp_Inits.cs
View File

@@ -174,7 +174,7 @@ namespace SabreTools
// Create the sorting object to use and rebuild the needed files
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1));
need.RebuildFromInputs(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false /*quickScan*/, false /*date*/,
need.RebuildGeneric(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false /*quickScan*/, false /*date*/,
false /*delete*/, false /*inverse*/, OutputFormat.TorrentGzip, true /*romba*/, asl, false /*updateDat*/,
null /*headerToCheckAgainst*/, 4 /*maxDegreeOfParallelism*/, _logger);
}
@@ -214,7 +214,7 @@ namespace SabreTools
// Now scan all of those depots and rebuild
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1);
datFile.RebuildFromDepot(onlineDepots, outputFolder, _tmpdir, false /*date*/,
datFile.RebuildDepot(onlineDepots, outputFolder, _tmpdir, false /*date*/,
false /*delete*/, false /*inverse*/, (copy ? OutputFormat.TorrentGzip : OutputFormat.TorrentZip), copy,
false /*updateDat*/, null /*headerToCheckAgainst*/, 4 /*maxDegreeOfParallelism*/, _logger);
}

3
SabreTools.Helper/Data/Build.cs
View File

@@ -459,13 +459,12 @@ namespace SabreTools.Helper.Data
}
// Verify Depot
if (/*subset == null* ||*/ subset == "ved" || subset == "verify-depot")
if (subset == null || subset == "ved" || subset == "verify-depot")
{
helptext.Add("");
helptext.Add(" -ved, --verify-depot Verify a folder against DATs");
helptext.Add(" -dat= Input DAT to verify against");
helptext.Add(" -t=, --temp= Set the temporary directory to use");
helptext.Add(" -qs, --quick Enable quick scanning of archives");
helptext.Add(" -h=, --header= Set a header skipper to use, blank means all");
}

123
SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs
View File

@@ -35,7 +35,7 @@ namespace SabreTools.Helper.Dats
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public bool RebuildFromDepot(List<string> inputs, string outDir, string tempDir, bool date, bool delete,
public bool RebuildDepot(List<string> inputs, string outDir, string tempDir, bool date, bool delete,
bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, string headerToCheckAgainst,
int maxDegreeOfParallelism, Logger logger)
{
@@ -172,7 +172,6 @@ namespace SabreTools.Helper.Dats
}
// Otherwise, we rebuild that file to all locations that we need to
RebuildIndividualFile(fileinfo, foundpath, outDir, tempDir, date, inverse, outputFormat, romba, updateDat, true /*isZip*/, headerToCheckAgainst, logger);
}
@@ -209,7 +208,7 @@ namespace SabreTools.Helper.Dats
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if rebuilding was a success, false otherwise</returns>
public bool RebuildFromInputs(List<string> inputs, string outDir, string tempDir, bool quickScan, bool date,
public bool RebuildGeneric(List<string> inputs, string outDir, string tempDir, bool quickScan, bool date,
bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat,
string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger)
{
@@ -290,7 +289,7 @@ namespace SabreTools.Helper.Dats
if (File.Exists(input))
{
logger.User("Checking file: '" + input + "'");
RebuildFromInputsHelper(input, outDir, tempDir, quickScan, date, delete, inverse,
RebuildGenericHelper(input, outDir, tempDir, quickScan, date, delete, inverse,
outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger);
}
@@ -301,7 +300,7 @@ namespace SabreTools.Helper.Dats
foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
{
logger.User("Checking file: '" + file + "'");
RebuildFromInputsHelper(file, outDir, tempDir, quickScan, date, delete, inverse,
RebuildGenericHelper(file, outDir, tempDir, quickScan, date, delete, inverse,
outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger);
}
}
@@ -339,7 +338,7 @@ namespace SabreTools.Helper.Dats
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
private void RebuildFromInputsHelper(string file, string outDir, string tempDir, bool quickScan, bool date,
private void RebuildGenericHelper(string file, string outDir, string tempDir, bool quickScan, bool date,
bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat,
string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger)
{
@@ -740,6 +739,116 @@ namespace SabreTools.Helper.Dats
return rebuilt;
}
/// <summary>
/// Process the DAT and verify from the depots
/// </summary>
/// <param name="inputs">List of input directories to compare against</param>
/// <param name="tempDir">Temporary directory for archive extraction</param>
/// <param name="hashOnly">True if only hashes should be checked, false for full file information</param>
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if verification was a success, false otherwise</returns>
public bool VerifyDepot(List<string> inputs, string tempDir, string headerToCheckAgainst, Logger logger)
{
// Check the temp directory
if (String.IsNullOrEmpty(tempDir))
{
tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
}
// Then create or clean the temp directory
if (!Directory.Exists(tempDir))
{
Directory.CreateDirectory(tempDir);
}
else
{
FileTools.CleanDirectory(tempDir);
}
bool success = true;
logger.User("Verifying all from supplied depots");
DateTime start = DateTime.Now;
// Now loop through and get only directories from the input paths
List<string> directories = new List<string>();
foreach (string input in inputs)
{
// Add to the list if the input is a directory
if (Directory.Exists(input))
{
logger.Verbose("Adding depot: '" + input + "'");
directories.Add(input);
}
}
// If we don't have any directories, we want to exit
if (directories.Count == 0)
{
return success;
}
// Now that we have a list of depots, we want to sort the input DAT by SHA-1
BucketBySHA1(false, logger, output: false);
// Then we want to loop through each of the hashes and see if we can rebuild
List<string> hashes = Keys.ToList();
foreach (string hash in hashes)
{
// Pre-empt any issues that could arise from string length
if (hash.Length != Constants.SHA1Length)
{
continue;
}
logger.User("Checking hash '" + hash + "'");
// Get the extension path for the hash
string subpath = Style.GetRombaPath(hash);
// Find the first depot that includes the hash
string foundpath = null;
foreach (string directory in directories)
{
if (File.Exists(Path.Combine(directory, subpath)))
{
foundpath = Path.Combine(directory, subpath);
break;
}
}
// If we didn't find a path, then we continue
if (foundpath == null)
{
continue;
}
// If we have a path, we want to try to get the rom information
Rom fileinfo = ArchiveTools.GetTorrentGZFileInfo(foundpath, logger);
// If the file information is null, then we continue
if (fileinfo == null)
{
continue;
}
// Now we want to remove all duplicates from the DAT
fileinfo.GetDuplicates(this, logger, remove: true);
}
logger.User("Verifying complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// If there are any entries in the DAT, output to the rebuild directory
_fileName = "fixDAT_" + _fileName;
_name = "fixDAT_" + _name;
_description = "fixDAT_" + _description;
WriteToFile(null, logger);
return success;
}
/// <summary>
/// Process the DAT and verify the output directory
/// </summary>
@@ -750,7 +859,7 @@ namespace SabreTools.Helper.Dats
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
/// <param name="logger">Logger object for file and console output</param>
/// <returns>True if verification was a success, false otherwise</returns>
public bool VerifyDirectory(List<string> inputs, string tempDir, bool hashOnly, bool quickScan, string headerToCheckAgainst, Logger logger)
public bool VerifyGeneric(List<string> inputs, string tempDir, bool hashOnly, bool quickScan, string headerToCheckAgainst, Logger logger)
{
// Check the temp directory exists
if (String.IsNullOrEmpty(tempDir))

2
SabreTools.Helper/Dats/Partials/DatFile.Writers.cs
View File

@@ -49,7 +49,7 @@ namespace SabreTools.Helper.Dats
}
// If output directory is empty, use the current folder
if (outDir.Trim() == "")
if (outDir == null || outDir.Trim() == "")
{
logger.Verbose("No output directory defined, defaulting to curent folder");
outDir = Environment.CurrentDirectory;

29
SabreTools/Partials/SabreTools.Inits.cs
View File

@@ -332,7 +332,7 @@ namespace SabreTools
}
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
datdata.RebuildFromInputs(inputs, outDir, tempDir, quickScan, date, delete, inverse, outputFormat, romba, asl,
datdata.RebuildGeneric(inputs, outDir, tempDir, quickScan, date, delete, inverse, outputFormat, romba, asl,
updateDat, headerToCheckAgainst, maxDegreeOfParallelism, _logger);
}
@@ -365,7 +365,7 @@ namespace SabreTools
}
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
datdata.RebuildFromDepot(inputs, outDir, tempDir, date, delete, inverse, outputFormat, romba,
datdata.RebuildDepot(inputs, outDir, tempDir, date, delete, inverse, outputFormat, romba,
updateDat, headerToCheckAgainst, maxDegreeOfParallelism, _logger);
}
@@ -841,7 +841,30 @@ namespace SabreTools
}
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
datdata.VerifyDirectory(inputs, tempDir, hashOnly, quickScan, headerToCheckAgainst, _logger);
datdata.VerifyGeneric(inputs, tempDir, hashOnly, quickScan, headerToCheckAgainst, _logger);
}
/// <summary>
/// Wrap verifying files from a depot using an input DAT
/// </summary>
/// <param name="datfiles">Names of the DATs to compare against</param>
/// <param name="inputs">Input directories to compare against</param>
/// <param name="tempDir">Temporary directory for archive extraction</param>
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
private static void InitVerifyDepot(List<string> datfiles, List<string> inputs, string tempDir, string headerToCheckAgainst)
{
DateTime start = DateTime.Now;
_logger.User("Populating internal DAT...");
// Add all of the input DATs into one huge internal DAT
DatFile datdata = new DatFile();
foreach (string datfile in datfiles)
{
datdata.Parse(datfile, 99, 99, _logger, keep: true, softlist: true);
}
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
datdata.VerifyDepot(inputs, tempDir, headerToCheckAgainst, _logger);
}
#endregion

17
SabreTools/SabreTools.cs
View File

@@ -65,7 +65,8 @@ namespace SabreTools
splitByType = false,
stats = false,
update = false,
verify = false;
verify = false,
verifyDepot = false;
// User flags
bool addBlankFilesForEmptyFolder = false,
@@ -222,6 +223,10 @@ namespace SabreTools
case "--verify":
verify = true;
break;
case "-ved":
case "--verify-depot":
verifyDepot = true;
break;
// User flags
case "-ab":
@@ -988,7 +993,7 @@ namespace SabreTools
}
// If none of the feature flags is enabled, show the help screen
if (!(datFromDir | extract | restore | sort | sortDepot | splitByExt | splitByHash | splitByLevel | splitByType | stats | update | verify))
if (!(datFromDir | extract | restore | sort | sortDepot | splitByExt | splitByHash | splitByLevel | splitByType | stats | update | verify | verifyDepot))
{
_logger.Error("At least one feature switch must be enabled");
_logger.Close();
@@ -1005,7 +1010,7 @@ namespace SabreTools
// If a switch that requires a filename is set and no file is, show the help screen
if (inputs.Count == 0
&& (datFromDir || extract || restore || splitByExt || splitByHash || splitByLevel || splitByType || stats || update))
&& (datFromDir || extract || restore || splitByExt || splitByHash || splitByLevel || splitByType || stats || update || verify || verifyDepot))
{
_logger.Error("This feature requires at least one input");
_logger.Close();
@@ -1116,6 +1121,12 @@ namespace SabreTools
InitVerify(datfiles, inputs, tempDir, hashOnly, quickScan, header);
}
// If we're using the depot verifier
else if (verifyDepot)
{
InitVerifyDepot(datfiles, inputs, tempDir, header);
}
// If nothing is set, show the help
else
{