diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index fd367bff..66d83f84 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -174,7 +174,7 @@ namespace SabreTools // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1)); - need.RebuildFromInputs(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false /*quickScan*/, false /*date*/, + need.RebuildGeneric(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false /*quickScan*/, false /*date*/, false /*delete*/, false /*inverse*/, OutputFormat.TorrentGzip, true /*romba*/, asl, false /*updateDat*/, null /*headerToCheckAgainst*/, 4 /*maxDegreeOfParallelism*/, _logger); } @@ -214,7 +214,7 @@ namespace SabreTools // Now scan all of those depots and rebuild ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1); - datFile.RebuildFromDepot(onlineDepots, outputFolder, _tmpdir, false /*date*/, + datFile.RebuildDepot(onlineDepots, outputFolder, _tmpdir, false /*date*/, false /*delete*/, false /*inverse*/, (copy ? OutputFormat.TorrentGzip : OutputFormat.TorrentZip), copy, false /*updateDat*/, null /*headerToCheckAgainst*/, 4 /*maxDegreeOfParallelism*/, _logger); } diff --git a/SabreTools.Helper/Data/Build.cs b/SabreTools.Helper/Data/Build.cs index ca3e6b3e..861c5cee 100644 --- a/SabreTools.Helper/Data/Build.cs +++ b/SabreTools.Helper/Data/Build.cs @@ -459,13 +459,12 @@ namespace SabreTools.Helper.Data } // Verify Depot - if (/*subset == null* ||*/ subset == "ved" || subset == "verify-depot") + if (subset == null || subset == "ved" || subset == "verify-depot") { helptext.Add(""); helptext.Add(" -ved, --verify-depot Verify a folder against DATs"); helptext.Add(" -dat= Input DAT to verify against"); helptext.Add(" -t=, --temp= Set the temporary directory to use"); - helptext.Add(" -qs, --quick Enable quick scanning of archives"); helptext.Add(" -h=, --header= Set a header skipper to use, blank means all"); } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs index 4cab7321..2faab23d 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs @@ -35,7 +35,7 @@ namespace SabreTools.Helper.Dats /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// Logger object for file and console output /// True if rebuilding was a success, false otherwise - public bool RebuildFromDepot(List inputs, string outDir, string tempDir, bool date, bool delete, + public bool RebuildDepot(List inputs, string outDir, string tempDir, bool date, bool delete, bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) { @@ -172,7 +172,6 @@ namespace SabreTools.Helper.Dats } // Otherwise, we rebuild that file to all locations that we need to - RebuildIndividualFile(fileinfo, foundpath, outDir, tempDir, date, inverse, outputFormat, romba, updateDat, true /*isZip*/, headerToCheckAgainst, logger); } @@ -209,7 +208,7 @@ namespace SabreTools.Helper.Dats /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// Logger object for file and console output /// True if rebuilding was a success, false otherwise - public bool RebuildFromInputs(List inputs, string outDir, string tempDir, bool quickScan, bool date, + public bool RebuildGeneric(List inputs, string outDir, string tempDir, bool quickScan, bool date, bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) { @@ -290,7 +289,7 @@ namespace SabreTools.Helper.Dats if (File.Exists(input)) { logger.User("Checking file: '" + input + "'"); - RebuildFromInputsHelper(input, outDir, tempDir, quickScan, date, delete, inverse, + RebuildGenericHelper(input, outDir, tempDir, quickScan, date, delete, inverse, outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger); } @@ -301,7 +300,7 @@ namespace SabreTools.Helper.Dats foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories)) { logger.User("Checking file: '" + file + "'"); - RebuildFromInputsHelper(file, outDir, tempDir, quickScan, date, delete, inverse, + RebuildGenericHelper(file, outDir, tempDir, quickScan, date, delete, inverse, outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger); } } @@ -339,7 +338,7 @@ namespace SabreTools.Helper.Dats /// True if the updated DAT should be output, false otherwise /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// Logger object for file and console output - private void RebuildFromInputsHelper(string file, string outDir, string tempDir, bool quickScan, bool date, + private void RebuildGenericHelper(string file, string outDir, string tempDir, bool quickScan, bool date, bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) { @@ -740,6 +739,116 @@ namespace SabreTools.Helper.Dats return rebuilt; } + /// + /// Process the DAT and verify from the depots + /// + /// List of input directories to compare against + /// Temporary directory for archive extraction + /// True if only hashes should be checked, false for full file information + /// True to enable external scanning of archives, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if verification was a success, false otherwise + public bool VerifyDepot(List inputs, string tempDir, string headerToCheckAgainst, Logger logger) + { + // Check the temp directory + if (String.IsNullOrEmpty(tempDir)) + { + tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + } + + // Then create or clean the temp directory + if (!Directory.Exists(tempDir)) + { + Directory.CreateDirectory(tempDir); + } + else + { + FileTools.CleanDirectory(tempDir); + } + + bool success = true; + + logger.User("Verifying all from supplied depots"); + DateTime start = DateTime.Now; + + // Now loop through and get only directories from the input paths + List directories = new List(); + foreach (string input in inputs) + { + // Add to the list if the input is a directory + if (Directory.Exists(input)) + { + logger.Verbose("Adding depot: '" + input + "'"); + directories.Add(input); + } + } + + // If we don't have any directories, we want to exit + if (directories.Count == 0) + { + return success; + } + + // Now that we have a list of depots, we want to sort the input DAT by SHA-1 + BucketBySHA1(false, logger, output: false); + + // Then we want to loop through each of the hashes and see if we can rebuild + List hashes = Keys.ToList(); + foreach (string hash in hashes) + { + // Pre-empt any issues that could arise from string length + if (hash.Length != Constants.SHA1Length) + { + continue; + } + + logger.User("Checking hash '" + hash + "'"); + + // Get the extension path for the hash + string subpath = Style.GetRombaPath(hash); + + // Find the first depot that includes the hash + string foundpath = null; + foreach (string directory in directories) + { + if (File.Exists(Path.Combine(directory, subpath))) + { + foundpath = Path.Combine(directory, subpath); + break; + } + } + + // If we didn't find a path, then we continue + if (foundpath == null) + { + continue; + } + + // If we have a path, we want to try to get the rom information + Rom fileinfo = ArchiveTools.GetTorrentGZFileInfo(foundpath, logger); + + // If the file information is null, then we continue + if (fileinfo == null) + { + continue; + } + + // Now we want to remove all duplicates from the DAT + fileinfo.GetDuplicates(this, logger, remove: true); + } + + logger.User("Verifying complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + // If there are any entries in the DAT, output to the rebuild directory + _fileName = "fixDAT_" + _fileName; + _name = "fixDAT_" + _name; + _description = "fixDAT_" + _description; + WriteToFile(null, logger); + + return success; + } + /// /// Process the DAT and verify the output directory /// @@ -750,7 +859,7 @@ namespace SabreTools.Helper.Dats /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// Logger object for file and console output /// True if verification was a success, false otherwise - public bool VerifyDirectory(List inputs, string tempDir, bool hashOnly, bool quickScan, string headerToCheckAgainst, Logger logger) + public bool VerifyGeneric(List inputs, string tempDir, bool hashOnly, bool quickScan, string headerToCheckAgainst, Logger logger) { // Check the temp directory exists if (String.IsNullOrEmpty(tempDir)) diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs index 77902fb3..78ea6bf6 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Writers.cs @@ -49,7 +49,7 @@ namespace SabreTools.Helper.Dats } // If output directory is empty, use the current folder - if (outDir.Trim() == "") + if (outDir == null || outDir.Trim() == "") { logger.Verbose("No output directory defined, defaulting to curent folder"); outDir = Environment.CurrentDirectory; diff --git a/SabreTools/Partials/SabreTools.Inits.cs b/SabreTools/Partials/SabreTools.Inits.cs index e3a55d68..8d48ff65 100644 --- a/SabreTools/Partials/SabreTools.Inits.cs +++ b/SabreTools/Partials/SabreTools.Inits.cs @@ -332,7 +332,7 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - datdata.RebuildFromInputs(inputs, outDir, tempDir, quickScan, date, delete, inverse, outputFormat, romba, asl, + datdata.RebuildGeneric(inputs, outDir, tempDir, quickScan, date, delete, inverse, outputFormat, romba, asl, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, _logger); } @@ -365,7 +365,7 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - datdata.RebuildFromDepot(inputs, outDir, tempDir, date, delete, inverse, outputFormat, romba, + datdata.RebuildDepot(inputs, outDir, tempDir, date, delete, inverse, outputFormat, romba, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, _logger); } @@ -841,7 +841,30 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - datdata.VerifyDirectory(inputs, tempDir, hashOnly, quickScan, headerToCheckAgainst, _logger); + datdata.VerifyGeneric(inputs, tempDir, hashOnly, quickScan, headerToCheckAgainst, _logger); + } + + /// + /// Wrap verifying files from a depot using an input DAT + /// + /// Names of the DATs to compare against + /// Input directories to compare against + /// Temporary directory for archive extraction + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + private static void InitVerifyDepot(List datfiles, List inputs, string tempDir, string headerToCheckAgainst) + { + DateTime start = DateTime.Now; + _logger.User("Populating internal DAT..."); + + // Add all of the input DATs into one huge internal DAT + DatFile datdata = new DatFile(); + foreach (string datfile in datfiles) + { + datdata.Parse(datfile, 99, 99, _logger, keep: true, softlist: true); + } + _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + datdata.VerifyDepot(inputs, tempDir, headerToCheckAgainst, _logger); } #endregion diff --git a/SabreTools/SabreTools.cs b/SabreTools/SabreTools.cs index 8765a143..e936d154 100644 --- a/SabreTools/SabreTools.cs +++ b/SabreTools/SabreTools.cs @@ -65,7 +65,8 @@ namespace SabreTools splitByType = false, stats = false, update = false, - verify = false; + verify = false, + verifyDepot = false; // User flags bool addBlankFilesForEmptyFolder = false, @@ -222,6 +223,10 @@ namespace SabreTools case "--verify": verify = true; break; + case "-ved": + case "--verify-depot": + verifyDepot = true; + break; // User flags case "-ab": @@ -988,7 +993,7 @@ namespace SabreTools } // If none of the feature flags is enabled, show the help screen - if (!(datFromDir | extract | restore | sort | sortDepot | splitByExt | splitByHash | splitByLevel | splitByType | stats | update | verify)) + if (!(datFromDir | extract | restore | sort | sortDepot | splitByExt | splitByHash | splitByLevel | splitByType | stats | update | verify | verifyDepot)) { _logger.Error("At least one feature switch must be enabled"); _logger.Close(); @@ -1005,7 +1010,7 @@ namespace SabreTools // If a switch that requires a filename is set and no file is, show the help screen if (inputs.Count == 0 - && (datFromDir || extract || restore || splitByExt || splitByHash || splitByLevel || splitByType || stats || update)) + && (datFromDir || extract || restore || splitByExt || splitByHash || splitByLevel || splitByType || stats || update || verify || verifyDepot)) { _logger.Error("This feature requires at least one input"); _logger.Close(); @@ -1116,6 +1121,12 @@ namespace SabreTools InitVerify(datfiles, inputs, tempDir, hashOnly, quickScan, header); } + // If we're using the depot verifier + else if (verifyDepot) + { + InitVerifyDepot(datfiles, inputs, tempDir, header); + } + // If nothing is set, show the help else {