From 2bb7283d64e43340a0ca7f41ad4a186daaaa94d1 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Wed, 18 Jan 2017 10:20:00 -0800 Subject: [PATCH] [DatFile] Add back CMP-style rebuild, not hooked up --- RombaSharp/Partials/RombaSharp_Inits.cs | 8 +- .../Dats/Partials/DatFile.Rebuild.cs | 318 +++++++++++++++++- SabreTools/Partials/SabreTools.Inits.cs | 2 +- 3 files changed, 322 insertions(+), 6 deletions(-) diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index f159b14c..a15fe49c 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -174,7 +174,9 @@ namespace SabreTools // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1)); - need.RebuildToOutput(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false /*inverse*/, OutputFormat.TorrentGzip, true, asl, false, null, 4, _logger); + need.RebuildToOutput(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false /*set*/, false /*quickScan*/, false /*date*/, + false /*delete*/, false /*inverse*/, OutputFormat.TorrentGzip, true /*romba*/, asl, false /*updateDat*/, + null /*headerToCheckAgainst*/, 4 /*maxDegreeOfParallelism*/, _logger); } /// @@ -212,7 +214,9 @@ namespace SabreTools // Now scan all of those depots and rebuild ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1); - datFile.RebuildToOutput(onlineDepots, outputFolder, _tmpdir, true, false, false, false /*inverse*/, (copy ? OutputFormat.TorrentGzip : OutputFormat.TorrentZip), copy, asl, false, null, 4, _logger); + datFile.RebuildToOutput(onlineDepots, outputFolder, _tmpdir, true /*set*/, true /*quickscan*/, false /*date*/, + false /*delete*/, false /*inverse*/, (copy ? OutputFormat.TorrentGzip : OutputFormat.TorrentZip), copy, asl, + false /*updateDat*/, null /*headerToCheckAgainst*/, 4 /*maxDegreeOfParallelism*/, _logger); } } diff --git a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs index 4289a07f..42047734 100644 --- a/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs +++ b/SabreTools.Helper/Dats/Partials/DatFile.Rebuild.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using SabreTools.Helper.Data; using SabreTools.Helper.Skippers; using SabreTools.Helper.Tools; +using SharpCompress.Common; #if MONO using System.IO; @@ -27,6 +28,7 @@ namespace SabreTools.Helper.Dats /// List of input files/folders to check /// Output directory to use to build to /// Temporary directory for archive extraction + /// True to enable set building output, false otherwise /// True to enable external scanning of archives, false otherwise /// True if the date from the DAT should be used if available, false otherwise /// True if input files should be deleted, false otherwise @@ -38,7 +40,7 @@ namespace SabreTools.Helper.Dats /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise /// Logger object for file and console output /// True if rebuilding was a success, false otherwise - public bool RebuildToOutput(List inputs, string outDir, string tempDir, bool quickScan, bool date, + public bool RebuildToOutput(List inputs, string outDir, string tempDir, bool set, bool quickScan, bool date, bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) { @@ -79,6 +81,316 @@ namespace SabreTools.Helper.Dats #endregion + bool success = true; + + // Now choose the correct rebuilder + if (set) + { + success = RebuildToOutputWithSets(inputs, outDir, tempDir, quickScan, date, delete, inverse, + outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger); + } + else + { + success = RebuildToOutputWithoutSets(inputs, outDir, tempDir, quickScan, date, delete, inverse, + outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger); + } + + return success; + } + + /// + /// Rebuild sets using CMP-style linear rebuilding + /// + /// List of input files/folders to check + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise + /// True if input files should be deleted, false otherwise + /// True if the DAT should be used as a filter instead of a template, false otherwise + /// Output format that files should be written to + /// True if files should be output in Romba depot folders, false otherwise + /// ArchiveScanLevel representing the archive handling levels + /// True if the updated DAT should be output, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if rebuilding was a success, false otherwise + private bool RebuildToOutputWithoutSets(List inputs, string outDir, string tempDir, bool quickScan, bool date, + bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, + string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) + { + bool success = true; + + #region Rebuild from sources in order + + switch (outputFormat) + { + case OutputFormat.Folder: + logger.User("Rebuilding all files to directory"); + break; + case OutputFormat.TapeArchive: + logger.User("Rebuilding all files to TAR"); + break; + case OutputFormat.Torrent7Zip: + logger.User("Rebuilding all files to Torrent7Z"); + break; + case OutputFormat.TorrentGzip: + logger.User("Rebuilding all files to TorrentGZ"); + break; + case OutputFormat.TorrentLrzip: + logger.User("Rebuilding all files to TorrentLRZ"); + break; + case OutputFormat.TorrentRar: + logger.User("Rebuilding all files to TorrentRAR"); + break; + case OutputFormat.TorrentXZ: + logger.User("Rebuilding all files to TorrentXZ"); + break; + case OutputFormat.TorrentZip: + logger.User("Rebuilding all files to TorrentZip"); + break; + } + DateTime start = DateTime.Now; + + // Now loop through all of the files in all of the inputs + foreach (string input in inputs) + { + // If the input is a file + if (File.Exists(input)) + { + RebuildToOutputWithoutSetsHelper(input, outDir, tempDir, quickScan, date, delete, inverse, + outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger); + } + + // If the input is a directory + else if (Directory.Exists(input)) + { + List files = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList(); + foreach (string file in files) + { + RebuildToOutputWithoutSetsHelper(file, outDir, tempDir, quickScan, date, delete, inverse, + outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger); + } + } + } + + logger.User("Rebuilding complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + #endregion + + return success; + } + + /// + /// Attempt to add a file to the output if it matches + /// + /// Name of the file to process + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise + /// True if input files should be deleted, false otherwise + /// True if the DAT should be used as a filter instead of a template, false otherwise + /// Output format that files should be written to + /// True if files should be output in Romba depot folders, false otherwise + /// ArchiveScanLevel representing the archive handling levels + /// True if the updated DAT should be output, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + private void RebuildToOutputWithoutSetsHelper(string file, string outDir, string tempDir, bool quickScan, bool date, + bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, + string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) + { + // If we somehow have a null filename, return + if (file == null) + { + return; + } + + // Define the temporary directory + string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar; + + // Get the required scanning level for the file + bool shouldExternalProcess = false; + bool shouldInternalProcess = false; + ArchiveTools.GetInternalExternalProcess(file, archiveScanLevel, logger, out shouldExternalProcess, out shouldInternalProcess); + + // If we're supposed to scan the file externally + if (shouldExternalProcess) + { + Rom rom = FileTools.GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); + RebuildToOutputWithoutSetsIndividual(rom, file, outDir, tempSubDir, date, inverse, outputFormat, + romba, updateDat, false /* isZip */, headerToCheckAgainst, logger); + } + + // If we're supposed to scan the file internally + if (shouldInternalProcess) + { + // If quickscan is set, do so + if (quickScan) + { + List extracted = ArchiveTools.GetArchiveFileInfo(file, logger); + + foreach (Rom rom in extracted) + { + RebuildToOutputWithoutSetsIndividual(rom, file, outDir, tempSubDir, date, inverse, outputFormat, + romba, updateDat, true /* isZip */, headerToCheckAgainst, logger); + } + } + // Otherwise, attempt to extract the files to the temporary directory + else + { + bool encounteredErrors = ArchiveTools.ExtractArchive(file, tempSubDir, archiveScanLevel, logger); + + // If the file was an archive and was extracted successfully, check it + if (!encounteredErrors) + { + logger.Verbose(Path.GetFileName(file) + " treated like an archive"); + List extracted = Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories).ToList(); + foreach (string entry in extracted) + { + Rom rom = FileTools.GetFileInfo(entry, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); + RebuildToOutputWithoutSetsIndividual(rom, file, outDir, tempSubDir, date, inverse, outputFormat, + romba, updateDat, false /* isZip */, headerToCheckAgainst, logger); + } + } + // Otherwise, just get the info on the file itself + else if (File.Exists(file)) + { + Rom rom = FileTools.GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); + RebuildToOutputWithoutSetsIndividual(rom, file, outDir, tempSubDir, date, inverse, outputFormat, + romba, updateDat, false /* isZip */, headerToCheckAgainst, logger); + } + } + } + + // Now delete the temp directory + try + { + Directory.Delete(tempSubDir, true); + } + catch { } + } + + /// + /// Find duplicates and rebuild individual files to output + /// + /// Information for the current file to rebuild from + /// Name of the file to process + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True if the date from the DAT should be used if available, false otherwise + /// True if the DAT should be used as a filter instead of a template, false otherwise + /// Output format that files should be written to + /// True if files should be output in Romba depot folders, false otherwise + /// True if the updated DAT should be output, false otherwise + /// True if the input file is an archive, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + private void RebuildToOutputWithoutSetsIndividual(Rom rom, string file, string outDir, string tempDir, bool date, + bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, bool isZip, string headerToCheckAgainst, Logger logger) + { + // Find if the file has duplicates in the DAT + bool hasDuplicates = rom.HasDuplicates(this, logger); + + // If it has duplicates and we're not filtering or we have no duplicates and we're filtering, rebuild it + if (hasDuplicates ^ inverse) + { + // Get the list of duplicates to rebuild to + List dupes = rom.GetDuplicates(this, logger, remove: updateDat); + + // If we don't have any duplicates, continue + if (dupes.Count == 0) + { + return; + } + + // If we have an archive input, get the real name of the file to use + if (isZip) + { + // Otherwise, extract the file to the temp folder + file = ArchiveTools.ExtractItem(file, rom.Name, tempDir, logger); + } + + // If we couldn't extract the file, then continue, + if (String.IsNullOrEmpty(file)) + { + return; + } + + // Now loop through the list and rebuild accordingly + foreach (Rom item in dupes) + { + switch (outputFormat) + { + case OutputFormat.Folder: + string outfile = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(item.Machine.Name), item.Name); + + // Make sure the output folder is created + Directory.CreateDirectory(Path.GetDirectoryName(outfile)); + + // Now copy the file over + try + { + File.Copy(file, outfile); + if (date && !String.IsNullOrEmpty(item.Date)) + { + File.SetCreationTime(outfile, DateTime.Parse(item.Date)); + } + } + catch { } + + break; + case OutputFormat.TapeArchive: + ArchiveTools.WriteTAR(file, outDir, item, logger, date: date); + break; + case OutputFormat.Torrent7Zip: + break; + case OutputFormat.TorrentGzip: + ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger); + break; + case OutputFormat.TorrentLrzip: + break; + case OutputFormat.TorrentRar: + break; + case OutputFormat.TorrentXZ: + break; + case OutputFormat.TorrentZip: + ArchiveTools.WriteTorrentZip(file, outDir, item, logger, date: date); + break; + } + + // And now clear the temp folder to get rid of any transient files + try + { + Directory.Delete(tempDir, true); + } + catch { } + } + } + } + + /// + /// Rebuild sets using RV-style set rebuilding + /// + /// List of input files/folders to check + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise + /// True if input files should be deleted, false otherwise + /// True if the DAT should be used as a filter instead of a template, false otherwise + /// Output format that files should be written to + /// True if files should be output in Romba depot folders, false otherwise + /// ArchiveScanLevel representing the archive handling levels + /// True if the updated DAT should be output, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if rebuilding was a success, false otherwise + private bool RebuildToOutputWithSets(List inputs, string outDir, string tempDir, bool quickScan, bool date, + bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, + string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger) + { bool success = true; DatFile matched = new DatFile(); List files = new List(); @@ -193,8 +505,8 @@ namespace SabreTools.Helper.Dats rom.Machine = new Machine(Path.GetFullPath(file), ""); current.Add(rom.Size + "-" + rom.CRC, rom); - // If we had a header, we want the full file information too - if (headerToCheckAgainst != null) + // If we had a header, we want the full file information too + if (headerToCheckAgainst != null) { rom = FileTools.GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan); rom.Machine = new Machine(Path.GetFullPath(file), ""); diff --git a/SabreTools/Partials/SabreTools.Inits.cs b/SabreTools/Partials/SabreTools.Inits.cs index 49cdfbeb..8c4c7495 100644 --- a/SabreTools/Partials/SabreTools.Inits.cs +++ b/SabreTools/Partials/SabreTools.Inits.cs @@ -332,7 +332,7 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - datdata.RebuildToOutput(inputs, outDir, tempDir, quickScan, date, delete, inverse, outputFormat, romba, asl, + datdata.RebuildToOutput(inputs, outDir, tempDir, true /*set*/, quickScan, date, delete, inverse, outputFormat, romba, asl, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, _logger); }