From 2366116d98c50f0ca98c25dba2c8767e7b12a446 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Mon, 24 Oct 2016 21:32:26 -0700 Subject: [PATCH] [DatFile] Move Convert/Rebuild/Verify to DatFile, overhaul Rebuild --- RombaSharp/Partials/RombaSharp_Inits.cs | 4 +- SabreTools.Helper/Dats/DatFile.cs | 613 ++++++++++++- SabreTools.Helper/Dats/DatItem.cs | 23 +- SabreTools.Helper/Tools/ArchiveTools.cs | 5 +- SabreTools.Helper/Tools/FileTools.cs | 1073 ----------------------- SabreTools/Partials/SabreTools_Inits.cs | 6 +- SimpleSort/SimpleSort.cs | 8 +- 7 files changed, 630 insertions(+), 1102 deletions(-) diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index ab9fe29e..b52f4e2b 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -188,7 +188,7 @@ namespace SabreTools // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1)); - FileTools.RebuildToOutput(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, 4, _logger); + need.RebuildToOutput(onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, 4, _logger); } /// @@ -226,7 +226,7 @@ namespace SabreTools // Now scan all of those depots and rebuild ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1); - FileTools.RebuildToOutput(datFile, onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, 4, _logger); + datFile.RebuildToOutput(onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, 4, _logger); } } diff --git a/SabreTools.Helper/Dats/DatFile.cs b/SabreTools.Helper/Dats/DatFile.cs index 1845cb62..803c906f 100644 --- a/SabreTools.Helper/Dats/DatFile.cs +++ b/SabreTools.Helper/Dats/DatFile.cs @@ -1590,13 +1590,17 @@ namespace SabreTools.Helper.Dats } // Then populate it with information - item.Machine.Name = tempgamename; - item.Machine.Description = gamedesc; - item.Machine.CloneOf = cloneof; - item.Machine.RomOf = romof; - item.Machine.SampleOf = sampleof; - item.Machine.Manufacturer = manufacturer; - item.Machine.Year = year; + item.Machine = new Machine + { + Name = tempgamename, + Description = gamedesc, + CloneOf = cloneof, + RomOf = romof, + SampleOf = sampleof, + Manufacturer = manufacturer, + Year = year, + }; + item.SystemID = sysid; item.SourceID = srcid; @@ -4266,6 +4270,601 @@ namespace SabreTools.Helper.Dats #endregion + #region Rebuilding and Verifying + + /// + /// Process inputs and convert to TorrentZip or TorrentGZ, optionally converting to Romba format + /// + /// List of inputs to convert over to TorrentZip or TorrentGZ + /// Output folder to rebuild to, blank is the current directory + /// Temporary directory to use in file extraction + /// True if files should be output in TorrentGZ format, false for TorrentZip + /// True if TorrentGZ files should be output in romba depot format, false otherwise + /// True if input files should be deleted, false otherwise + /// ArchiveScanLevel representing how files should be treated + /// Logger object for file and console output + /// True if processing was a success, false otherwise + public bool ConvertFiles(List inputs, string outDir, string tempDir, bool tgz, + bool romba, bool delete, ArchiveScanLevel archiveScanLevel, Logger logger) + { + bool success = true; + + // First, check that the output directory exists + if (!Directory.Exists(outDir)) + { + Directory.CreateDirectory(outDir); + outDir = Path.GetFullPath(outDir); + } + + // Then create or clean the temp directory + if (!Directory.Exists(tempDir)) + { + Directory.CreateDirectory(tempDir); + } + else + { + FileTools.CleanDirectory(tempDir); + } + + // Now process all of the inputs + foreach (string input in inputs) + { + logger.User("Examining file " + input); + + // Get if the file should be scanned internally and externally + bool shouldExternalProcess, shouldInternalProcess; + ArchiveTools.GetInternalExternalProcess(input, archiveScanLevel, logger, out shouldExternalProcess, out shouldInternalProcess); + + // Do an external scan of the file, if necessary + if (shouldExternalProcess) + { + // If a DAT is defined, we want to make sure that this file is not in there + Rom rom = FileTools.GetFileInfo(input, logger); + if (this != null && Files.Count > 0) + { + if (rom.HasDuplicates(this, logger)) + { + logger.User("File '" + input + "' existed in the DAT, skipping..."); + continue; + } + } + + logger.User("Processing file " + input); + + if (tgz) + { + success &= ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); + } + else + { + success &= ArchiveTools.WriteToArchive(input, outDir, rom, logger); + } + } + + // Process the file as an archive, if necessary + if (shouldInternalProcess) + { + // Now, if the file is a supported archive type, also run on all files within + bool encounteredErrors = ArchiveTools.ExtractArchive(input, tempDir, archiveScanLevel, logger); + + // If no errors were encountered, we loop through the temp directory + if (!encounteredErrors) + { + logger.Verbose("Archive found! Successfully extracted"); + foreach (string file in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) + { + // If a DAT is defined, we want to make sure that this file is not in there + Rom rom = FileTools.GetFileInfo(file, logger); + if (this != null && Files.Count > 0) + { + if (rom.HasDuplicates(this, logger)) + { + logger.User("File '" + file + "' existed in the DAT, skipping..."); + continue; + } + } + + logger.User("Processing file " + input); + + if (tgz) + { + success &= ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger); + } + else + { + success &= ArchiveTools.WriteToArchive(file, outDir, rom, logger); + } + } + + FileTools.CleanDirectory(tempDir); + } + } + + // Delete the source file if we're supposed to + if (delete) + { + try + { + logger.User("Attempting to delete " + input); + File.Delete(input); + } + catch (Exception ex) + { + logger.Error(ex.ToString()); + success &= false; + } + } + } + + // Now one final delete of the temp directory + while (Directory.Exists(tempDir)) + { + try + { + Directory.Delete(tempDir, true); + } + catch + { + continue; + } + } + + // If we're in romba mode and the size file doesn't exist, create it + if (romba && !File.Exists(Path.Combine(outDir, ".romba_size"))) + { + // Get the size of all of the files in the output folder + long size = 0; + foreach (string file in Directory.EnumerateFiles(outDir, "*", SearchOption.AllDirectories)) + { + FileInfo tempinfo = new FileInfo(file); + size += tempinfo.Length; + } + + // Write out the value to each of the romba depot files + StreamWriter tw = new StreamWriter(File.Open(Path.Combine(outDir, ".romba_size"), FileMode.Create, FileAccess.Write)); + StreamWriter twb = new StreamWriter(File.Open(Path.Combine(outDir, ".romba_size.backup"), FileMode.Create, FileAccess.Write)); + + tw.Write(size); + twb.Write(size); + + tw.Dispose(); + twb.Dispose(); + } + + return success; + } + + /// + /// Process the DAT and find all matches in input files and folders + /// + /// List of input files/folders to check + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise + /// True if files should be output to folder, false otherwise + /// True if input files should be deleted, false otherwise + /// True if output files should be written to TorrentGZ instead of TorrentZip + /// True if files should be output in Romba depot folders, false otherwise + /// ArchiveScanLevel representing the archive handling levels + /// True if the updated DAT should be output, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if rebuilding was a success, false otherwise + public bool RebuildToOutput(List inputs, string outDir, string tempDir, bool quickScan, bool date, + bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, + int maxDegreeOfParallelism, Logger logger) + { + // First, check that the output directory exists + if (!Directory.Exists(outDir)) + { + Directory.CreateDirectory(outDir); + outDir = Path.GetFullPath(outDir); + } + + // Check the temp directory + if (String.IsNullOrEmpty(tempDir)) + { + tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); + } + + // Then create or clean the temp directory + if (!Directory.Exists(tempDir)) + { + Directory.CreateDirectory(tempDir); + } + else + { + FileTools.CleanDirectory(tempDir); + } + + bool success = true; + DatFile matched = new DatFile(); + List files = new List(); + + #region Retrieve a list of all files + + logger.User("Retrieving list all files from input"); + DateTime start = DateTime.Now; + + // Create a list of just files from inputs + Parallel.ForEach(inputs, + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, + input => { + if (File.Exists(input)) + { + logger.Verbose("File found: '" + input + "'"); + files.Add(Path.GetFullPath(input)); + } + else if (Directory.Exists(input)) + { + logger.Verbose("Directory found: '" + input + "'"); + Parallel.ForEach(Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories), + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, + file => + { + logger.Verbose("File found: '" + file + "'"); + files.Add(Path.GetFullPath(file)); + }); + } + else + { + logger.Error("'" + input + "' is not a file or directory!"); + } + }); + logger.User("Retrieving complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + #endregion + + DatFile current = new DatFile(); + + #region Create a dat from input files + + logger.User("Getting hash information for all input files"); + start = DateTime.Now; + + // Now that we have a list of just files, we get a DAT from the input files + foreach (string file in files) + { + // Define the temporary directory + string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar; + + // Get the required scanning level for the file + bool shouldExternalProcess = false; + bool shouldInternalProcess = false; + ArchiveTools.GetInternalExternalProcess(file, archiveScanLevel, logger, out shouldExternalProcess, out shouldInternalProcess); + + // If we're supposed to scan the file externally + if (shouldExternalProcess) + { + Rom rom = FileTools.GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); + rom.Name = Path.GetFullPath(file); + + string key = rom.Size + "-" + rom.CRC; + if (current.Files.ContainsKey(key)) + { + current.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + current.Files.Add(key, temp); + } + } + + // If we're supposed to scan the file internally + if (shouldInternalProcess) + { + // If quickscan is set, do so + if (quickScan) + { + List extracted = ArchiveTools.GetArchiveFileInfo(file, logger); + + foreach (Rom rom in extracted) + { + Rom newrom = rom; + newrom.Machine = new Machine(Path.GetFullPath(file), ""); + + string key = rom.Size + "-" + rom.CRC; + if (current.Files.ContainsKey(key)) + { + current.Files[key].Add(newrom); + } + else + { + List temp = new List(); + temp.Add(newrom); + current.Files.Add(key, temp); + } + } + } + // Otherwise, attempt to extract the files to the temporary directory + else + { + bool encounteredErrors = ArchiveTools.ExtractArchive(file, tempSubDir, archiveScanLevel, logger); + + // If the file was an archive and was extracted successfully, check it + if (!encounteredErrors) + { + logger.Verbose(Path.GetFileName(file) + " treated like an archive"); + List extracted = Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories).ToList(); + Parallel.ForEach(extracted, + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, + entry => + { + Rom rom = FileTools.GetFileInfo(entry, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); + rom.Machine = new Machine(Path.GetFullPath(file), ""); + + string key = rom.Size + "-" + rom.CRC; + if (current.Files.ContainsKey(key)) + { + current.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + current.Files.Add(key, temp); + } + }); + } + // Otherwise, just get the info on the file itself + else if (File.Exists(file)) + { + Rom rom = FileTools.GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); + rom.Name = Path.GetFullPath(file); + + string key = rom.Size + "-" + rom.CRC; + if (current.Files.ContainsKey(key)) + { + current.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + current.Files.Add(key, temp); + } + } + } + } + + // Now delete the temp directory + try + { + Directory.Delete(tempSubDir, true); + } + catch { } + } + + logger.User("Getting hash information complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + #endregion + + // Create a mapping from destination file to source file + Dictionary toFromMap = new Dictionary(); + + #region Find all required files for rebuild + + logger.User("Determining files to rebuild"); + start = DateTime.Now; + + // Order the DATs by hash first to make things easier + BucketByCRC(false, logger, output: false); + current.BucketByCRC(false, logger, output: false); + + // Now loop over and find all files that need to be rebuilt + List keys = current.Files.Keys.ToList(); + foreach (string key in keys) + { + // If the input DAT doesn't have the key, then nothing from the current DAT are there + if (!Files.ContainsKey(key)) + { + continue; + } + + // Otherwise, we try to find duplicates + List datItems = current.Files[key]; + foreach (Rom rom in datItems) + { + List found = rom.GetDuplicates(this, logger, false); + + // Now add all of the duplicates mapped to the current file + foreach (Rom mid in found) + { + try + { + toFromMap.Add(mid, rom); + } + catch { } + } + } + } + + logger.User("Determining complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + #endregion + + // Now bucket the list of keys by game so that we can rebuild properly + SortedDictionary> keysGroupedByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, logger, output: false); + + #region Rebuild games in order + + logger.User("Rebuilding all files to " + (toFolder ? "directory" : (tgz ? "TorrentGZ" : "TorrentZip"))); + start = DateTime.Now; + + // Now loop through the keys and create the correct output items + List games = keysGroupedByGame.Keys.ToList(); + foreach (string game in games) + { + // Define the temporary directory + string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar; + + // Create an empty list for getting paths for rebuilding + List pathsToFiles = new List(); + + // Loop through all of the matched items in the game + List itemsInGame = keysGroupedByGame[game]; + List romsInGame = new List(); + foreach (Rom rom in itemsInGame) + { + // Get the rom that's mapped to this item + Rom source = (Rom)toFromMap[rom]; + + // If the file is in an archive, we need to treat it specially + string machinename = source.Machine.Name.ToLowerInvariant(); + if (machinename.EndsWith(".7z") + || machinename.EndsWith(".gz") + || machinename.EndsWith(".rar") + || machinename.EndsWith(".zip")) + { + pathsToFiles.Add(ArchiveTools.ExtractItem(source.Machine.Name, Path.GetFileName(source.Name), tempSubDir, logger)); + } + + // Otherwise, we want to just add the full path + else + { + pathsToFiles.Add(source.Name); + } + + // Now add the rom to the output list + romsInGame.Add(rom); + } + + // And now rebuild accordingly + if (toFolder) + { + for (int i = 0; i < romsInGame.Count; i++) + { + string infile = pathsToFiles[i]; + Rom outrom = romsInGame[i]; + string outfile = Path.Combine(outDir, outrom.Machine.Name, outrom.Machine.Name); + + // Make sure the output folder is created + Directory.CreateDirectory(Path.GetDirectoryName(outfile)); + + // Now copy the file over + try + { + File.Copy(infile, outfile); + } + catch { } + } + } + else if (tgz) + { + for (int i = 0; i < itemsInGame.Count; i++) + { + string infile = pathsToFiles[i]; + Rom outrom = romsInGame[i]; + ArchiveTools.WriteTorrentGZ(infile, outDir, romba, logger); + } + } + else + { + ArchiveTools.WriteToArchive(pathsToFiles, outDir, romsInGame, logger); + } + + // And now clear the temp folder to get rid of any transient files + try + { + Directory.Delete(tempSubDir, true); + } + catch { } + } + + logger.User("Rebuilding complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + #endregion + + return success; + } + + /// + /// Process the DAT and verify the output directory + /// + /// DAT to use to verify the directory + /// List of input directories to compare against + /// Temporary directory for archive extraction + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if verification was a success, false otherwise + public bool VerifyDirectory(List inputs, string tempDir, string headerToCheckAgainst, Logger logger) + { + // First create or clean the temp directory + if (!Directory.Exists(tempDir)) + { + Directory.CreateDirectory(tempDir); + } + else + { + FileTools.CleanDirectory(tempDir); + } + + bool success = true; + + /* + We want the cross section of what's the folder and what's in the DAT. Right now, it just has what's in the DAT that's not in the folder + */ + + // Then, loop through and check each of the inputs + logger.User("Processing files:\n"); + foreach (string input in inputs) + { + PopulateDatFromDir(input, false /* noMD5 */, false /* noSHA1 */, true /* bare */, false /* archivesAsFiles */, + true /* enableGzip */, false /* addBlanks */, false /* addDate */, tempDir /* tempDir */, false /* copyFiles */, + headerToCheckAgainst, 4 /* maxDegreeOfParallelism */, logger); + } + + // Setup the fixdat + DatFile matched = (DatFile)CloneHeader(); + matched.Files = new SortedDictionary>(); + matched.FileName = "fixDat_" + matched.FileName; + matched.Name = "fixDat_" + matched.Name; + matched.Description = "fixDat_" + matched.Description; + matched.OutputFormat = OutputFormat.Logiqx; + + // Now that all files are parsed, get only files found in directory + bool found = false; + foreach (List roms in Files.Values) + { + List newroms = DatItem.Merge(roms, logger); + foreach (Rom rom in newroms) + { + if (rom.SourceID == 99) + { + found = true; + string key = rom.Size + "-" + rom.CRC; + if (matched.Files.ContainsKey(key)) + { + matched.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + matched.Files.Add(key, temp); + } + } + } + } + + // Now output the fixdat to the main folder + if (found) + { + matched.WriteToFile("", logger, stats: true); + } + else + { + logger.User("No fixDat needed"); + } + + return success; + } + + #endregion + #region Splitting /// diff --git a/SabreTools.Helper/Dats/DatItem.cs b/SabreTools.Helper/Dats/DatItem.cs index a2be9657..ccce3ef6 100644 --- a/SabreTools.Helper/Dats/DatItem.cs +++ b/SabreTools.Helper/Dats/DatItem.cs @@ -845,16 +845,16 @@ namespace SabreTools.Helper.Dats /// True if it sorted correctly, false otherwise public static bool Sort(ref List roms, bool norename) { - try + roms.Sort(delegate (DatItem x, DatItem y) { - roms.Sort(delegate (DatItem x, DatItem y) + try { NaturalComparer nc = new NaturalComparer(); if (x.SystemID == y.SystemID) { if (x.SourceID == y.SourceID) { - if (x.Machine.Name == y.Machine.Name) + if (x.Machine != null && y.Machine != null && x.Machine.Name == y.Machine.Name) { if ((x.Type == ItemType.Rom || x.Type == ItemType.Disk) && (y.Type == ItemType.Rom || y.Type == ItemType.Disk)) { @@ -886,14 +886,15 @@ namespace SabreTools.Helper.Dats return (norename ? nc.Compare(x.Machine.Name, y.Machine.Name) : x.SourceID - y.SourceID); } return (norename ? nc.Compare(x.Machine.Name, y.Machine.Name) : x.SystemID - y.SystemID); - }); - return true; - } - catch (Exception) - { - // Absorb the error - return false; - } + } + catch (Exception) + { + // Absorb the error + return 0; + } + }); + + return true; } #endregion diff --git a/SabreTools.Helper/Tools/ArchiveTools.cs b/SabreTools.Helper/Tools/ArchiveTools.cs index 1e1690b3..73ff16b1 100644 --- a/SabreTools.Helper/Tools/ArchiveTools.cs +++ b/SabreTools.Helper/Tools/ArchiveTools.cs @@ -205,7 +205,7 @@ namespace SabreTools.Helper.Tools string realEntry = ""; Stream ms = ExtractStream(input, entryName, out realEntry, logger); - realEntry = Path.GetFullPath(Path.Combine(tempDir, realEntry)); + realEntry = Path.Combine(Path.GetFullPath(tempDir), realEntry); if (!Directory.Exists(Path.GetDirectoryName(realEntry))) { Directory.CreateDirectory(Path.GetDirectoryName(realEntry)); @@ -217,6 +217,7 @@ namespace SabreTools.Helper.Tools int ilen; while ((ilen = ms.Read(ibuffer, 0, _bufferSize)) > 0) { + fs.Write(ibuffer, 0, ilen); fs.Flush(); } @@ -786,7 +787,7 @@ namespace SabreTools.Helper.Tools } // Get the output archive name from the first rebuild rom - string archiveFileName = Path.Combine(outDir, roms[0].Machine.Name + ".zip"); + string archiveFileName = Path.Combine(outDir, roms[0].Machine.Name + (roms[0].Machine.Name.EndsWith(".zip") ? "" : ".zip")); // Set internal variables Stream writeStream = null; diff --git a/SabreTools.Helper/Tools/FileTools.cs b/SabreTools.Helper/Tools/FileTools.cs index 6c87d927..ad32668a 100644 --- a/SabreTools.Helper/Tools/FileTools.cs +++ b/SabreTools.Helper/Tools/FileTools.cs @@ -436,1079 +436,6 @@ namespace SabreTools.Helper.Tools #endregion - #region Rebuilding and Verifying - - /// - /// Process inputs and convert to TorrentZip or TorrentGZ, optionally converting to Romba format - /// - /// DatFile to use as a filter in conversion, null otherwise - /// List of inputs to convert over to TorrentZip or TorrentGZ - /// Output folder to rebuild to, blank is the current directory - /// Temporary directory to use in file extraction - /// True if files should be output in TorrentGZ format, false for TorrentZip - /// True if TorrentGZ files should be output in romba depot format, false otherwise - /// True if input files should be deleted, false otherwise - /// ArchiveScanLevel representing how files should be treated - /// Logger object for file and console output - /// True if processing was a success, false otherwise - public static bool ConvertFiles(DatFile datFile, List inputs, string outDir, string tempDir, bool tgz, - bool romba, bool delete, ArchiveScanLevel archiveScanLevel, Logger logger) - { - bool success = true; - - // First, check that the output directory exists - if (!Directory.Exists(outDir)) - { - Directory.CreateDirectory(outDir); - outDir = Path.GetFullPath(outDir); - } - - // Then create or clean the temp directory - if (!Directory.Exists(tempDir)) - { - Directory.CreateDirectory(tempDir); - } - else - { - CleanDirectory(tempDir); - } - - // Now process all of the inputs - foreach (string input in inputs) - { - logger.User("Examining file " + input); - - // Get if the file should be scanned internally and externally - bool shouldExternalProcess, shouldInternalProcess; - ArchiveTools.GetInternalExternalProcess(input, archiveScanLevel, logger, out shouldExternalProcess, out shouldInternalProcess); - - // Do an external scan of the file, if necessary - if (shouldExternalProcess) - { - // If a DAT is defined, we want to make sure that this file is not in there - Rom rom = FileTools.GetFileInfo(input, logger); - if (datFile != null && datFile.Files.Count > 0) - { - if (rom.HasDuplicates(datFile, logger)) - { - logger.User("File '" + input + "' existed in the DAT, skipping..."); - continue; - } - } - - logger.User("Processing file " + input); - - if (tgz) - { - success &= ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); - } - else - { - success &= ArchiveTools.WriteToArchive(input, outDir, rom, logger); - } - } - - // Process the file as an archive, if necessary - if (shouldInternalProcess) - { - // Now, if the file is a supported archive type, also run on all files within - bool encounteredErrors = ArchiveTools.ExtractArchive(input, tempDir, archiveScanLevel, logger); - - // If no errors were encountered, we loop through the temp directory - if (!encounteredErrors) - { - logger.Verbose("Archive found! Successfully extracted"); - foreach (string file in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) - { - // If a DAT is defined, we want to make sure that this file is not in there - Rom rom = FileTools.GetFileInfo(file, logger); - if (datFile != null && datFile.Files.Count > 0) - { - if (rom.HasDuplicates(datFile, logger)) - { - logger.User("File '" + file + "' existed in the DAT, skipping..."); - continue; - } - } - - logger.User("Processing file " + input); - - if (tgz) - { - success &= ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger); - } - else - { - success &= ArchiveTools.WriteToArchive(file, outDir, rom, logger); - } - } - - CleanDirectory(tempDir); - } - } - - // Delete the source file if we're supposed to - if (delete) - { - try - { - logger.User("Attempting to delete " + input); - File.Delete(input); - } - catch (Exception ex) - { - logger.Error(ex.ToString()); - success &= false; - } - } - } - - // Now one final delete of the temp directory - while (Directory.Exists(tempDir)) - { - try - { - Directory.Delete(tempDir, true); - } - catch - { - continue; - } - } - - // If we're in romba mode and the size file doesn't exist, create it - if (romba && !File.Exists(Path.Combine(outDir, ".romba_size"))) - { - // Get the size of all of the files in the output folder - long size = 0; - foreach (string file in Directory.EnumerateFiles(outDir, "*", SearchOption.AllDirectories)) - { - FileInfo tempinfo = new FileInfo(file); - size += tempinfo.Length; - } - - // Write out the value to each of the romba depot files - StreamWriter tw = new StreamWriter(File.Open(Path.Combine(outDir, ".romba_size"), FileMode.Create, FileAccess.Write)); - StreamWriter twb = new StreamWriter(File.Open(Path.Combine(outDir, ".romba_size.backup"), FileMode.Create, FileAccess.Write)); - - tw.Write(size); - twb.Write(size); - - tw.Dispose(); - twb.Dispose(); - } - - return success; - } - - /// - /// Process the DAT and find all matches in input files and folders - /// - /// DAT to compare against - /// List of input files/folders to check - /// Output directory to use to build to - /// Temporary directory for archive extraction - /// True to enable external scanning of archives, false otherwise - /// True if the date from the DAT should be used if available, false otherwise - /// True if files should be output to folder, false otherwise - /// True if input files should be deleted, false otherwise - /// True if output files should be written to TorrentGZ instead of TorrentZip - /// True if files should be output in Romba depot folders, false otherwise - /// ArchiveScanLevel representing the archive handling levels - /// True if the updated DAT should be output, false otherwise - /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise - /// Logger object for file and console output - /// True if rebuilding was a success, false otherwise - /// - /// This currently processes files as follows: - /// 1) Get all file names from the input files/folders - /// 2) Loop through and process each file individually - /// a) Hash the file - /// b) Check against the DAT for duplicates - /// c) Check for headers - /// d) Check headerless rom for duplicates - /// - public static bool RebuildToOutput(DatFile datFile, List inputs, string outDir, string tempDir, bool quickScan, bool date, - bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, - int maxDegreeOfParallelism, Logger logger) - { - // First, check that the output directory exists - if (!Directory.Exists(outDir)) - { - Directory.CreateDirectory(outDir); - outDir = Path.GetFullPath(outDir); - } - - // Then create or clean the temp directory - if (!Directory.Exists(tempDir)) - { - Directory.CreateDirectory(tempDir); - } - else - { - CleanDirectory(tempDir); - } - - bool success = true; - DatFile matched = new DatFile(); - - logger.User("Retrieving list all files from input"); - DateTime start = DateTime.Now; - - // Create a list of just files from inputs - List files = new List(); - Parallel.ForEach(inputs, - new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, - input => { - if (File.Exists(input)) - { - logger.Verbose("File found: '" + input + "'"); - files.Add(Path.GetFullPath(input)); - } - else if (Directory.Exists(input)) - { - logger.Verbose("Directory found: '" + input + "'"); - Parallel.ForEach(Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories), - new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, - file => - { - logger.Verbose("File found: '" + file + "'"); - files.Add(Path.GetFullPath(file)); - }); - } - else - { - logger.Error("'" + input + "' is not a file or directory!"); - } - }); - logger.User("Retrieving complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - - // Then, loop through and check each of the inputs - logger.User("Processing files:\n"); - int cursorTop = Console.CursorTop; - for (int i = 0; i < files.Count; i++) - { - success &= RebuildToOutputHelper(datFile, matched, files[i], i, files.Count, cursorTop, outDir, tempDir, quickScan, - date, toFolder, delete, tgz, romba, archiveScanLevel, headerToCheckAgainst, logger); - if (tempDir != Path.GetTempPath()) - { - CleanDirectory(tempDir); - } - if (success && delete) - { - try - { - File.Delete(files[i]); - } - catch { } - } - } - - // Now one final delete of the temp directory - while (Directory.Exists(tempDir)) - { - try - { - if (tempDir != Path.GetTempPath()) - { - Directory.Delete(tempDir, true); - } - } - catch - { - continue; - } - } - - // Now output the stats for the built files - logger.ClearBeneath(Constants.HeaderHeight); - Console.SetCursorPosition(0, Constants.HeaderHeight + 1); - logger.User("Stats of the matched ROMs:"); - StreamWriter sw = new StreamWriter(new MemoryStream()); - matched.OutputStats(sw, StatOutputFormat.None, logger, recalculate: true, baddumpCol: true, nodumpCol: true); - sw.Dispose(); - - // Now output the fixdat based on the original input if asked - if (updateDat) - { - datFile.FileName = "fixDat_" + datFile.FileName; - datFile.Name = "fixDat_" + datFile.Name; - datFile.Description = "fixDat_" + datFile.Description; - datFile.OutputFormat = OutputFormat.Logiqx; - datFile.WriteToFile("", logger); - } - - return success; - } - - /// - /// Process the DAT and find all matches in input files and folders - /// - /// DAT to compare against - /// List of input files/folders to check - /// Output directory to use to build to - /// Temporary directory for archive extraction - /// True to enable external scanning of archives, false otherwise - /// True if the date from the DAT should be used if available, false otherwise - /// True if files should be output to folder, false otherwise - /// True if input files should be deleted, false otherwise - /// True if output files should be written to TorrentGZ instead of TorrentZip - /// True if files should be output in Romba depot folders, false otherwise - /// ArchiveScanLevel representing the archive handling levels - /// True if the updated DAT should be output, false otherwise - /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise - /// Logger object for file and console output - /// True if rebuilding was a success, false otherwise - /// - /// This is a slightly different implementation of the RebuildToOutput code: - /// 1) Create a DFD from the input files (custom DFD with full pathnames?) - /// 2) Find the files that need to be used for rebuild - /// 3) Order them by game(if not outputting to TGZ) - /// 4) Write out each archive in order - /// - public static bool RebuildToOutputAlternate(DatFile datFile, List inputs, string outDir, string tempDir, bool quickScan, bool date, - bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, - int maxDegreeOfParallelism, Logger logger) - { - // First, check that the output directory exists - if (!Directory.Exists(outDir)) - { - Directory.CreateDirectory(outDir); - outDir = Path.GetFullPath(outDir); - } - - // Then create or clean the temp directory - if (!Directory.Exists(tempDir)) - { - Directory.CreateDirectory(tempDir); - } - else - { - CleanDirectory(tempDir); - } - - bool success = true; - DatFile matched = new DatFile(); - List files = new List(); - - #region Retrieve a list of all files - - logger.User("Retrieving list all files from input"); - DateTime start = DateTime.Now; - - // Create a list of just files from inputs - Parallel.ForEach(inputs, - new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, - input => { - if (File.Exists(input)) - { - logger.Verbose("File found: '" + input + "'"); - files.Add(Path.GetFullPath(input)); - } - else if (Directory.Exists(input)) - { - logger.Verbose("Directory found: '" + input + "'"); - Parallel.ForEach(Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories), - new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, - file => - { - logger.Verbose("File found: '" + file + "'"); - files.Add(Path.GetFullPath(file)); - }); - } - else - { - logger.Error("'" + input + "' is not a file or directory!"); - } - }); - logger.User("Retrieving complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - - #endregion - - DatFile current = new DatFile(); - - #region Create a dat from input files - - // Now that we have a list of just files, we get a DAT from the input files - foreach (string file in files) - { - // Define the temporary directory - string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar; - - // Get the required scanning level for the file - bool shouldExternalProcess = false; - bool shouldInternalProcess = false; - ArchiveTools.GetInternalExternalProcess(file, archiveScanLevel, logger, out shouldExternalProcess, out shouldInternalProcess); - - // If we're supposed to scan the file externally - if (shouldExternalProcess) - { - Rom rom = GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); - rom.Name = Path.GetFullPath(file); - - string key = rom.Size + "-" + rom.CRC; - if (current.Files.ContainsKey(key)) - { - current.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - current.Files.Add(key, temp); - } - } - - // If we're supposed to scan the file internally - if (shouldInternalProcess) - { - // If quickscan is set, do so - if (quickScan) - { - List extracted = ArchiveTools.GetArchiveFileInfo(file, logger); - - foreach (Rom rom in extracted) - { - Rom newrom = rom; - newrom.Machine = new Machine(Path.GetFullPath(file), ""); - - string key = rom.Size + "-" + rom.CRC; - if (current.Files.ContainsKey(key)) - { - current.Files[key].Add(newrom); - } - else - { - List temp = new List(); - temp.Add(newrom); - current.Files.Add(key, temp); - } - } - } - // Otherwise, attempt to extract the files to the temporary directory - else - { - bool encounteredErrors = ArchiveTools.ExtractArchive(file, tempSubDir, archiveScanLevel, logger); - - // If the file was an archive and was extracted successfully, check it - if (!encounteredErrors) - { - logger.Verbose(Path.GetFileName(file) + " treated like an archive"); - List extracted = Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories).ToList(); - Parallel.ForEach(extracted, - new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, - entry => - { - Rom rom = GetFileInfo(entry, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); - rom.Name = Path.Combine(Path.GetFullPath(file), Path.GetFileName(entry)); - rom.Machine = new Machine(Path.GetFullPath(file), ""); - - string key = rom.Size + "-" + rom.CRC; - if (current.Files.ContainsKey(key)) - { - current.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - current.Files.Add(key, temp); - } - }); - } - // Otherwise, just get the info on the file itself - else if (File.Exists(file)) - { - Rom rom = GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst); - rom.Name = Path.GetFullPath(file); - - string key = rom.Size + "-" + rom.CRC; - if (current.Files.ContainsKey(key)) - { - current.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - current.Files.Add(key, temp); - } - } - } - } - - // Now delete the temp directory - try - { - Directory.Delete(tempSubDir, true); - } - catch { } - } - - #endregion - - // Create a mapping from destination file to source file - Dictionary toFromMap = new Dictionary(); - - #region Find all required files for rebuild - - // Order the DATs by hash first to make things easier - datFile.BucketByCRC(false, logger, output: false); - current.BucketByCRC(false, logger, output: false); - - // Now loop over and find all files that need to be rebuilt - List keys = current.Files.Keys.ToList(); - foreach (string key in keys) - { - // If the input DAT doesn't have the key, then nothing from the current DAT are there - if (!datFile.Files.ContainsKey(key)) - { - continue; - } - - // Otherwise, we try to find duplicates - List datItems = current.Files[key]; - foreach (Rom rom in datItems) - { - List found = rom.GetDuplicates(datFile, logger, false); - - // Now add all of the duplicates mapped to the current file - foreach (Rom mid in found) - { - try - { - toFromMap.Add(mid, rom); - } - catch { } - } - } - } - - #endregion - - // Now bucket the list of keys by game so that we can rebuild properly - SortedDictionary> keysGroupedByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, logger, output: false); - - #region Rebuild games in order - - // Now loop through the keys and create the correct output items - List games = keysGroupedByGame.Keys.ToList(); - foreach (string game in games) - { - // Define the temporary directory - string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar; - - // Create an empty list for getting paths for rebuilding - List pathsToFiles = new List(); - - // Loop through all of the matched items in the game - List itemsInGame = keysGroupedByGame[game]; - List romsInGame = new List(); - foreach (Rom rom in itemsInGame) - { - // Get the rom that's mapped to this item - Rom source = (Rom)toFromMap[rom]; - - // If the file is in an archive, we need to treat it specially - string machinename = source.Machine.Name.ToLowerInvariant(); - if (machinename.EndsWith(".7z") - || machinename.EndsWith(".gz") - || machinename.EndsWith(".rar") - || machinename.EndsWith(".zip")) - { - pathsToFiles.Add(ArchiveTools.ExtractItem(source.Machine.Name, source.Name, tempSubDir, logger)); - } - - // Otherwise, we want to just add the full path - else - { - pathsToFiles.Add(source.Name); - } - - // Now add the rom to the output list - romsInGame.Add(source); - } - - // And now rebuild accordingly - if (toFolder) - { - for (int i = 0; i < romsInGame.Count; i++) - { - string infile = pathsToFiles[i]; - Rom outrom = romsInGame[i]; - string outfile = Path.Combine(outDir, outrom.Machine.Name, outrom.Machine.Name); - - // Make sure the output folder is created - Directory.CreateDirectory(Path.GetDirectoryName(outfile)); - - // Now copy the file over - try - { - File.Copy(infile, outfile); - } - catch { } - } - } - else if (tgz) - { - for (int i = 0; i < itemsInGame.Count; i++) - { - string infile = pathsToFiles[i]; - Rom outrom = romsInGame[i]; - ArchiveTools.WriteTorrentGZ(infile, outDir, romba, logger); - } - } - else - { - ArchiveTools.WriteToArchive(pathsToFiles, outDir, romsInGame, logger); - } - - // And now clear the temp folder to get rid of any transient files - try - { - Directory.Delete(tempSubDir, true); - } - catch { } - } - - #endregion - - return success; - } - - /// - /// Process an individual file against the DAT for rebuilding - /// - /// DAT to compare against - /// List of files that were matched by the DAT - /// Name of the input file - /// Index of the current file - /// Total number of files - /// Top cursor position to use - /// Output directory to use to build to - /// Temporary directory for archive extraction - /// True to enable external scanning of archives, false otherwise - /// True if the date from the DAT should be used if available, false otherwise - /// True if files should be output to folder, false otherwise - /// True if input files should be deleted, false otherwise - /// True if output files should be written to TorrentGZ instead of TorrentZip - /// True if files should be output in Romba depot folders, false otherwise - /// ArchiveScanLevel representing the archive handling levels - /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise - /// Logger object for file and console output - /// True if this is in a recurse step and the file should be deleted, false otherwise (default) - /// True if it was processed properly, false otherwise - private static bool RebuildToOutputHelper(DatFile datFile, DatFile matched, string input, int index, int total, int cursorTop, - string outDir, string tempDir, bool quickScan, bool date, bool toFolder, bool delete, bool tgz, bool romba, - ArchiveScanLevel archiveScanLevel, string headerToCheckAgainst, Logger logger, bool recurse = false) - { - bool success = true; - - // Get the full path of the input for movement purposes - string percentage = (index == 0 ? "0.00" : Math.Round((100 * ((double)index / total)), 2, MidpointRounding.AwayFromZero).ToString()); - string statement = percentage + "% - " + input; - logger.ClearBeneath(cursorTop + 1); - logger.WriteExact(statement, cursorTop, 0); - - // Get if the file should be scanned internally and externally - bool shouldExternalScan, shouldInternalScan; - ArchiveTools.GetInternalExternalProcess(input, archiveScanLevel, logger, out shouldExternalScan, out shouldInternalScan); - - // Hash and match the external files - if (shouldExternalScan) - { - Rom rom = FileTools.GetFileInfo(input, logger); - - // If we have a blank RomData, it's an error - if (rom.Name == null) - { - return false; - } - - // Try to find the matches to the file that was found - List foundroms = rom.GetDuplicates(datFile, logger); - logger.Verbose("File '" + input + "' had " + foundroms.Count + " matches in the DAT!"); - foreach (Rom found in foundroms) - { - logger.Verbose("Matched name: " + found.Name); - - // Add rom to the matched list - string key = found.Size + "-" + found.CRC; - if (matched.Files.ContainsKey(key)) - { - matched.Files[key].Add(found); - } - else - { - List temp = new List(); - temp.Add(found); - matched.Files.Add(key, temp); - } - - if (toFolder) - { - // Copy file to output directory - string gamedir = Path.Combine(outDir, found.Machine.Name); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'"); - try - { - File.Copy(input, Path.Combine(gamedir, Path.GetFileName(found.Name))); - } - catch { } - } - else - { - if (tgz) - { - ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); - } - else - { - ArchiveTools.WriteToArchive(input, outDir, found, logger, date: date); - } - } - } - - // Now get the transformed file if it exists - SkipperRule rule = Skipper.GetMatchingRule(input, headerToCheckAgainst, logger); - - // If we have have a non-empty rule, apply it - if (rule.Tests != null && rule.Tests.Count != 0) - { - // Otherwise, apply the rule to the file - string newinput = input + ".new"; - rule.TransformFile(input, newinput, logger); - Rom drom = FileTools.GetFileInfo(newinput, logger); - - // If we have a blank RomData, it's an error - if (String.IsNullOrEmpty(drom.Name)) - { - return false; - } - - // Try to find the matches to the file that was found - List founddroms = drom.GetDuplicates(datFile, logger); - logger.Verbose("File '" + newinput + "' had " + founddroms.Count + " matches in the DAT!"); - foreach (Rom found in founddroms) - { - // Add rom to the matched list - string key = found.Size + "-" + found.CRC; - if (matched.Files.ContainsKey(key)) - { - matched.Files[key].Add(found); - } - else - { - List temp = new List(); - temp.Add(found); - matched.Files.Add(key, temp); - } - - // First output the headerless rom - logger.Verbose("Matched name: " + found.Name); - - if (toFolder) - { - // Copy file to output directory - string gamedir = Path.Combine(outDir, found.Machine.Name); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'"); - try - { - File.Copy(newinput, Path.Combine(gamedir, Path.GetFileName(found.Name))); - } - catch { } - } - else - { - if (tgz) - { - ArchiveTools.WriteTorrentGZ(newinput, outDir, romba, logger); - } - else - { - ArchiveTools.WriteToArchive(newinput, outDir, found, logger, date: date); - } - } - - // Then output the headered rom (renamed) - Rom newfound = found; - newfound.Name = Path.GetFileNameWithoutExtension(newfound.Name) + " (" + rom.CRC + ")" + Path.GetExtension(newfound.Name); - newfound.Size = rom.Size; - newfound.CRC = rom.CRC; - newfound.MD5 = rom.MD5; - newfound.SHA1 = rom.SHA1; - - // Add rom to the matched list - key = newfound.Size + "-" + newfound.CRC; - if (matched.Files.ContainsKey(key)) - { - matched.Files[key].Add(newfound); - } - else - { - List temp = new List(); - temp.Add(newfound); - matched.Files.Add(key, temp); - } - - if (toFolder) - { - // Copy file to output directory - string gamedir = Path.Combine(outDir, found.Machine.Name); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + newfound.Name + "'"); - try - { - File.Copy(input, Path.Combine(gamedir, Path.GetFileName(newfound.Name))); - } - catch { } - } - else - { - logger.Verbose("Matched name: " + newfound.Name); - if (tgz) - { - ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); - } - else - { - ArchiveTools.WriteToArchive(input, outDir, newfound, logger, date: date); - } - } - } - - // Now remove this temporary file - try - { - File.Delete(newinput); - } - catch - { - // Don't log file deletion errors - } - } - } - - // If we should scan the file as an archive - if (shouldInternalScan) - { - // If external scanning is enabled, use that method instead - if (quickScan) - { - logger.Verbose("Beginning quick scan of contents from '" + input + "'"); - List internalRomData = ArchiveTools.GetArchiveFileInfo(input, logger); - logger.Verbose(internalRomData.Count + " entries found in '" + input + "'"); - - // If the list is populated, then the file was a filled archive - if (internalRomData.Count > 0) - { - foreach (Rom rom in internalRomData) - { - // Try to find the matches to the file that was found - List foundroms = rom.GetDuplicates(datFile, logger); - logger.Verbose("File '" + rom.Name + "' had " + foundroms.Count + " matches in the DAT!"); - foreach (Rom found in foundroms) - { - // Add rom to the matched list - string key = found.Size + "-" + found.CRC; - if (matched.Files.ContainsKey(key)) - { - matched.Files[key].Add(found); - } - else - { - List temp = new List(); - temp.Add(found); - matched.Files.Add(key, temp); - } - - if (toFolder) - { - // Copy file to output directory - logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + found.Name + "'"); - string outfile = ArchiveTools.ExtractItem(input, rom.Name, tempDir, logger); - if (File.Exists(outfile)) - { - string gamedir = Path.Combine(outDir, found.Machine.Name); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - try - { - File.Move(outfile, Path.Combine(gamedir, Path.GetFileName(found.Name))); - } - catch { } - } - } - else - { - // Copy file between archives - logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'"); - - if (Build.MonoEnvironment || tgz) - { - string outfile = ArchiveTools.ExtractItem(input, rom.Name, tempDir, logger); - if (File.Exists(outfile)) - { - if (tgz) - { - ArchiveTools.WriteTorrentGZ(outfile, outDir, romba, logger); - } - else - { - ArchiveTools.WriteToArchive(outfile, outDir, found, logger); - } - - try - { - File.Delete(outfile); - } - catch { } - } - } - else - { - ArchiveTools.CopyFileBetweenArchives(input, outDir, rom.Name, found, logger); - } - } - } - } - } - } - else - { - // Now, if the file is a supported archive type, also run on all files within - bool encounteredErrors = ArchiveTools.ExtractArchive(input, tempDir, archiveScanLevel, logger); - - // Remove the current file if we are in recursion so it's not picked up in the next step - if (recurse) - { - try - { - File.Delete(input); - } - catch (Exception) - { - // Don't log file deletion errors - } - } - - // If no errors were encountered, we loop through the temp directory - if (!encounteredErrors) - { - logger.Verbose("Archive found! Successfully extracted"); - foreach (string file in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) - { - success &= RebuildToOutputHelper(datFile, matched, file, index, total, cursorTop, outDir, tempDir, quickScan, - date, toFolder, delete, tgz, romba, archiveScanLevel, headerToCheckAgainst, logger, recurse: true); - } - } - } - } - - return success; - } - - /// - /// Process the DAT and verify the output directory - /// - /// DAT to use to verify the directory - /// List of input directories to compare against - /// Temporary directory for archive extraction - /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise - /// Logger object for file and console output - /// True if verification was a success, false otherwise - public static bool VerifyDirectory(DatFile datFile, List inputs, string tempDir, string headerToCheckAgainst, Logger logger) - { - // First create or clean the temp directory - if (!Directory.Exists(tempDir)) - { - Directory.CreateDirectory(tempDir); - } - else - { - CleanDirectory(tempDir); - } - - bool success = true; - - /* - We want the cross section of what's the folder and what's in the DAT. Right now, it just has what's in the DAT that's not in the folder - */ - - // Then, loop through and check each of the inputs - logger.User("Processing files:\n"); - foreach (string input in inputs) - { - datFile.PopulateDatFromDir(input, false /* noMD5 */, false /* noSHA1 */, true /* bare */, false /* archivesAsFiles */, - true /* enableGzip */, false /* addBlanks */, false /* addDate */, tempDir /* tempDir */, false /* copyFiles */, - headerToCheckAgainst, 4 /* maxDegreeOfParallelism */, logger); - } - - // Setup the fixdat - DatFile matched = (DatFile)datFile.CloneHeader(); - matched.Files = new SortedDictionary>(); - matched.FileName = "fixDat_" + matched.FileName; - matched.Name = "fixDat_" + matched.Name; - matched.Description = "fixDat_" + matched.Description; - matched.OutputFormat = OutputFormat.Logiqx; - - // Now that all files are parsed, get only files found in directory - bool found = false; - foreach (List roms in datFile.Files.Values) - { - List newroms = DatItem.Merge(roms, logger); - foreach (Rom rom in newroms) - { - if (rom.SourceID == 99) - { - found = true; - string key = rom.Size + "-" + rom.CRC; - if (matched.Files.ContainsKey(key)) - { - matched.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - matched.Files.Add(key, temp); - } - } - } - } - - // Now output the fixdat to the main folder - if (found) - { - matched.WriteToFile("", logger, stats: true); - } - else - { - logger.User("No fixDat needed"); - } - - return success; - } - - #endregion - #region Stream Information /// diff --git a/SabreTools/Partials/SabreTools_Inits.cs b/SabreTools/Partials/SabreTools_Inits.cs index 492501dc..81766f9b 100644 --- a/SabreTools/Partials/SabreTools_Inits.cs +++ b/SabreTools/Partials/SabreTools_Inits.cs @@ -64,7 +64,7 @@ namespace SabreTools } _logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - return FileTools.ConvertFiles(datdata, inputs, outDir, tempDir, tgz, romba, delete, asl, _logger); + return datdata.ConvertFiles(inputs, outDir, tempDir, tgz, romba, delete, asl, _logger); } /// @@ -336,7 +336,7 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - FileTools.RebuildToOutput(datdata, inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, + datdata.RebuildToOutput(inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, _logger); } @@ -686,7 +686,7 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - FileTools.VerifyDirectory(datdata, inputs, tempDir, headerToCheckAgainst, _logger); + datdata.VerifyDirectory(inputs, tempDir, headerToCheckAgainst, _logger); } #endregion diff --git a/SimpleSort/SimpleSort.cs b/SimpleSort/SimpleSort.cs index be4a1d58..2d132e60 100644 --- a/SimpleSort/SimpleSort.cs +++ b/SimpleSort/SimpleSort.cs @@ -64,7 +64,7 @@ namespace SabreTools gz = 2, rar = 2, zip = 0; - string header = "", + string header = null, outDir = "", tempDir = ""; List datfiles = new List(); @@ -390,7 +390,7 @@ namespace SabreTools } logger.User("Organizing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - return FileTools.ConvertFiles(datdata, inputs, outDir, tempDir, tgz, romba, delete, asl, logger); + return datdata.ConvertFiles(inputs, outDir, tempDir, tgz, romba, delete, asl, logger); } /// @@ -430,7 +430,7 @@ namespace SabreTools } logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - FileTools.RebuildToOutput(datdata, inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, 4, logger); + datdata.RebuildToOutput(inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, 4, logger); } /// @@ -457,7 +457,7 @@ namespace SabreTools } logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - FileTools.VerifyDirectory(datdata, inputs, tempDir, headerToCheckAgainst, logger); + datdata.VerifyDirectory(inputs, tempDir, headerToCheckAgainst, logger); } } } \ No newline at end of file