From aabb44fb33097020728ef760acdc1a4cacbdb80b Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Sat, 17 Sep 2016 18:00:18 -0700 Subject: [PATCH] [SimpleSort] Start adding better rebuild code --- SabreTools.Helper/Objects/SimpleSort.cs | 185 +++++++++++++++++++++++- 1 file changed, 178 insertions(+), 7 deletions(-) diff --git a/SabreTools.Helper/Objects/SimpleSort.cs b/SabreTools.Helper/Objects/SimpleSort.cs index 34c76637..eebef0bc 100644 --- a/SabreTools.Helper/Objects/SimpleSort.cs +++ b/SabreTools.Helper/Objects/SimpleSort.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Threading.Tasks; namespace SabreTools.Helper { @@ -23,6 +25,7 @@ namespace SabreTools.Helper private ArchiveScanLevel _rar; private ArchiveScanLevel _zip; private Logger _logger; + private int _maxDegreeOfParallelism = 4; // Hardcoded for now, should be an input later // Other private variables private int _cursorTop; @@ -32,7 +35,7 @@ namespace SabreTools.Helper /// /// Create a new SimpleSort object /// - /// Name of the DAT to compare against + /// DAT to compare against /// List of input files/folders to check /// Output directory to use to build to /// Temporary directory for archive extraction @@ -199,12 +202,7 @@ namespace SabreTools.Helper /// c) Check for headers /// d) Check headerless rom for duplicates /// - /// This is actually rather slow and inefficient. Instead, it should do the following: - /// 1) Get all file names from the input files/folders (parallel) - /// 2) Loop through and get the file info from every file (including headerless) - /// 3) Find all duplicate files in the input DAT(s) - /// 4) Order by output game - /// 5) Rebuild all files + /// This is actually rather slow and inefficient. See below for more correct implemenation /// public bool RebuildToOutput() { @@ -281,6 +279,179 @@ namespace SabreTools.Helper return success; } + /// + /// Process the DAT and find all matches in input files and folders + /// + /// True if rebuilding was a success, false otherwise + /// + /// This implemenation of the code should do the following: + /// 1) Get all file names from the input files/folders (parallel) + /// 2) Loop through and get the file info from every file (including headerless) + /// 3) Find all duplicate files in the input DAT(s) + /// 4) Order by output game + /// 5) Rebuild all files + /// + public bool RebuiltToOutputAlternate() + { + bool success = true; + + // Create a list of just files from inputs + _logger.User("Finding all files..."); + List files = new List(); + Parallel.ForEach(_inputs, + new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism }, + input => + { + if (File.Exists(input)) + { + _logger.Log("File found: '" + input + "'"); + lock (files) + { + files.Add(Path.GetFullPath(input)); + } + } + else if (Directory.Exists(input)) + { + _logger.Log("Directory found: '" + input + "'"); + + List infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList(); + Parallel.ForEach(infiles, + new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism }, + file => + { + _logger.Log("File found: '" + input + "'"); + lock (files) + { + files.Add(Path.GetFullPath(file)); + } + }); + } + else + { + _logger.Error("'" + input + "' is not a file or directory!"); + } + }); + _logger.User("Finding files complete!"); + + // TODO: The below code does NOT check for headerless files as well. This is a problem. + + // Now loop through all of the files and check them, DFD style + _logger.User("Getting source file information..."); + Dat matchdat = new Dat + { + Files = new Dictionary>(), + }; + foreach (string file in files) + { + // Get if the file should be scanned internally and externally + bool shouldExternalScan, shouldInternalScan; + FileTools.GetInternalExternalProcess(file, _7z, _gz, _rar, _zip, _logger, out shouldExternalScan, out shouldInternalScan); + + // Hash and match the external files + if (shouldExternalScan) + { + Rom rom = FileTools.GetSingleFileInfo(file); + + // If we have a blank RomData, it's an error + if (rom.Name == null) + { + continue; + } + + // Otherwise, set the machine name as the full path to the file + rom.Machine.Name = Path.GetDirectoryName(Path.GetFullPath(file)); + + // Add the rom information to the Dat + if (matchdat.Files.ContainsKey(rom.Machine.Name.ToLowerInvariant())) + { + matchdat.Files[rom.Machine.Name.ToLowerInvariant()].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + matchdat.Files.Add(rom.Machine.Name.ToLowerInvariant(), temp); + } + } + + // If we should scan the file as an archive + if (shouldInternalScan) + { + // If external scanning is enabled, use that method instead + if (_quickScan) + { + _logger.Log("Beginning quick scan of contents from '" + file + "'"); + List internalRomData = FileTools.GetArchiveFileInfo(file, _logger); + _logger.Log(internalRomData.Count + " entries found in '" + file + "'"); + + // Now add all of the roms to the DAT + for (int i = 0; i < internalRomData.Count; i++) + { + RebuildToOutputAlternateParseRomHelper(file, ref matchdat); + } + } + // Otherwise, try to extract the file to the temp folder + else + { + // Now, if the file is a supported archive type, also run on all files within + bool encounteredErrors = FileTools.ExtractArchive(file, _tempDir, _7z, _gz, _rar, _zip, _logger); + + // If we succeeded in extracting, loop through the files + if (!encounteredErrors) + { + List extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList(); + foreach (string extractedFile in extractedFiles) + { + RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat); + } + } + // Otherwise, skip extracting and just get information on the file itself (if we didn't already) + else if (!shouldExternalScan) + { + RebuildToOutputAlternateParseRomHelper(file, ref matchdat); + } + } + } + } + _logger.User("Getting source file information complete!"); + + return success; + } + + /// + /// Wrap adding a file to the dictionary in custom DFD + /// + /// Name of the file to attempt to add + /// Reference to the Dat to add to + /// True if the file could be added, false otherwise + public bool RebuildToOutputAlternateParseRomHelper(string file, ref Dat matchdat) + { + Rom rom = FileTools.GetSingleFileInfo(file); + + // If we have a blank RomData, it's an error + if (rom.Name == null) + { + return false; + } + + // Otherwise, set the machine name as the full path to the file + rom.Machine.Name = Path.GetDirectoryName(Path.GetFullPath(file)); + + // Add the rom information to the Dat + if (matchdat.Files.ContainsKey(rom.Machine.Name.ToLowerInvariant())) + { + matchdat.Files[rom.Machine.Name.ToLowerInvariant()].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + matchdat.Files.Add(rom.Machine.Name.ToLowerInvariant(), temp); + } + + return true; + } + /// /// Process an individual file against the DAT for rebuilding ///