diff --git a/SabreTools.Helper/Tools/FileTools.cs b/SabreTools.Helper/Tools/FileTools.cs index c7be5586..30da6a9d 100644 --- a/SabreTools.Helper/Tools/FileTools.cs +++ b/SabreTools.Helper/Tools/FileTools.cs @@ -1091,283 +1091,6 @@ namespace SabreTools.Helper.Tools return success; } - /// - /// Process the DAT and find all matches in input files and folders - /// - /// DAT to compare against - /// List of input files/folders to check - /// Output directory to use to build to - /// Temporary directory for archive extraction - /// True to enable external scanning of archives, false otherwise - /// True if the date from the DAT should be used if available, false otherwise - /// True if files should be output to folder, false otherwise - /// True if input files should be deleted, false otherwise - /// True if output files should be written to TorrentGZ instead of TorrentZip - /// True if files should be output in Romba depot folders, false otherwise - /// ArchiveScanLevel representing the archive handling levels - /// True if the updated DAT should be output, false otherwise - /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise - /// Logger object for file and console output - /// True if rebuilding was a success, false otherwise - /// - /// This implemenation of the code should do the following: - /// 1) Get all file names from the input files/folders (parallel) - /// 2) Loop through and get the file info from every file (including headerless) - /// 3) Find all duplicate files in the input DAT(s) - /// 4) Order by output game - /// 5) Rebuild all files - /// - public static bool RebuiltToOutputAlternate(DatFile datFile, List inputs, string outDir, string tempDir, bool quickScan, bool date, - bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, - int maxDegreeOfParallelism, Logger logger) - { - // First, check that the output directory exists - if (!Directory.Exists(outDir)) - { - Directory.CreateDirectory(outDir); - outDir = Path.GetFullPath(outDir); - } - - // Then create or clean the temp directory - if (!Directory.Exists(tempDir)) - { - Directory.CreateDirectory(tempDir); - } - else - { - CleanDirectory(tempDir); - } - - bool success = true; - - #region Find all files - - // Create a list of just files from inputs - logger.User("Finding all files..."); - List files = new List(); - Parallel.ForEach(inputs, - new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, - input => - { - if (File.Exists(input)) - { - logger.Verbose("File found: '" + input + "'"); - lock (files) - { - files.Add(Path.GetFullPath(input)); - } - } - else if (Directory.Exists(input)) - { - logger.Verbose("Directory found: '" + input + "'"); - - List infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList(); - Parallel.ForEach(infiles, - new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, - file => - { - logger.Verbose("File found: '" + input + "'"); - lock (files) - { - files.Add(Path.GetFullPath(file)); - } - }); - } - else - { - logger.Error("'" + input + "' is not a file or directory!"); - } - }); - logger.User("Finding files complete!"); - - #endregion - - #region Get source file information - - // Now loop through all of the files and check them, DFD style - logger.User("Getting source file information..."); - DatFile matchdat = new DatFile - { - Files = new SortedDictionary>(), - }; - foreach (string file in files) - { - // Get if the file should be scanned internally and externally - bool shouldExternalScan, shouldInternalScan; - ArchiveTools.GetInternalExternalProcess(file, archiveScanLevel, logger, out shouldExternalScan, out shouldInternalScan); - - // Hash and match the external files - if (shouldExternalScan) - { - RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger); - } - - // If we should scan the file as an archive - if (shouldInternalScan) - { - // If external scanning is enabled, use that method instead - if (quickScan) - { - logger.Verbose("Beginning quick scan of contents from '" + file + "'"); - List internalRomData = ArchiveTools.GetArchiveFileInfo(file, logger); - logger.Verbose(internalRomData.Count + " entries found in '" + file + "'"); - - // Now add all of the roms to the DAT - for (int i = 0; i < internalRomData.Count; i++) - { - RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger); - } - } - // Otherwise, try to extract the file to the temp folder - else - { - // Now, if the file is a supported archive type, also run on all files within - bool encounteredErrors = ArchiveTools.ExtractArchive(file, tempDir, archiveScanLevel, logger); - - // If we succeeded in extracting, loop through the files - if (!encounteredErrors) - { - List extractedFiles = Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories).ToList(); - foreach (string extractedFile in extractedFiles) - { - RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, headerToCheckAgainst, logger); - } - } - // Otherwise, skip extracting and just get information on the file itself (if we didn't already) - else if (!shouldExternalScan) - { - RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger); - } - - // Clean the temp directory for the next round - if (Directory.Exists(tempDir)) - { - CleanDirectory(tempDir); - } - } - } - } - logger.User("Getting source file information complete!"); - - #endregion - - #region Find all files to rebuild and bucket by game - - // Create a dictionary of from/to Rom mappings - Dictionary toFromMap = new Dictionary(); - - // Now populate it - foreach (string key in matchdat.Files.Keys) - { - foreach (DatItem rom in matchdat.Files[key]) - { - List matched = rom.GetDuplicates(datFile, logger, true); - foreach (DatItem match in matched) - { - try - { - toFromMap.Add(match, rom); - } - catch { } - } - } - } - - // Then bucket the keys by game for better output - SortedDictionary> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, logger); - - #endregion - - #region Rebuild all files - - // At this point, we have "toFromMap" which maps output files to input files as well as - // as SortedDictionary called keysByGame which is the output files sorted by game in - // alphabetical order. We should be able to use these to do everything we need =) - - // Now write out each game sequentially - foreach (string key in keysByGame.Keys) - { - - } - - #endregion - - return success; - } - - /// - /// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::" - /// - /// Name of the file to attempt to add - /// Reference to the Dat to add to - /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise - /// Logger object for file and console output - /// True if the file could be added, false otherwise - private static bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, string headerToCheckAgainst, Logger logger) - { - Rom rom = FileTools.GetFileInfo(file, logger); - - // If we have a blank RomData, it's an error - if (rom.Name == null) - { - return false; - } - - // Otherwise, set the machine name as the full path to the file - rom.Machine.Name = Path.GetDirectoryName(Path.GetFullPath(file)); - - // Add the rom information to the Dat - string key = rom.Size + "-" + rom.CRC; - if (matchdat.Files.ContainsKey(key)) - { - matchdat.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - matchdat.Files.Add(key, temp); - } - - // Now attempt to see if the file has a header - FileStream input = File.OpenRead(file); - SkipperRule rule = Skipper.GetMatchingRule(input, headerToCheckAgainst, logger); - - // If there's a match, get the new information from the stream - if (rule.Tests != null && rule.Tests.Count != 0) - { - // Create the input and output streams - MemoryStream output = new MemoryStream(); - - // Transform the stream and get the information from it - rule.TransformStream(input, output, logger, false, true); - Rom romNH = FileTools.GetStreamInfo(output, output.Length); - romNH.Name = "HEAD::" + rom.Name; - romNH.Machine.Name = rom.Machine.Name; - - // Add the rom information to the Dat - key = romNH.Size + "-" + romNH.CRC; - if (matchdat.Files.ContainsKey(key)) - { - matchdat.Files[key].Add(romNH); - } - else - { - List temp = new List(); - temp.Add(romNH); - matchdat.Files.Add(key, temp); - } - - // Dispose of the stream - output.Dispose(); - } - - // Dispose of the stream - input.Dispose(); - - return true; - } - /// /// Process the DAT and verify the output directory ///