diff --git a/RombaSharp/Partials/RombaSharp_Inits.cs b/RombaSharp/Partials/RombaSharp_Inits.cs index ce254ed0..e3af03b7 100644 --- a/RombaSharp/Partials/RombaSharp_Inits.cs +++ b/RombaSharp/Partials/RombaSharp_Inits.cs @@ -184,8 +184,7 @@ namespace SabreTools // Create the sorting object to use and rebuild the needed files ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1)); - SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, _logger); - ss.RebuildToOutput(); + FileTools.RebuildToOutput(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, 4, _logger); } /// @@ -223,8 +222,7 @@ namespace SabreTools // Now scan all of those depots and rebuild ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1); - SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, _logger); - ss.RebuildToOutput(); + FileTools.RebuildToOutput(datFile, onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, 4, _logger); } } diff --git a/SabreTools.Helper/Objects/SimpleSort.cs b/SabreTools.Helper/Objects/SimpleSort.cs deleted file mode 100644 index aaa0ada6..00000000 --- a/SabreTools.Helper/Objects/SimpleSort.cs +++ /dev/null @@ -1,925 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Threading.Tasks; - -namespace SabreTools.Helper -{ - public class SimpleSort - { - // Private instance variables - private DatFile _datdata; - private List _inputs; - private string _outDir; - private string _tempDir; - private bool _quickScan; - private bool _date; - private bool _toFolder; - private bool _delete; - private bool _tgz; - private bool _romba; - private bool _updateDat; - private ArchiveScanLevel _archiveScanLevel; - private string _headerToCheckAgainst; - private Logger _logger; - private int _maxDegreeOfParallelism = 4; // Hardcoded for now, should be an input later - - // Other private variables - private int _cursorTop; - private int _cursorLeft; - private DatFile _matched; - - /// - /// Create a new SimpleSort object - /// - /// DAT to compare against - /// List of input files/folders to check - /// Output directory to use to build to - /// Temporary directory for archive extraction - /// True to enable external scanning of archives, false otherwise - /// True if the date from the DAT should be used if available, false otherwise - /// True if files should be output to folder, false otherwise - /// True if input files should be deleted, false otherwise - /// True if output files should be written to TorrentGZ instead of TorrentZip - /// True if files should be output in Romba depot folders, false otherwise - /// ArchiveScanLevel representing the archive handling levels - /// True if the updated DAT should be output, false otherwise - /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise - /// Logger object for file and console output - public SimpleSort(DatFile datdata, List inputs, string outDir, string tempDir, - bool quickScan, bool date, bool toFolder, bool delete, bool tgz, bool romba, - ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, Logger logger) - { - _datdata = datdata; - _inputs = inputs; - _outDir = (outDir == "" ? "Rebuild" : outDir); - _tempDir = (String.IsNullOrEmpty(tempDir) ? Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) : tempDir); - _quickScan = quickScan; - _date = date; - _toFolder = toFolder; - _delete = delete; - _tgz = tgz; - _romba = romba; - _archiveScanLevel = archiveScanLevel; - _updateDat = updateDat; - _headerToCheckAgainst = headerToCheckAgainst; - _logger = logger; - - _cursorTop = Console.CursorTop; - _cursorLeft = Console.CursorLeft; - _matched = new DatFile - { - Files = new SortedDictionary>(), - }; - } - - /// - /// Process the DAT and find all matches in input files and folders - /// - /// True if rebuilding was a success, false otherwise - /// - /// This currently processes files as follows: - /// 1) Get all file names from the input files/folders - /// 2) Loop through and process each file individually - /// a) Hash the file - /// b) Check against the DAT for duplicates - /// c) Check for headers - /// d) Check headerless rom for duplicates - /// - /// This is actually rather slow and inefficient. See below for more correct implemenation - /// - public bool RebuildToOutput() - { - // First, check that the output directory exists - if (!Directory.Exists(_outDir)) - { - Directory.CreateDirectory(_outDir); - _outDir = Path.GetFullPath(_outDir); - } - - // Then create or clean the temp directory - if (!Directory.Exists(_tempDir)) - { - Directory.CreateDirectory(_tempDir); - } - else - { - FileTools.CleanDirectory(_tempDir); - } - - bool success = true; - - _logger.User("Retrieving list all files from input"); - DateTime start = DateTime.Now; - - // Create a list of just files from inputs - List files = new List(); - Parallel.ForEach(_inputs, - new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism, }, - input => { - if (File.Exists(input)) - { - _logger.Verbose("File found: '" + input + "'"); - files.Add(Path.GetFullPath(input)); - } - else if (Directory.Exists(input)) - { - _logger.Verbose("Directory found: '" + input + "'"); - Parallel.ForEach(Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories), - new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism, }, - file => - { - _logger.Verbose("File found: '" + file + "'"); - files.Add(Path.GetFullPath(file)); - }); - } - else - { - _logger.Error("'" + input + "' is not a file or directory!"); - } - }); - _logger.User("Retrieving complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - - // Then, loop through and check each of the inputs - _logger.User("Processing files:\n"); - _cursorTop = Console.CursorTop; - for (int i = 0; i < files.Count; i++) - { - success &= RebuildToOutputHelper(files[i], i, files.Count); - if (_tempDir != Path.GetTempPath()) - { - FileTools.CleanDirectory(_tempDir); - } - if (success && _delete) - { - try - { - File.Delete(files[i]); - } - catch { } - } - } - - // Now one final delete of the temp directory - while (Directory.Exists(_tempDir)) - { - try - { - if (_tempDir != Path.GetTempPath()) - { - Directory.Delete(_tempDir, true); - } - } - catch - { - continue; - } - } - - // Now output the stats for the built files - _logger.ClearBeneath(Constants.HeaderHeight); - Console.SetCursorPosition(0, Constants.HeaderHeight + 1); - _logger.User("Stats of the matched ROMs:"); - StreamWriter sw = new StreamWriter(new MemoryStream()); - _matched.OutputStats(sw, StatOutputFormat.None, _logger, recalculate: true, baddumpCol: true, nodumpCol: true); - sw.Dispose(); - - // Now output the fixdat based on the original input if asked - if (_updateDat) - { - _datdata.FileName = "fixDat_" + _datdata.FileName; - _datdata.Name = "fixDat_" + _datdata.Name; - _datdata.Description = "fixDat_" + _datdata.Description; - _datdata.OutputFormat = OutputFormat.Logiqx; - _datdata.WriteToFile("", _logger); - } - - return success; - } - - /// - /// Process an individual file against the DAT for rebuilding - /// - /// Name of the input file - /// Index of the current file - /// Total number of files - /// True if this is in a recurse step and the file should be deleted, false otherwise (default) - /// True if it was processed properly, false otherwise - private bool RebuildToOutputHelper(string input, int index, int total, bool recurse = false) - { - bool success = true; - - // Get the full path of the input for movement purposes - string percentage = (index == 0 ? "0.00" : Math.Round((100 * ((double)index / total)), 2, MidpointRounding.AwayFromZero).ToString()); - string statement = percentage + "% - " + input; - _logger.ClearBeneath(_cursorTop + 1); - _logger.WriteExact(statement, _cursorTop, 0); - - // Get if the file should be scanned internally and externally - bool shouldExternalScan, shouldInternalScan; - ArchiveTools.GetInternalExternalProcess(input, _archiveScanLevel, _logger, out shouldExternalScan, out shouldInternalScan); - - // Hash and match the external files - if (shouldExternalScan) - { - Rom rom = FileTools.GetFileInfo(input, _logger); - - // If we have a blank RomData, it's an error - if (rom.Name == null) - { - return false; - } - - // Try to find the matches to the file that was found - List foundroms = rom.GetDuplicates(_datdata, _logger); - _logger.Verbose("File '" + input + "' had " + foundroms.Count + " matches in the DAT!"); - foreach (Rom found in foundroms) - { - _logger.Verbose("Matched name: " + found.Name); - - // Add rom to the matched list - string key = found.Size + "-" + found.CRC; - if (_matched.Files.ContainsKey(key)) - { - _matched.Files[key].Add(found); - } - else - { - List temp = new List(); - temp.Add(found); - _matched.Files.Add(key, temp); - } - - if (_toFolder) - { - // Copy file to output directory - string gamedir = Path.Combine(_outDir, found.MachineName); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - _logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.SHA1 : found.Name) + "'"); - try - { - File.Copy(input, Path.Combine(gamedir, Path.GetFileName(found.Name))); - } - catch { } - } - else - { - if (_tgz) - { - ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger); - } - else - { - ArchiveTools.WriteToArchive(input, _outDir, found, _logger, date: _date); - } - } - } - - // Now get the transformed file if it exists - SkipperRule rule = Skipper.GetMatchingRule(input, _headerToCheckAgainst, _logger); - - // If we have have a non-empty rule, apply it - if (rule.Tests != null && rule.Tests.Count != 0) - { - // Otherwise, apply the rule to the file - string newinput = input + ".new"; - rule.TransformFile(input, newinput, _logger); - Rom drom = FileTools.GetFileInfo(newinput, _logger); - - // If we have a blank RomData, it's an error - if (String.IsNullOrEmpty(drom.Name)) - { - return false; - } - - // Try to find the matches to the file that was found - List founddroms = drom.GetDuplicates(_datdata, _logger); - _logger.Verbose("File '" + newinput + "' had " + founddroms.Count + " matches in the DAT!"); - foreach (Rom found in founddroms) - { - // Add rom to the matched list - string key = found.Size + "-" + found.CRC; - if (_matched.Files.ContainsKey(key)) - { - _matched.Files[key].Add(found); - } - else - { - List temp = new List(); - temp.Add(found); - _matched.Files.Add(key, temp); - } - - // First output the headerless rom - _logger.Verbose("Matched name: " + found.Name); - - if (_toFolder) - { - // Copy file to output directory - string gamedir = Path.Combine(_outDir, found.MachineName); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - _logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.SHA1 : found.Name) + "'"); - try - { - File.Copy(newinput, Path.Combine(gamedir, Path.GetFileName(found.Name))); - } - catch { } - } - else - { - if (_tgz) - { - ArchiveTools.WriteTorrentGZ(newinput, _outDir, _romba, _logger); - } - else - { - ArchiveTools.WriteToArchive(newinput, _outDir, found, _logger, date: _date); - } - } - - // Then output the headered rom (renamed) - Rom newfound = found; - newfound.Name = Path.GetFileNameWithoutExtension(newfound.Name) + " (" + rom.CRC + ")" + Path.GetExtension(newfound.Name); - newfound.Size = rom.Size; - newfound.CRC = rom.CRC; - newfound.MD5 = rom.MD5; - newfound.SHA1 = rom.SHA1; - - // Add rom to the matched list - key = newfound.Size + "-" + newfound.CRC; - if (_matched.Files.ContainsKey(key)) - { - _matched.Files[key].Add(newfound); - } - else - { - List temp = new List(); - temp.Add(newfound); - _matched.Files.Add(key, temp); - } - - if (_toFolder) - { - // Copy file to output directory - string gamedir = Path.Combine(_outDir, found.MachineName); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - _logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + newfound.Name + "'"); - try - { - File.Copy(input, Path.Combine(gamedir, Path.GetFileName(newfound.Name))); - } - catch { } - } - else - { - _logger.Verbose("Matched name: " + newfound.Name); - if (_tgz) - { - ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger); - } - else - { - ArchiveTools.WriteToArchive(input, _outDir, newfound, _logger, date: _date); - } - } - } - - // Now remove this temporary file - try - { - File.Delete(newinput); - } - catch - { - // Don't log file deletion errors - } - } - } - - // If we should scan the file as an archive - if (shouldInternalScan) - { - // If external scanning is enabled, use that method instead - if (_quickScan) - { - _logger.Verbose("Beginning quick scan of contents from '" + input + "'"); - List internalRomData = ArchiveTools.GetArchiveFileInfo(input, _logger); - _logger.Verbose(internalRomData.Count + " entries found in '" + input + "'"); - - // If the list is populated, then the file was a filled archive - if (internalRomData.Count > 0) - { - foreach (Rom rom in internalRomData) - { - // Try to find the matches to the file that was found - List foundroms = rom.GetDuplicates(_datdata, _logger); - _logger.Verbose("File '" + rom.Name + "' had " + foundroms.Count + " matches in the DAT!"); - foreach (Rom found in foundroms) - { - // Add rom to the matched list - string key = found.Size + "-" + found.CRC; - if (_matched.Files.ContainsKey(key)) - { - _matched.Files[key].Add(found); - } - else - { - List temp = new List(); - temp.Add(found); - _matched.Files.Add(key, temp); - } - - if (_toFolder) - { - // Copy file to output directory - _logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + found.Name + "'"); - string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, _tempDir, _logger); - if (File.Exists(outfile)) - { - string gamedir = Path.Combine(_outDir, found.MachineName); - if (!Directory.Exists(gamedir)) - { - Directory.CreateDirectory(gamedir); - } - - try - { - File.Move(outfile, Path.Combine(gamedir, Path.GetFileName(found.Name))); - } - catch { } - } - } - else - { - // Copy file between archives - _logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.SHA1 : found.Name) + "'"); - - if (Build.MonoEnvironment || _tgz) - { - string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, _tempDir, _logger); - if (File.Exists(outfile)) - { - if (_tgz) - { - ArchiveTools.WriteTorrentGZ(outfile, _outDir, _romba, _logger); - } - else - { - ArchiveTools.WriteToArchive(outfile, _outDir, found, _logger); - } - - try - { - File.Delete(outfile); - } - catch { } - } - } - else - { - ArchiveTools.CopyFileBetweenArchives(input, _outDir, rom.Name, found, _logger); - } - } - } - } - } - } - else - { - // Now, if the file is a supported archive type, also run on all files within - bool encounteredErrors = ArchiveTools.ExtractArchive(input, _tempDir, _archiveScanLevel, _logger); - - // Remove the current file if we are in recursion so it's not picked up in the next step - if (recurse) - { - try - { - File.Delete(input); - } - catch (Exception) - { - // Don't log file deletion errors - } - } - - // If no errors were encountered, we loop through the temp directory - if (!encounteredErrors) - { - _logger.Verbose("Archive found! Successfully extracted"); - foreach (string file in Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories)) - { - success &= RebuildToOutputHelper(file, index, total, true); - } - } - } - } - - return success; - } - - /// - /// Process the DAT and find all matches in input files and folders - /// - /// True if rebuilding was a success, false otherwise - /// - /// This implemenation of the code should do the following: - /// 1) Get all file names from the input files/folders (parallel) - /// 2) Loop through and get the file info from every file (including headerless) - /// 3) Find all duplicate files in the input DAT(s) - /// 4) Order by output game - /// 5) Rebuild all files - /// - public bool RebuiltToOutputAlternate() - { - // First, check that the output directory exists - if (!Directory.Exists(_outDir)) - { - Directory.CreateDirectory(_outDir); - _outDir = Path.GetFullPath(_outDir); - } - - // Then create or clean the temp directory - if (!Directory.Exists(_tempDir)) - { - Directory.CreateDirectory(_tempDir); - } - else - { - FileTools.CleanDirectory(_tempDir); - } - - bool success = true; - - #region Find all files - - // Create a list of just files from inputs - _logger.User("Finding all files..."); - List files = new List(); - Parallel.ForEach(_inputs, - new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism }, - input => - { - if (File.Exists(input)) - { - _logger.Verbose("File found: '" + input + "'"); - lock (files) - { - files.Add(Path.GetFullPath(input)); - } - } - else if (Directory.Exists(input)) - { - _logger.Verbose("Directory found: '" + input + "'"); - - List infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList(); - Parallel.ForEach(infiles, - new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism }, - file => - { - _logger.Verbose("File found: '" + input + "'"); - lock (files) - { - files.Add(Path.GetFullPath(file)); - } - }); - } - else - { - _logger.Error("'" + input + "' is not a file or directory!"); - } - }); - _logger.User("Finding files complete!"); - - #endregion - - #region Get source file information - - // Now loop through all of the files and check them, DFD style - _logger.User("Getting source file information..."); - DatFile matchdat = new DatFile - { - Files = new SortedDictionary>(), - }; - foreach (string file in files) - { - // Get if the file should be scanned internally and externally - bool shouldExternalScan, shouldInternalScan; - ArchiveTools.GetInternalExternalProcess(file, _archiveScanLevel, _logger, out shouldExternalScan, out shouldInternalScan); - - // Hash and match the external files - if (shouldExternalScan) - { - RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger); - } - - // If we should scan the file as an archive - if (shouldInternalScan) - { - // If external scanning is enabled, use that method instead - if (_quickScan) - { - _logger.Verbose("Beginning quick scan of contents from '" + file + "'"); - List internalRomData = ArchiveTools.GetArchiveFileInfo(file, _logger); - _logger.Verbose(internalRomData.Count + " entries found in '" + file + "'"); - - // Now add all of the roms to the DAT - for (int i = 0; i < internalRomData.Count; i++) - { - RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger); - } - } - // Otherwise, try to extract the file to the temp folder - else - { - // Now, if the file is a supported archive type, also run on all files within - bool encounteredErrors = ArchiveTools.ExtractArchive(file, _tempDir, _archiveScanLevel, _logger); - - // If we succeeded in extracting, loop through the files - if (!encounteredErrors) - { - List extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList(); - foreach (string extractedFile in extractedFiles) - { - RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, _logger); - } - } - // Otherwise, skip extracting and just get information on the file itself (if we didn't already) - else if (!shouldExternalScan) - { - RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger); - } - - // Clean the temp directory for the next round - if (Directory.Exists(_tempDir)) - { - FileTools.CleanDirectory(_tempDir); - } - } - } - } - _logger.User("Getting source file information complete!"); - - #endregion - - #region Find all files to rebuild and bucket by game - - // Create a dictionary of from/to Rom mappings - Dictionary toFromMap = new Dictionary(); - - // Now populate it - foreach (string key in matchdat.Files.Keys) - { - foreach (DatItem rom in matchdat.Files[key]) - { - List matched = rom.GetDuplicates(_datdata, _logger, true); - foreach (DatItem match in matched) - { - try - { - toFromMap.Add(match, rom); - } - catch { } - } - } - } - - // Then bucket the keys by game for better output - SortedDictionary> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, _logger); - - #endregion - - #region Rebuild all files - - // At this point, we have "toFromMap" which maps output files to input files as well as - // as SortedDictionary called keysByGame which is the output files sorted by game in - // alphabetical order. We should be able to use these to do everything we need =) - - // Now write out each game sequentially - foreach (string key in keysByGame.Keys) - { - - } - - #endregion - - return success; - } - - /// - /// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::" - /// - /// Name of the file to attempt to add - /// Reference to the Dat to add to - /// Logger object for file and console output - /// True if the file could be added, false otherwise - private bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, Logger logger) - { - Rom rom = FileTools.GetFileInfo(file, logger); - - // If we have a blank RomData, it's an error - if (rom.Name == null) - { - return false; - } - - // Otherwise, set the machine name as the full path to the file - rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file)); - - // Add the rom information to the Dat - string key = rom.Size + "-" + rom.CRC; - if (matchdat.Files.ContainsKey(key)) - { - matchdat.Files[key].Add(rom); - } - else - { - List temp = new List(); - temp.Add(rom); - matchdat.Files.Add(key, temp); - } - - // Now attempt to see if the file has a header - FileStream input = File.OpenRead(file); - SkipperRule rule = Skipper.GetMatchingRule(input, _headerToCheckAgainst, _logger); - - // If there's a match, get the new information from the stream - if (rule.Tests != null && rule.Tests.Count != 0) - { - // Create the input and output streams - MemoryStream output = new MemoryStream(); - - // Transform the stream and get the information from it - rule.TransformStream(input, output, _logger, false, true); - Rom romNH = FileTools.GetStreamInfo(output, output.Length); - romNH.Name = "HEAD::" + rom.Name; - romNH.MachineName = rom.MachineName; - - // Add the rom information to the Dat - key = romNH.Size + "-" + romNH.CRC; - if (matchdat.Files.ContainsKey(key)) - { - matchdat.Files[key].Add(romNH); - } - else - { - List temp = new List(); - temp.Add(romNH); - matchdat.Files.Add(key, temp); - } - - // Dispose of the stream - output.Dispose(); - } - - // Dispose of the stream - input.Dispose(); - - return true; - } - - /// - /// Clean an individual folder based on the DAT - /// - /// True if the cleaning succeeded, false otherwise - /// This method is incomplete, it need to be finished before it can be used - private bool InplaceRebuild() - { - bool success = true; - - /* - The process of rebuilding inplace is as follows: - 0) Resort the input roms by Game since that's more important in this case - 1) Scan the current folder according to the level specified (no recursion) - a - If file is a match in all aspects, set correct flag and pass - + If the file has a header, skip? - b - If file is a match in hash but not name, rename, set correct flag and pass - c - If file is not a match, extract it to the output folder and remove from archive, set incorrect flag - 2) For all files that have been removed, check to see if they could be rebuilt to another location - a - This behaves similarly (and indeed could call) "RebuildToFolder" - b - If a file is a match and rebuilt, remove it from the output folder - */ - - // Assuming archived sets, move all toplevel folders to temp - foreach (string directory in Directory.EnumerateDirectories(_outDir, "*", SearchOption.TopDirectoryOnly)) - { - Directory.Move(directory, Path.Combine(_tempDir, Path.GetFileNameWithoutExtension(directory))); - } - - // Now process the inputs (assumed that it's archived sets as of right now - Dictionary> scanned = new Dictionary>(); - foreach (string archive in Directory.EnumerateFiles(_outDir, "*", SearchOption.AllDirectories)) - { - // If we are in quickscan, get the list of roms that way - List roms = new List(); - if (_quickScan) - { - roms = ArchiveTools.GetArchiveFileInfo(Path.GetFullPath(archive), _logger); - } - // Otherwise, extract it and get info one by one - else - { - string temparcdir = Path.Combine(_tempDir, Path.GetFileNameWithoutExtension(archive)); - ArchiveTools.ExtractArchive(Path.GetFullPath(archive), temparcdir, _logger); - foreach (string tempfile in Directory.EnumerateFiles(temparcdir, "*", SearchOption.AllDirectories)) - { - roms.Add(FileTools.GetFileInfo(Path.GetFullPath(tempfile), _logger)); - } - - // Clear the temporary archive directory - FileTools.CleanDirectory(temparcdir); - } - - // Then add each of the found files to the new dictionary - foreach (Rom rom in roms) - { - string key = rom.Size + "-" + rom.CRC; - if (scanned.ContainsKey(key)) - { - scanned[key].Add(rom); - } - else - { - List templist = new List(); - templist.Add(rom); - scanned.Add(key, templist); - } - } - } - - // If nothing was found, we that it was successful - if (scanned.Count == 0) - { - return success; - } - - // Now that we have all of the from DAT and from folder roms, we try to match them, removing the perfect matches - Dictionary> remove = new Dictionary>(); - foreach (string key in scanned.Keys) - { - // If the key doesn't even exist in the DAT, then mark the entire key for removal - if (!_datdata.Files.ContainsKey(key)) - { - if (remove.ContainsKey(key)) - { - remove[key].AddRange(scanned[key]); - } - else - { - remove.Add(key, scanned[key]); - } - } - // Otherwise check each of the values individually - else - { - List romsList = _datdata.Files[key]; - List scannedList = scanned[key]; - foreach (Rom rom in scannedList) - { - if (!romsList.Contains(rom)) - { - if (remove.ContainsKey(key)) - { - remove[key].Add(rom); - } - else - { - List templist = new List(); - templist.Add(rom); - remove.Add(key, templist); - } - } - } - } - } - - // At this point, we have the complete list of files from the DAT, a complete - // list of files that were scanned from the archives, and a complete list of - // the files to be removed because they aren't matches. I think at this point, - // we need to see if any of the files in "removed" can be rebuilt to something - // that is missing. But we don't have a list of missings, so how do we get this - // set of roms? Missing would be (_datdata.Roms - matches) I think. So if we - // get this additional set, we then run it against the "removed" set and rebuild - // as we go based on what we can do. Here is where we need some smarts. If the - // game to rebuild from and to are the same, we want to copy within. You - // should create a new helper function that "renames" an entry within the same - // archive to help this along. Everything else rebuilding should be copied from - // archive to archive. Once remove has been traversed, we will extract and remove - // all of the files that have been found and put them in the temporary folder. - - return success; - } - } -} diff --git a/SabreTools.Helper/SabreTools.Helper.csproj b/SabreTools.Helper/SabreTools.Helper.csproj index 9b5f1a56..338a8a57 100644 --- a/SabreTools.Helper/SabreTools.Helper.csproj +++ b/SabreTools.Helper/SabreTools.Helper.csproj @@ -111,7 +111,6 @@ - diff --git a/SabreTools.Helper/Tools/FileTools.cs b/SabreTools.Helper/Tools/FileTools.cs index 176eba8b..bc394719 100644 --- a/SabreTools.Helper/Tools/FileTools.cs +++ b/SabreTools.Helper/Tools/FileTools.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; using System.Security.Cryptography; +using System.Threading.Tasks; using System.Xml; using System.Xml.Schema; @@ -560,7 +561,7 @@ namespace SabreTools.Helper } } - FileTools.CleanDirectory(tempDir); + CleanDirectory(tempDir); } } @@ -618,6 +619,774 @@ namespace SabreTools.Helper return success; } + /// + /// Process the DAT and find all matches in input files and folders + /// + /// DAT to compare against + /// List of input files/folders to check + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise + /// True if files should be output to folder, false otherwise + /// True if input files should be deleted, false otherwise + /// True if output files should be written to TorrentGZ instead of TorrentZip + /// True if files should be output in Romba depot folders, false otherwise + /// ArchiveScanLevel representing the archive handling levels + /// True if the updated DAT should be output, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if rebuilding was a success, false otherwise + /// + /// This currently processes files as follows: + /// 1) Get all file names from the input files/folders + /// 2) Loop through and process each file individually + /// a) Hash the file + /// b) Check against the DAT for duplicates + /// c) Check for headers + /// d) Check headerless rom for duplicates + /// + /// This is actually rather slow and inefficient. See below for more correct implemenation + /// + public static bool RebuildToOutput(DatFile datFile, List inputs, string outDir, string tempDir, bool quickScan, bool date, + bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, + int maxDegreeOfParallelism, Logger logger) + { + // First, check that the output directory exists + if (!Directory.Exists(outDir)) + { + Directory.CreateDirectory(outDir); + outDir = Path.GetFullPath(outDir); + } + + // Then create or clean the temp directory + if (!Directory.Exists(tempDir)) + { + Directory.CreateDirectory(tempDir); + } + else + { + CleanDirectory(tempDir); + } + + bool success = true; + DatFile matched = new DatFile(); + + logger.User("Retrieving list all files from input"); + DateTime start = DateTime.Now; + + // Create a list of just files from inputs + List files = new List(); + Parallel.ForEach(inputs, + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, + input => { + if (File.Exists(input)) + { + logger.Verbose("File found: '" + input + "'"); + files.Add(Path.GetFullPath(input)); + } + else if (Directory.Exists(input)) + { + logger.Verbose("Directory found: '" + input + "'"); + Parallel.ForEach(Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories), + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, }, + file => + { + logger.Verbose("File found: '" + file + "'"); + files.Add(Path.GetFullPath(file)); + }); + } + else + { + logger.Error("'" + input + "' is not a file or directory!"); + } + }); + logger.User("Retrieving complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); + + // Then, loop through and check each of the inputs + logger.User("Processing files:\n"); + int cursorTop = Console.CursorTop; + for (int i = 0; i < files.Count; i++) + { + success &= RebuildToOutputHelper(datFile, matched, files[i], i, files.Count, cursorTop, outDir, tempDir, quickScan, + date, toFolder, delete, tgz, romba, archiveScanLevel, headerToCheckAgainst, logger); + if (tempDir != Path.GetTempPath()) + { + CleanDirectory(tempDir); + } + if (success && delete) + { + try + { + File.Delete(files[i]); + } + catch { } + } + } + + // Now one final delete of the temp directory + while (Directory.Exists(tempDir)) + { + try + { + if (tempDir != Path.GetTempPath()) + { + Directory.Delete(tempDir, true); + } + } + catch + { + continue; + } + } + + // Now output the stats for the built files + logger.ClearBeneath(Constants.HeaderHeight); + Console.SetCursorPosition(0, Constants.HeaderHeight + 1); + logger.User("Stats of the matched ROMs:"); + StreamWriter sw = new StreamWriter(new MemoryStream()); + matched.OutputStats(sw, StatOutputFormat.None, logger, recalculate: true, baddumpCol: true, nodumpCol: true); + sw.Dispose(); + + // Now output the fixdat based on the original input if asked + if (updateDat) + { + datFile.FileName = "fixDat_" + datFile.FileName; + datFile.Name = "fixDat_" + datFile.Name; + datFile.Description = "fixDat_" + datFile.Description; + datFile.OutputFormat = OutputFormat.Logiqx; + datFile.WriteToFile("", logger); + } + + return success; + } + + /// + /// Process an individual file against the DAT for rebuilding + /// + /// DAT to compare against + /// List of files that were matched by the DAT + /// Name of the input file + /// Index of the current file + /// Total number of files + /// Top cursor position to use + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise + /// True if files should be output to folder, false otherwise + /// True if input files should be deleted, false otherwise + /// True if output files should be written to TorrentGZ instead of TorrentZip + /// True if files should be output in Romba depot folders, false otherwise + /// ArchiveScanLevel representing the archive handling levels + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if this is in a recurse step and the file should be deleted, false otherwise (default) + /// True if it was processed properly, false otherwise + private static bool RebuildToOutputHelper(DatFile datFile, DatFile matched, string input, int index, int total, int cursorTop, + string outDir, string tempDir, bool quickScan, bool date, bool toFolder, bool delete, bool tgz, bool romba, + ArchiveScanLevel archiveScanLevel, string headerToCheckAgainst, Logger logger, bool recurse = false) + { + bool success = true; + + // Get the full path of the input for movement purposes + string percentage = (index == 0 ? "0.00" : Math.Round((100 * ((double)index / total)), 2, MidpointRounding.AwayFromZero).ToString()); + string statement = percentage + "% - " + input; + logger.ClearBeneath(cursorTop + 1); + logger.WriteExact(statement, cursorTop, 0); + + // Get if the file should be scanned internally and externally + bool shouldExternalScan, shouldInternalScan; + ArchiveTools.GetInternalExternalProcess(input, archiveScanLevel, logger, out shouldExternalScan, out shouldInternalScan); + + // Hash and match the external files + if (shouldExternalScan) + { + Rom rom = FileTools.GetFileInfo(input, logger); + + // If we have a blank RomData, it's an error + if (rom.Name == null) + { + return false; + } + + // Try to find the matches to the file that was found + List foundroms = rom.GetDuplicates(datFile, logger); + logger.Verbose("File '" + input + "' had " + foundroms.Count + " matches in the DAT!"); + foreach (Rom found in foundroms) + { + logger.Verbose("Matched name: " + found.Name); + + // Add rom to the matched list + string key = found.Size + "-" + found.CRC; + if (matched.Files.ContainsKey(key)) + { + matched.Files[key].Add(found); + } + else + { + List temp = new List(); + temp.Add(found); + matched.Files.Add(key, temp); + } + + if (toFolder) + { + // Copy file to output directory + string gamedir = Path.Combine(outDir, found.MachineName); + if (!Directory.Exists(gamedir)) + { + Directory.CreateDirectory(gamedir); + } + + logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'"); + try + { + File.Copy(input, Path.Combine(gamedir, Path.GetFileName(found.Name))); + } + catch { } + } + else + { + if (tgz) + { + ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); + } + else + { + ArchiveTools.WriteToArchive(input, outDir, found, logger, date: date); + } + } + } + + // Now get the transformed file if it exists + SkipperRule rule = Skipper.GetMatchingRule(input, headerToCheckAgainst, logger); + + // If we have have a non-empty rule, apply it + if (rule.Tests != null && rule.Tests.Count != 0) + { + // Otherwise, apply the rule to the file + string newinput = input + ".new"; + rule.TransformFile(input, newinput, logger); + Rom drom = FileTools.GetFileInfo(newinput, logger); + + // If we have a blank RomData, it's an error + if (String.IsNullOrEmpty(drom.Name)) + { + return false; + } + + // Try to find the matches to the file that was found + List founddroms = drom.GetDuplicates(datFile, logger); + logger.Verbose("File '" + newinput + "' had " + founddroms.Count + " matches in the DAT!"); + foreach (Rom found in founddroms) + { + // Add rom to the matched list + string key = found.Size + "-" + found.CRC; + if (matched.Files.ContainsKey(key)) + { + matched.Files[key].Add(found); + } + else + { + List temp = new List(); + temp.Add(found); + matched.Files.Add(key, temp); + } + + // First output the headerless rom + logger.Verbose("Matched name: " + found.Name); + + if (toFolder) + { + // Copy file to output directory + string gamedir = Path.Combine(outDir, found.MachineName); + if (!Directory.Exists(gamedir)) + { + Directory.CreateDirectory(gamedir); + } + + logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'"); + try + { + File.Copy(newinput, Path.Combine(gamedir, Path.GetFileName(found.Name))); + } + catch { } + } + else + { + if (tgz) + { + ArchiveTools.WriteTorrentGZ(newinput, outDir, romba, logger); + } + else + { + ArchiveTools.WriteToArchive(newinput, outDir, found, logger, date: date); + } + } + + // Then output the headered rom (renamed) + Rom newfound = found; + newfound.Name = Path.GetFileNameWithoutExtension(newfound.Name) + " (" + rom.CRC + ")" + Path.GetExtension(newfound.Name); + newfound.Size = rom.Size; + newfound.CRC = rom.CRC; + newfound.MD5 = rom.MD5; + newfound.SHA1 = rom.SHA1; + + // Add rom to the matched list + key = newfound.Size + "-" + newfound.CRC; + if (matched.Files.ContainsKey(key)) + { + matched.Files[key].Add(newfound); + } + else + { + List temp = new List(); + temp.Add(newfound); + matched.Files.Add(key, temp); + } + + if (toFolder) + { + // Copy file to output directory + string gamedir = Path.Combine(outDir, found.MachineName); + if (!Directory.Exists(gamedir)) + { + Directory.CreateDirectory(gamedir); + } + + logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + newfound.Name + "'"); + try + { + File.Copy(input, Path.Combine(gamedir, Path.GetFileName(newfound.Name))); + } + catch { } + } + else + { + logger.Verbose("Matched name: " + newfound.Name); + if (tgz) + { + ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger); + } + else + { + ArchiveTools.WriteToArchive(input, outDir, newfound, logger, date: date); + } + } + } + + // Now remove this temporary file + try + { + File.Delete(newinput); + } + catch + { + // Don't log file deletion errors + } + } + } + + // If we should scan the file as an archive + if (shouldInternalScan) + { + // If external scanning is enabled, use that method instead + if (quickScan) + { + logger.Verbose("Beginning quick scan of contents from '" + input + "'"); + List internalRomData = ArchiveTools.GetArchiveFileInfo(input, logger); + logger.Verbose(internalRomData.Count + " entries found in '" + input + "'"); + + // If the list is populated, then the file was a filled archive + if (internalRomData.Count > 0) + { + foreach (Rom rom in internalRomData) + { + // Try to find the matches to the file that was found + List foundroms = rom.GetDuplicates(datFile, logger); + logger.Verbose("File '" + rom.Name + "' had " + foundroms.Count + " matches in the DAT!"); + foreach (Rom found in foundroms) + { + // Add rom to the matched list + string key = found.Size + "-" + found.CRC; + if (matched.Files.ContainsKey(key)) + { + matched.Files[key].Add(found); + } + else + { + List temp = new List(); + temp.Add(found); + matched.Files.Add(key, temp); + } + + if (toFolder) + { + // Copy file to output directory + logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + found.Name + "'"); + string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, tempDir, logger); + if (File.Exists(outfile)) + { + string gamedir = Path.Combine(outDir, found.MachineName); + if (!Directory.Exists(gamedir)) + { + Directory.CreateDirectory(gamedir); + } + + try + { + File.Move(outfile, Path.Combine(gamedir, Path.GetFileName(found.Name))); + } + catch { } + } + } + else + { + // Copy file between archives + logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'"); + + if (Build.MonoEnvironment || tgz) + { + string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, tempDir, logger); + if (File.Exists(outfile)) + { + if (tgz) + { + ArchiveTools.WriteTorrentGZ(outfile, outDir, romba, logger); + } + else + { + ArchiveTools.WriteToArchive(outfile, outDir, found, logger); + } + + try + { + File.Delete(outfile); + } + catch { } + } + } + else + { + ArchiveTools.CopyFileBetweenArchives(input, outDir, rom.Name, found, logger); + } + } + } + } + } + } + else + { + // Now, if the file is a supported archive type, also run on all files within + bool encounteredErrors = ArchiveTools.ExtractArchive(input, tempDir, archiveScanLevel, logger); + + // Remove the current file if we are in recursion so it's not picked up in the next step + if (recurse) + { + try + { + File.Delete(input); + } + catch (Exception) + { + // Don't log file deletion errors + } + } + + // If no errors were encountered, we loop through the temp directory + if (!encounteredErrors) + { + logger.Verbose("Archive found! Successfully extracted"); + foreach (string file in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) + { + success &= RebuildToOutputHelper(datFile, matched, file, index, total, cursorTop, outDir, tempDir, quickScan, + date, toFolder, delete, tgz, romba, archiveScanLevel, headerToCheckAgainst, logger, recurse: true); + } + } + } + } + + return success; + } + + /// + /// Process the DAT and find all matches in input files and folders + /// + /// DAT to compare against + /// List of input files/folders to check + /// Output directory to use to build to + /// Temporary directory for archive extraction + /// True to enable external scanning of archives, false otherwise + /// True if the date from the DAT should be used if available, false otherwise + /// True if files should be output to folder, false otherwise + /// True if input files should be deleted, false otherwise + /// True if output files should be written to TorrentGZ instead of TorrentZip + /// True if files should be output in Romba depot folders, false otherwise + /// ArchiveScanLevel representing the archive handling levels + /// True if the updated DAT should be output, false otherwise + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if rebuilding was a success, false otherwise + /// + /// This implemenation of the code should do the following: + /// 1) Get all file names from the input files/folders (parallel) + /// 2) Loop through and get the file info from every file (including headerless) + /// 3) Find all duplicate files in the input DAT(s) + /// 4) Order by output game + /// 5) Rebuild all files + /// + public static bool RebuiltToOutputAlternate(DatFile datFile, List inputs, string outDir, string tempDir, bool quickScan, bool date, + bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, + int maxDegreeOfParallelism, Logger logger) + { + // First, check that the output directory exists + if (!Directory.Exists(outDir)) + { + Directory.CreateDirectory(outDir); + outDir = Path.GetFullPath(outDir); + } + + // Then create or clean the temp directory + if (!Directory.Exists(tempDir)) + { + Directory.CreateDirectory(tempDir); + } + else + { + CleanDirectory(tempDir); + } + + bool success = true; + + #region Find all files + + // Create a list of just files from inputs + logger.User("Finding all files..."); + List files = new List(); + Parallel.ForEach(inputs, + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, + input => + { + if (File.Exists(input)) + { + logger.Verbose("File found: '" + input + "'"); + lock (files) + { + files.Add(Path.GetFullPath(input)); + } + } + else if (Directory.Exists(input)) + { + logger.Verbose("Directory found: '" + input + "'"); + + List infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList(); + Parallel.ForEach(infiles, + new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism }, + file => + { + logger.Verbose("File found: '" + input + "'"); + lock (files) + { + files.Add(Path.GetFullPath(file)); + } + }); + } + else + { + logger.Error("'" + input + "' is not a file or directory!"); + } + }); + logger.User("Finding files complete!"); + + #endregion + + #region Get source file information + + // Now loop through all of the files and check them, DFD style + logger.User("Getting source file information..."); + DatFile matchdat = new DatFile + { + Files = new SortedDictionary>(), + }; + foreach (string file in files) + { + // Get if the file should be scanned internally and externally + bool shouldExternalScan, shouldInternalScan; + ArchiveTools.GetInternalExternalProcess(file, archiveScanLevel, logger, out shouldExternalScan, out shouldInternalScan); + + // Hash and match the external files + if (shouldExternalScan) + { + RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger); + } + + // If we should scan the file as an archive + if (shouldInternalScan) + { + // If external scanning is enabled, use that method instead + if (quickScan) + { + logger.Verbose("Beginning quick scan of contents from '" + file + "'"); + List internalRomData = ArchiveTools.GetArchiveFileInfo(file, logger); + logger.Verbose(internalRomData.Count + " entries found in '" + file + "'"); + + // Now add all of the roms to the DAT + for (int i = 0; i < internalRomData.Count; i++) + { + RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger); + } + } + // Otherwise, try to extract the file to the temp folder + else + { + // Now, if the file is a supported archive type, also run on all files within + bool encounteredErrors = ArchiveTools.ExtractArchive(file, tempDir, archiveScanLevel, logger); + + // If we succeeded in extracting, loop through the files + if (!encounteredErrors) + { + List extractedFiles = Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories).ToList(); + foreach (string extractedFile in extractedFiles) + { + RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, headerToCheckAgainst, logger); + } + } + // Otherwise, skip extracting and just get information on the file itself (if we didn't already) + else if (!shouldExternalScan) + { + RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger); + } + + // Clean the temp directory for the next round + if (Directory.Exists(tempDir)) + { + CleanDirectory(tempDir); + } + } + } + } + logger.User("Getting source file information complete!"); + + #endregion + + #region Find all files to rebuild and bucket by game + + // Create a dictionary of from/to Rom mappings + Dictionary toFromMap = new Dictionary(); + + // Now populate it + foreach (string key in matchdat.Files.Keys) + { + foreach (DatItem rom in matchdat.Files[key]) + { + List matched = rom.GetDuplicates(datFile, logger, true); + foreach (DatItem match in matched) + { + try + { + toFromMap.Add(match, rom); + } + catch { } + } + } + } + + // Then bucket the keys by game for better output + SortedDictionary> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, logger); + + #endregion + + #region Rebuild all files + + // At this point, we have "toFromMap" which maps output files to input files as well as + // as SortedDictionary called keysByGame which is the output files sorted by game in + // alphabetical order. We should be able to use these to do everything we need =) + + // Now write out each game sequentially + foreach (string key in keysByGame.Keys) + { + + } + + #endregion + + return success; + } + + /// + /// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::" + /// + /// Name of the file to attempt to add + /// Reference to the Dat to add to + /// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise + /// Logger object for file and console output + /// True if the file could be added, false otherwise + private static bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, string headerToCheckAgainst, Logger logger) + { + Rom rom = FileTools.GetFileInfo(file, logger); + + // If we have a blank RomData, it's an error + if (rom.Name == null) + { + return false; + } + + // Otherwise, set the machine name as the full path to the file + rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file)); + + // Add the rom information to the Dat + string key = rom.Size + "-" + rom.CRC; + if (matchdat.Files.ContainsKey(key)) + { + matchdat.Files[key].Add(rom); + } + else + { + List temp = new List(); + temp.Add(rom); + matchdat.Files.Add(key, temp); + } + + // Now attempt to see if the file has a header + FileStream input = File.OpenRead(file); + SkipperRule rule = Skipper.GetMatchingRule(input, headerToCheckAgainst, logger); + + // If there's a match, get the new information from the stream + if (rule.Tests != null && rule.Tests.Count != 0) + { + // Create the input and output streams + MemoryStream output = new MemoryStream(); + + // Transform the stream and get the information from it + rule.TransformStream(input, output, logger, false, true); + Rom romNH = FileTools.GetStreamInfo(output, output.Length); + romNH.Name = "HEAD::" + rom.Name; + romNH.MachineName = rom.MachineName; + + // Add the rom information to the Dat + key = romNH.Size + "-" + romNH.CRC; + if (matchdat.Files.ContainsKey(key)) + { + matchdat.Files[key].Add(romNH); + } + else + { + List temp = new List(); + temp.Add(romNH); + matchdat.Files.Add(key, temp); + } + + // Dispose of the stream + output.Dispose(); + } + + // Dispose of the stream + input.Dispose(); + + return true; + } + /// /// Process the DAT and verify the output directory /// diff --git a/SabreTools/Partials/SabreTools_Inits.cs b/SabreTools/Partials/SabreTools_Inits.cs index 1199a28e..d4fa217b 100644 --- a/SabreTools/Partials/SabreTools_Inits.cs +++ b/SabreTools/Partials/SabreTools_Inits.cs @@ -323,9 +323,7 @@ namespace SabreTools } _logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date, - toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, _logger); - ss.RebuildToOutput(); + FileTools.RebuildToOutput(datdata, inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, 4, _logger); } /// diff --git a/SimpleSort/SimpleSortApp.cs b/SimpleSort/SimpleSortApp.cs index 31d3228d..27118606 100644 --- a/SimpleSort/SimpleSortApp.cs +++ b/SimpleSort/SimpleSortApp.cs @@ -426,9 +426,7 @@ namespace SabreTools } logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff")); - SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date, - toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, logger); - ss.RebuildToOutput(); + FileTools.RebuildToOutput(datdata, inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, 4, logger); } ///