mirror of
https://github.com/claunia/SabreTools.git
synced 2025-12-16 19:14:27 +00:00
[ALL] Port rest of features over from SimpleSort
This commit is contained in:
Matt Nadareski
@@ -184,8 +184,7 @@ namespace SabreTools
|
|||||||
|
|
||||||
// Create the sorting object to use and rebuild the needed files
|
// Create the sorting object to use and rebuild the needed files
|
||||||
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1));
|
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers((onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1), (onlyNeeded ? 0 : 1));
|
||||||
SimpleSort ss = new SimpleSort(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, _logger);
|
FileTools.RebuildToOutput(need, onlyDirs, _depots.Keys.ToList()[0], _tmpdir, false, false, false, false, true, true, asl, false, null, 4, _logger);
|
||||||
ss.RebuildToOutput();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -223,8 +222,7 @@ namespace SabreTools
|
|||||||
|
|
||||||
// Now scan all of those depots and rebuild
|
// Now scan all of those depots and rebuild
|
||||||
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1);
|
ArchiveScanLevel asl = ArchiveTools.GetArchiveScanLevelFromNumbers(1, 1, 1, 1);
|
||||||
SimpleSort ss = new SimpleSort(datFile, onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, _logger);
|
FileTools.RebuildToOutput(datFile, onlineDepots, outputFolder, _tmpdir, true, false, false, false, copy, copy, asl, false, null, 4, _logger);
|
||||||
ss.RebuildToOutput();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,925 +0,0 @@
|
|||||||
using System;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
|
|
||||||
namespace SabreTools.Helper
|
|
||||||
{
|
|
||||||
public class SimpleSort
|
|
||||||
{
|
|
||||||
// Private instance variables
|
|
||||||
private DatFile _datdata;
|
|
||||||
private List<string> _inputs;
|
|
||||||
private string _outDir;
|
|
||||||
private string _tempDir;
|
|
||||||
private bool _quickScan;
|
|
||||||
private bool _date;
|
|
||||||
private bool _toFolder;
|
|
||||||
private bool _delete;
|
|
||||||
private bool _tgz;
|
|
||||||
private bool _romba;
|
|
||||||
private bool _updateDat;
|
|
||||||
private ArchiveScanLevel _archiveScanLevel;
|
|
||||||
private string _headerToCheckAgainst;
|
|
||||||
private Logger _logger;
|
|
||||||
private int _maxDegreeOfParallelism = 4; // Hardcoded for now, should be an input later
|
|
||||||
|
|
||||||
// Other private variables
|
|
||||||
private int _cursorTop;
|
|
||||||
private int _cursorLeft;
|
|
||||||
private DatFile _matched;
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Create a new SimpleSort object
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="datdata">DAT to compare against</param>
|
|
||||||
/// <param name="inputs">List of input files/folders to check</param>
|
|
||||||
/// <param name="outDir">Output directory to use to build to</param>
|
|
||||||
/// <param name="tempDir">Temporary directory for archive extraction</param>
|
|
||||||
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
|
|
||||||
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
|
|
||||||
/// <param name="toFolder">True if files should be output to folder, false otherwise</param>
|
|
||||||
/// <param name="delete">True if input files should be deleted, false otherwise</param>
|
|
||||||
/// <param name="tgz">True if output files should be written to TorrentGZ instead of TorrentZip</param>
|
|
||||||
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
|
|
||||||
/// <param name="archiveScanLevel">ArchiveScanLevel representing the archive handling levels</param>
|
|
||||||
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
|
|
||||||
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
|
|
||||||
/// <param name="logger">Logger object for file and console output</param>
|
|
||||||
public SimpleSort(DatFile datdata, List<string> inputs, string outDir, string tempDir,
|
|
||||||
bool quickScan, bool date, bool toFolder, bool delete, bool tgz, bool romba,
|
|
||||||
ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst, Logger logger)
|
|
||||||
{
|
|
||||||
_datdata = datdata;
|
|
||||||
_inputs = inputs;
|
|
||||||
_outDir = (outDir == "" ? "Rebuild" : outDir);
|
|
||||||
_tempDir = (String.IsNullOrEmpty(tempDir) ? Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()) : tempDir);
|
|
||||||
_quickScan = quickScan;
|
|
||||||
_date = date;
|
|
||||||
_toFolder = toFolder;
|
|
||||||
_delete = delete;
|
|
||||||
_tgz = tgz;
|
|
||||||
_romba = romba;
|
|
||||||
_archiveScanLevel = archiveScanLevel;
|
|
||||||
_updateDat = updateDat;
|
|
||||||
_headerToCheckAgainst = headerToCheckAgainst;
|
|
||||||
_logger = logger;
|
|
||||||
|
|
||||||
_cursorTop = Console.CursorTop;
|
|
||||||
_cursorLeft = Console.CursorLeft;
|
|
||||||
_matched = new DatFile
|
|
||||||
{
|
|
||||||
Files = new SortedDictionary<string, List<DatItem>>(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Process the DAT and find all matches in input files and folders
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if rebuilding was a success, false otherwise</returns>
|
|
||||||
/// <remarks>
|
|
||||||
/// This currently processes files as follows:
|
|
||||||
/// 1) Get all file names from the input files/folders
|
|
||||||
/// 2) Loop through and process each file individually
|
|
||||||
/// a) Hash the file
|
|
||||||
/// b) Check against the DAT for duplicates
|
|
||||||
/// c) Check for headers
|
|
||||||
/// d) Check headerless rom for duplicates
|
|
||||||
///
|
|
||||||
/// This is actually rather slow and inefficient. See below for more correct implemenation
|
|
||||||
/// </remarks>
|
|
||||||
public bool RebuildToOutput()
|
|
||||||
{
|
|
||||||
// First, check that the output directory exists
|
|
||||||
if (!Directory.Exists(_outDir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(_outDir);
|
|
||||||
_outDir = Path.GetFullPath(_outDir);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Then create or clean the temp directory
|
|
||||||
if (!Directory.Exists(_tempDir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(_tempDir);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
FileTools.CleanDirectory(_tempDir);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
_logger.User("Retrieving list all files from input");
|
|
||||||
DateTime start = DateTime.Now;
|
|
||||||
|
|
||||||
// Create a list of just files from inputs
|
|
||||||
List<string> files = new List<string>();
|
|
||||||
Parallel.ForEach(_inputs,
|
|
||||||
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism, },
|
|
||||||
input => {
|
|
||||||
if (File.Exists(input))
|
|
||||||
{
|
|
||||||
_logger.Verbose("File found: '" + input + "'");
|
|
||||||
files.Add(Path.GetFullPath(input));
|
|
||||||
}
|
|
||||||
else if (Directory.Exists(input))
|
|
||||||
{
|
|
||||||
_logger.Verbose("Directory found: '" + input + "'");
|
|
||||||
Parallel.ForEach(Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories),
|
|
||||||
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism, },
|
|
||||||
file =>
|
|
||||||
{
|
|
||||||
_logger.Verbose("File found: '" + file + "'");
|
|
||||||
files.Add(Path.GetFullPath(file));
|
|
||||||
});
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_logger.Error("'" + input + "' is not a file or directory!");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
_logger.User("Retrieving complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
|
|
||||||
|
|
||||||
// Then, loop through and check each of the inputs
|
|
||||||
_logger.User("Processing files:\n");
|
|
||||||
_cursorTop = Console.CursorTop;
|
|
||||||
for (int i = 0; i < files.Count; i++)
|
|
||||||
{
|
|
||||||
success &= RebuildToOutputHelper(files[i], i, files.Count);
|
|
||||||
if (_tempDir != Path.GetTempPath())
|
|
||||||
{
|
|
||||||
FileTools.CleanDirectory(_tempDir);
|
|
||||||
}
|
|
||||||
if (success && _delete)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Delete(files[i]);
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now one final delete of the temp directory
|
|
||||||
while (Directory.Exists(_tempDir))
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (_tempDir != Path.GetTempPath())
|
|
||||||
{
|
|
||||||
Directory.Delete(_tempDir, true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now output the stats for the built files
|
|
||||||
_logger.ClearBeneath(Constants.HeaderHeight);
|
|
||||||
Console.SetCursorPosition(0, Constants.HeaderHeight + 1);
|
|
||||||
_logger.User("Stats of the matched ROMs:");
|
|
||||||
StreamWriter sw = new StreamWriter(new MemoryStream());
|
|
||||||
_matched.OutputStats(sw, StatOutputFormat.None, _logger, recalculate: true, baddumpCol: true, nodumpCol: true);
|
|
||||||
sw.Dispose();
|
|
||||||
|
|
||||||
// Now output the fixdat based on the original input if asked
|
|
||||||
if (_updateDat)
|
|
||||||
{
|
|
||||||
_datdata.FileName = "fixDat_" + _datdata.FileName;
|
|
||||||
_datdata.Name = "fixDat_" + _datdata.Name;
|
|
||||||
_datdata.Description = "fixDat_" + _datdata.Description;
|
|
||||||
_datdata.OutputFormat = OutputFormat.Logiqx;
|
|
||||||
_datdata.WriteToFile("", _logger);
|
|
||||||
}
|
|
||||||
|
|
||||||
return success;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Process an individual file against the DAT for rebuilding
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="input">Name of the input file</param>
|
|
||||||
/// <param name="index">Index of the current file</param>
|
|
||||||
/// <param name="total">Total number of files</param>
|
|
||||||
/// <param name="recurse">True if this is in a recurse step and the file should be deleted, false otherwise (default)</param>
|
|
||||||
/// <returns>True if it was processed properly, false otherwise</returns>
|
|
||||||
private bool RebuildToOutputHelper(string input, int index, int total, bool recurse = false)
|
|
||||||
{
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
// Get the full path of the input for movement purposes
|
|
||||||
string percentage = (index == 0 ? "0.00" : Math.Round((100 * ((double)index / total)), 2, MidpointRounding.AwayFromZero).ToString());
|
|
||||||
string statement = percentage + "% - " + input;
|
|
||||||
_logger.ClearBeneath(_cursorTop + 1);
|
|
||||||
_logger.WriteExact(statement, _cursorTop, 0);
|
|
||||||
|
|
||||||
// Get if the file should be scanned internally and externally
|
|
||||||
bool shouldExternalScan, shouldInternalScan;
|
|
||||||
ArchiveTools.GetInternalExternalProcess(input, _archiveScanLevel, _logger, out shouldExternalScan, out shouldInternalScan);
|
|
||||||
|
|
||||||
// Hash and match the external files
|
|
||||||
if (shouldExternalScan)
|
|
||||||
{
|
|
||||||
Rom rom = FileTools.GetFileInfo(input, _logger);
|
|
||||||
|
|
||||||
// If we have a blank RomData, it's an error
|
|
||||||
if (rom.Name == null)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to find the matches to the file that was found
|
|
||||||
List<DatItem> foundroms = rom.GetDuplicates(_datdata, _logger);
|
|
||||||
_logger.Verbose("File '" + input + "' had " + foundroms.Count + " matches in the DAT!");
|
|
||||||
foreach (Rom found in foundroms)
|
|
||||||
{
|
|
||||||
_logger.Verbose("Matched name: " + found.Name);
|
|
||||||
|
|
||||||
// Add rom to the matched list
|
|
||||||
string key = found.Size + "-" + found.CRC;
|
|
||||||
if (_matched.Files.ContainsKey(key))
|
|
||||||
{
|
|
||||||
_matched.Files[key].Add(found);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> temp = new List<DatItem>();
|
|
||||||
temp.Add(found);
|
|
||||||
_matched.Files.Add(key, temp);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (_toFolder)
|
|
||||||
{
|
|
||||||
// Copy file to output directory
|
|
||||||
string gamedir = Path.Combine(_outDir, found.MachineName);
|
|
||||||
if (!Directory.Exists(gamedir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(gamedir);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.SHA1 : found.Name) + "'");
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Copy(input, Path.Combine(gamedir, Path.GetFileName(found.Name)));
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (_tgz)
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteToArchive(input, _outDir, found, _logger, date: _date);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now get the transformed file if it exists
|
|
||||||
SkipperRule rule = Skipper.GetMatchingRule(input, _headerToCheckAgainst, _logger);
|
|
||||||
|
|
||||||
// If we have have a non-empty rule, apply it
|
|
||||||
if (rule.Tests != null && rule.Tests.Count != 0)
|
|
||||||
{
|
|
||||||
// Otherwise, apply the rule to the file
|
|
||||||
string newinput = input + ".new";
|
|
||||||
rule.TransformFile(input, newinput, _logger);
|
|
||||||
Rom drom = FileTools.GetFileInfo(newinput, _logger);
|
|
||||||
|
|
||||||
// If we have a blank RomData, it's an error
|
|
||||||
if (String.IsNullOrEmpty(drom.Name))
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Try to find the matches to the file that was found
|
|
||||||
List<DatItem> founddroms = drom.GetDuplicates(_datdata, _logger);
|
|
||||||
_logger.Verbose("File '" + newinput + "' had " + founddroms.Count + " matches in the DAT!");
|
|
||||||
foreach (Rom found in founddroms)
|
|
||||||
{
|
|
||||||
// Add rom to the matched list
|
|
||||||
string key = found.Size + "-" + found.CRC;
|
|
||||||
if (_matched.Files.ContainsKey(key))
|
|
||||||
{
|
|
||||||
_matched.Files[key].Add(found);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> temp = new List<DatItem>();
|
|
||||||
temp.Add(found);
|
|
||||||
_matched.Files.Add(key, temp);
|
|
||||||
}
|
|
||||||
|
|
||||||
// First output the headerless rom
|
|
||||||
_logger.Verbose("Matched name: " + found.Name);
|
|
||||||
|
|
||||||
if (_toFolder)
|
|
||||||
{
|
|
||||||
// Copy file to output directory
|
|
||||||
string gamedir = Path.Combine(_outDir, found.MachineName);
|
|
||||||
if (!Directory.Exists(gamedir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(gamedir);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.SHA1 : found.Name) + "'");
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Copy(newinput, Path.Combine(gamedir, Path.GetFileName(found.Name)));
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (_tgz)
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteTorrentGZ(newinput, _outDir, _romba, _logger);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteToArchive(newinput, _outDir, found, _logger, date: _date);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Then output the headered rom (renamed)
|
|
||||||
Rom newfound = found;
|
|
||||||
newfound.Name = Path.GetFileNameWithoutExtension(newfound.Name) + " (" + rom.CRC + ")" + Path.GetExtension(newfound.Name);
|
|
||||||
newfound.Size = rom.Size;
|
|
||||||
newfound.CRC = rom.CRC;
|
|
||||||
newfound.MD5 = rom.MD5;
|
|
||||||
newfound.SHA1 = rom.SHA1;
|
|
||||||
|
|
||||||
// Add rom to the matched list
|
|
||||||
key = newfound.Size + "-" + newfound.CRC;
|
|
||||||
if (_matched.Files.ContainsKey(key))
|
|
||||||
{
|
|
||||||
_matched.Files[key].Add(newfound);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> temp = new List<DatItem>();
|
|
||||||
temp.Add(newfound);
|
|
||||||
_matched.Files.Add(key, temp);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (_toFolder)
|
|
||||||
{
|
|
||||||
// Copy file to output directory
|
|
||||||
string gamedir = Path.Combine(_outDir, found.MachineName);
|
|
||||||
if (!Directory.Exists(gamedir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(gamedir);
|
|
||||||
}
|
|
||||||
|
|
||||||
_logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + newfound.Name + "'");
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Copy(input, Path.Combine(gamedir, Path.GetFileName(newfound.Name)));
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_logger.Verbose("Matched name: " + newfound.Name);
|
|
||||||
if (_tgz)
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteTorrentGZ(input, _outDir, _romba, _logger);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteToArchive(input, _outDir, newfound, _logger, date: _date);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now remove this temporary file
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Delete(newinput);
|
|
||||||
}
|
|
||||||
catch
|
|
||||||
{
|
|
||||||
// Don't log file deletion errors
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we should scan the file as an archive
|
|
||||||
if (shouldInternalScan)
|
|
||||||
{
|
|
||||||
// If external scanning is enabled, use that method instead
|
|
||||||
if (_quickScan)
|
|
||||||
{
|
|
||||||
_logger.Verbose("Beginning quick scan of contents from '" + input + "'");
|
|
||||||
List<Rom> internalRomData = ArchiveTools.GetArchiveFileInfo(input, _logger);
|
|
||||||
_logger.Verbose(internalRomData.Count + " entries found in '" + input + "'");
|
|
||||||
|
|
||||||
// If the list is populated, then the file was a filled archive
|
|
||||||
if (internalRomData.Count > 0)
|
|
||||||
{
|
|
||||||
foreach (Rom rom in internalRomData)
|
|
||||||
{
|
|
||||||
// Try to find the matches to the file that was found
|
|
||||||
List<DatItem> foundroms = rom.GetDuplicates(_datdata, _logger);
|
|
||||||
_logger.Verbose("File '" + rom.Name + "' had " + foundroms.Count + " matches in the DAT!");
|
|
||||||
foreach (Rom found in foundroms)
|
|
||||||
{
|
|
||||||
// Add rom to the matched list
|
|
||||||
string key = found.Size + "-" + found.CRC;
|
|
||||||
if (_matched.Files.ContainsKey(key))
|
|
||||||
{
|
|
||||||
_matched.Files[key].Add(found);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> temp = new List<DatItem>();
|
|
||||||
temp.Add(found);
|
|
||||||
_matched.Files.Add(key, temp);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (_toFolder)
|
|
||||||
{
|
|
||||||
// Copy file to output directory
|
|
||||||
_logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + found.Name + "'");
|
|
||||||
string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, _tempDir, _logger);
|
|
||||||
if (File.Exists(outfile))
|
|
||||||
{
|
|
||||||
string gamedir = Path.Combine(_outDir, found.MachineName);
|
|
||||||
if (!Directory.Exists(gamedir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(gamedir);
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Move(outfile, Path.Combine(gamedir, Path.GetFileName(found.Name)));
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Copy file between archives
|
|
||||||
_logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (_tgz ? found.SHA1 : found.Name) + "'");
|
|
||||||
|
|
||||||
if (Build.MonoEnvironment || _tgz)
|
|
||||||
{
|
|
||||||
string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, _tempDir, _logger);
|
|
||||||
if (File.Exists(outfile))
|
|
||||||
{
|
|
||||||
if (_tgz)
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteTorrentGZ(outfile, _outDir, _romba, _logger);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ArchiveTools.WriteToArchive(outfile, _outDir, found, _logger);
|
|
||||||
}
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Delete(outfile);
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
ArchiveTools.CopyFileBetweenArchives(input, _outDir, rom.Name, found, _logger);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Now, if the file is a supported archive type, also run on all files within
|
|
||||||
bool encounteredErrors = ArchiveTools.ExtractArchive(input, _tempDir, _archiveScanLevel, _logger);
|
|
||||||
|
|
||||||
// Remove the current file if we are in recursion so it's not picked up in the next step
|
|
||||||
if (recurse)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
File.Delete(input);
|
|
||||||
}
|
|
||||||
catch (Exception)
|
|
||||||
{
|
|
||||||
// Don't log file deletion errors
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no errors were encountered, we loop through the temp directory
|
|
||||||
if (!encounteredErrors)
|
|
||||||
{
|
|
||||||
_logger.Verbose("Archive found! Successfully extracted");
|
|
||||||
foreach (string file in Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories))
|
|
||||||
{
|
|
||||||
success &= RebuildToOutputHelper(file, index, total, true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return success;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Process the DAT and find all matches in input files and folders
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if rebuilding was a success, false otherwise</returns>
|
|
||||||
/// <remarks>
|
|
||||||
/// This implemenation of the code should do the following:
|
|
||||||
/// 1) Get all file names from the input files/folders (parallel)
|
|
||||||
/// 2) Loop through and get the file info from every file (including headerless)
|
|
||||||
/// 3) Find all duplicate files in the input DAT(s)
|
|
||||||
/// 4) Order by output game
|
|
||||||
/// 5) Rebuild all files
|
|
||||||
/// </remarks>
|
|
||||||
public bool RebuiltToOutputAlternate()
|
|
||||||
{
|
|
||||||
// First, check that the output directory exists
|
|
||||||
if (!Directory.Exists(_outDir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(_outDir);
|
|
||||||
_outDir = Path.GetFullPath(_outDir);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Then create or clean the temp directory
|
|
||||||
if (!Directory.Exists(_tempDir))
|
|
||||||
{
|
|
||||||
Directory.CreateDirectory(_tempDir);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
FileTools.CleanDirectory(_tempDir);
|
|
||||||
}
|
|
||||||
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
#region Find all files
|
|
||||||
|
|
||||||
// Create a list of just files from inputs
|
|
||||||
_logger.User("Finding all files...");
|
|
||||||
List<string> files = new List<string>();
|
|
||||||
Parallel.ForEach(_inputs,
|
|
||||||
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
|
|
||||||
input =>
|
|
||||||
{
|
|
||||||
if (File.Exists(input))
|
|
||||||
{
|
|
||||||
_logger.Verbose("File found: '" + input + "'");
|
|
||||||
lock (files)
|
|
||||||
{
|
|
||||||
files.Add(Path.GetFullPath(input));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (Directory.Exists(input))
|
|
||||||
{
|
|
||||||
_logger.Verbose("Directory found: '" + input + "'");
|
|
||||||
|
|
||||||
List<string> infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList();
|
|
||||||
Parallel.ForEach(infiles,
|
|
||||||
new ParallelOptions { MaxDegreeOfParallelism = _maxDegreeOfParallelism },
|
|
||||||
file =>
|
|
||||||
{
|
|
||||||
_logger.Verbose("File found: '" + input + "'");
|
|
||||||
lock (files)
|
|
||||||
{
|
|
||||||
files.Add(Path.GetFullPath(file));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
_logger.Error("'" + input + "' is not a file or directory!");
|
|
||||||
}
|
|
||||||
});
|
|
||||||
_logger.User("Finding files complete!");
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region Get source file information
|
|
||||||
|
|
||||||
// Now loop through all of the files and check them, DFD style
|
|
||||||
_logger.User("Getting source file information...");
|
|
||||||
DatFile matchdat = new DatFile
|
|
||||||
{
|
|
||||||
Files = new SortedDictionary<string, List<DatItem>>(),
|
|
||||||
};
|
|
||||||
foreach (string file in files)
|
|
||||||
{
|
|
||||||
// Get if the file should be scanned internally and externally
|
|
||||||
bool shouldExternalScan, shouldInternalScan;
|
|
||||||
ArchiveTools.GetInternalExternalProcess(file, _archiveScanLevel, _logger, out shouldExternalScan, out shouldInternalScan);
|
|
||||||
|
|
||||||
// Hash and match the external files
|
|
||||||
if (shouldExternalScan)
|
|
||||||
{
|
|
||||||
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
|
|
||||||
}
|
|
||||||
|
|
||||||
// If we should scan the file as an archive
|
|
||||||
if (shouldInternalScan)
|
|
||||||
{
|
|
||||||
// If external scanning is enabled, use that method instead
|
|
||||||
if (_quickScan)
|
|
||||||
{
|
|
||||||
_logger.Verbose("Beginning quick scan of contents from '" + file + "'");
|
|
||||||
List<Rom> internalRomData = ArchiveTools.GetArchiveFileInfo(file, _logger);
|
|
||||||
_logger.Verbose(internalRomData.Count + " entries found in '" + file + "'");
|
|
||||||
|
|
||||||
// Now add all of the roms to the DAT
|
|
||||||
for (int i = 0; i < internalRomData.Count; i++)
|
|
||||||
{
|
|
||||||
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Otherwise, try to extract the file to the temp folder
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Now, if the file is a supported archive type, also run on all files within
|
|
||||||
bool encounteredErrors = ArchiveTools.ExtractArchive(file, _tempDir, _archiveScanLevel, _logger);
|
|
||||||
|
|
||||||
// If we succeeded in extracting, loop through the files
|
|
||||||
if (!encounteredErrors)
|
|
||||||
{
|
|
||||||
List<string> extractedFiles = Directory.EnumerateFiles(_tempDir, "*", SearchOption.AllDirectories).ToList();
|
|
||||||
foreach (string extractedFile in extractedFiles)
|
|
||||||
{
|
|
||||||
RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, _logger);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Otherwise, skip extracting and just get information on the file itself (if we didn't already)
|
|
||||||
else if (!shouldExternalScan)
|
|
||||||
{
|
|
||||||
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, _logger);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clean the temp directory for the next round
|
|
||||||
if (Directory.Exists(_tempDir))
|
|
||||||
{
|
|
||||||
FileTools.CleanDirectory(_tempDir);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_logger.User("Getting source file information complete!");
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region Find all files to rebuild and bucket by game
|
|
||||||
|
|
||||||
// Create a dictionary of from/to Rom mappings
|
|
||||||
Dictionary<DatItem, DatItem> toFromMap = new Dictionary<DatItem, DatItem>();
|
|
||||||
|
|
||||||
// Now populate it
|
|
||||||
foreach (string key in matchdat.Files.Keys)
|
|
||||||
{
|
|
||||||
foreach (DatItem rom in matchdat.Files[key])
|
|
||||||
{
|
|
||||||
List<DatItem> matched = rom.GetDuplicates(_datdata, _logger, true);
|
|
||||||
foreach (DatItem match in matched)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
toFromMap.Add(match, rom);
|
|
||||||
}
|
|
||||||
catch { }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Then bucket the keys by game for better output
|
|
||||||
SortedDictionary<string, List<DatItem>> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, _logger);
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region Rebuild all files
|
|
||||||
|
|
||||||
// At this point, we have "toFromMap" which maps output files to input files as well as
|
|
||||||
// as SortedDictionary called keysByGame which is the output files sorted by game in
|
|
||||||
// alphabetical order. We should be able to use these to do everything we need =)
|
|
||||||
|
|
||||||
// Now write out each game sequentially
|
|
||||||
foreach (string key in keysByGame.Keys)
|
|
||||||
{
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
return success;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::"
|
|
||||||
/// </summary>
|
|
||||||
/// <param name="file">Name of the file to attempt to add</param>
|
|
||||||
/// <param name="matchdat">Reference to the Dat to add to</param>
|
|
||||||
/// <param name="logger">Logger object for file and console output</param>
|
|
||||||
/// <returns>True if the file could be added, false otherwise</returns>
|
|
||||||
private bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, Logger logger)
|
|
||||||
{
|
|
||||||
Rom rom = FileTools.GetFileInfo(file, logger);
|
|
||||||
|
|
||||||
// If we have a blank RomData, it's an error
|
|
||||||
if (rom.Name == null)
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Otherwise, set the machine name as the full path to the file
|
|
||||||
rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file));
|
|
||||||
|
|
||||||
// Add the rom information to the Dat
|
|
||||||
string key = rom.Size + "-" + rom.CRC;
|
|
||||||
if (matchdat.Files.ContainsKey(key))
|
|
||||||
{
|
|
||||||
matchdat.Files[key].Add(rom);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> temp = new List<DatItem>();
|
|
||||||
temp.Add(rom);
|
|
||||||
matchdat.Files.Add(key, temp);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now attempt to see if the file has a header
|
|
||||||
FileStream input = File.OpenRead(file);
|
|
||||||
SkipperRule rule = Skipper.GetMatchingRule(input, _headerToCheckAgainst, _logger);
|
|
||||||
|
|
||||||
// If there's a match, get the new information from the stream
|
|
||||||
if (rule.Tests != null && rule.Tests.Count != 0)
|
|
||||||
{
|
|
||||||
// Create the input and output streams
|
|
||||||
MemoryStream output = new MemoryStream();
|
|
||||||
|
|
||||||
// Transform the stream and get the information from it
|
|
||||||
rule.TransformStream(input, output, _logger, false, true);
|
|
||||||
Rom romNH = FileTools.GetStreamInfo(output, output.Length);
|
|
||||||
romNH.Name = "HEAD::" + rom.Name;
|
|
||||||
romNH.MachineName = rom.MachineName;
|
|
||||||
|
|
||||||
// Add the rom information to the Dat
|
|
||||||
key = romNH.Size + "-" + romNH.CRC;
|
|
||||||
if (matchdat.Files.ContainsKey(key))
|
|
||||||
{
|
|
||||||
matchdat.Files[key].Add(romNH);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> temp = new List<DatItem>();
|
|
||||||
temp.Add(romNH);
|
|
||||||
matchdat.Files.Add(key, temp);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dispose of the stream
|
|
||||||
output.Dispose();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dispose of the stream
|
|
||||||
input.Dispose();
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Clean an individual folder based on the DAT
|
|
||||||
/// </summary>
|
|
||||||
/// <returns>True if the cleaning succeeded, false otherwise</returns>
|
|
||||||
/// <remarks>This method is incomplete, it need to be finished before it can be used</remarks>
|
|
||||||
private bool InplaceRebuild()
|
|
||||||
{
|
|
||||||
bool success = true;
|
|
||||||
|
|
||||||
/*
|
|
||||||
The process of rebuilding inplace is as follows:
|
|
||||||
0) Resort the input roms by Game since that's more important in this case
|
|
||||||
1) Scan the current folder according to the level specified (no recursion)
|
|
||||||
a - If file is a match in all aspects, set correct flag and pass
|
|
||||||
+ If the file has a header, skip?
|
|
||||||
b - If file is a match in hash but not name, rename, set correct flag and pass
|
|
||||||
c - If file is not a match, extract it to the output folder and remove from archive, set incorrect flag
|
|
||||||
2) For all files that have been removed, check to see if they could be rebuilt to another location
|
|
||||||
a - This behaves similarly (and indeed could call) "RebuildToFolder"
|
|
||||||
b - If a file is a match and rebuilt, remove it from the output folder
|
|
||||||
*/
|
|
||||||
|
|
||||||
// Assuming archived sets, move all toplevel folders to temp
|
|
||||||
foreach (string directory in Directory.EnumerateDirectories(_outDir, "*", SearchOption.TopDirectoryOnly))
|
|
||||||
{
|
|
||||||
Directory.Move(directory, Path.Combine(_tempDir, Path.GetFileNameWithoutExtension(directory)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now process the inputs (assumed that it's archived sets as of right now
|
|
||||||
Dictionary<string, List<DatItem>> scanned = new Dictionary<string, List<DatItem>>();
|
|
||||||
foreach (string archive in Directory.EnumerateFiles(_outDir, "*", SearchOption.AllDirectories))
|
|
||||||
{
|
|
||||||
// If we are in quickscan, get the list of roms that way
|
|
||||||
List<Rom> roms = new List<Rom>();
|
|
||||||
if (_quickScan)
|
|
||||||
{
|
|
||||||
roms = ArchiveTools.GetArchiveFileInfo(Path.GetFullPath(archive), _logger);
|
|
||||||
}
|
|
||||||
// Otherwise, extract it and get info one by one
|
|
||||||
else
|
|
||||||
{
|
|
||||||
string temparcdir = Path.Combine(_tempDir, Path.GetFileNameWithoutExtension(archive));
|
|
||||||
ArchiveTools.ExtractArchive(Path.GetFullPath(archive), temparcdir, _logger);
|
|
||||||
foreach (string tempfile in Directory.EnumerateFiles(temparcdir, "*", SearchOption.AllDirectories))
|
|
||||||
{
|
|
||||||
roms.Add(FileTools.GetFileInfo(Path.GetFullPath(tempfile), _logger));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clear the temporary archive directory
|
|
||||||
FileTools.CleanDirectory(temparcdir);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Then add each of the found files to the new dictionary
|
|
||||||
foreach (Rom rom in roms)
|
|
||||||
{
|
|
||||||
string key = rom.Size + "-" + rom.CRC;
|
|
||||||
if (scanned.ContainsKey(key))
|
|
||||||
{
|
|
||||||
scanned[key].Add(rom);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> templist = new List<DatItem>();
|
|
||||||
templist.Add(rom);
|
|
||||||
scanned.Add(key, templist);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If nothing was found, we that it was successful
|
|
||||||
if (scanned.Count == 0)
|
|
||||||
{
|
|
||||||
return success;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now that we have all of the from DAT and from folder roms, we try to match them, removing the perfect matches
|
|
||||||
Dictionary<string, List<DatItem>> remove = new Dictionary<string, List<DatItem>>();
|
|
||||||
foreach (string key in scanned.Keys)
|
|
||||||
{
|
|
||||||
// If the key doesn't even exist in the DAT, then mark the entire key for removal
|
|
||||||
if (!_datdata.Files.ContainsKey(key))
|
|
||||||
{
|
|
||||||
if (remove.ContainsKey(key))
|
|
||||||
{
|
|
||||||
remove[key].AddRange(scanned[key]);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
remove.Add(key, scanned[key]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Otherwise check each of the values individually
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> romsList = _datdata.Files[key];
|
|
||||||
List<DatItem> scannedList = scanned[key];
|
|
||||||
foreach (Rom rom in scannedList)
|
|
||||||
{
|
|
||||||
if (!romsList.Contains(rom))
|
|
||||||
{
|
|
||||||
if (remove.ContainsKey(key))
|
|
||||||
{
|
|
||||||
remove[key].Add(rom);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
List<DatItem> templist = new List<DatItem>();
|
|
||||||
templist.Add(rom);
|
|
||||||
remove.Add(key, templist);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// At this point, we have the complete list of files from the DAT, a complete
|
|
||||||
// list of files that were scanned from the archives, and a complete list of
|
|
||||||
// the files to be removed because they aren't matches. I think at this point,
|
|
||||||
// we need to see if any of the files in "removed" can be rebuilt to something
|
|
||||||
// that is missing. But we don't have a list of missings, so how do we get this
|
|
||||||
// set of roms? Missing would be (_datdata.Roms - matches) I think. So if we
|
|
||||||
// get this additional set, we then run it against the "removed" set and rebuild
|
|
||||||
// as we go based on what we can do. Here is where we need some smarts. If the
|
|
||||||
// game to rebuild from and to are the same, we want to copy within. You
|
|
||||||
// should create a new helper function that "renames" an entry within the same
|
|
||||||
// archive to help this along. Everything else rebuilding should be copied from
|
|
||||||
// archive to archive. Once remove has been traversed, we will extract and remove
|
|
||||||
// all of the files that have been found and put them in the temporary folder.
|
|
||||||
|
|
||||||
return success;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -111,7 +111,6 @@
|
|||||||
<Compile Include="Objects\Dat\Release.cs" />
|
<Compile Include="Objects\Dat\Release.cs" />
|
||||||
<Compile Include="Objects\Dat\Sample.cs" />
|
<Compile Include="Objects\Dat\Sample.cs" />
|
||||||
<Compile Include="Objects\Dat\Rom.cs" />
|
<Compile Include="Objects\Dat\Rom.cs" />
|
||||||
<Compile Include="Objects\SimpleSort.cs" />
|
|
||||||
<Compile Include="Objects\Archive\ZipFileEntry.cs" />
|
<Compile Include="Objects\Archive\ZipFileEntry.cs" />
|
||||||
<Compile Include="Objects\Archive\ZipFile.cs" />
|
<Compile Include="Objects\Archive\ZipFile.cs" />
|
||||||
<Compile Include="Objects\Skippers\Skipper.cs" />
|
<Compile Include="Objects\Skippers\Skipper.cs" />
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ using System.Collections.Generic;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Security.Cryptography;
|
using System.Security.Cryptography;
|
||||||
|
using System.Threading.Tasks;
|
||||||
using System.Xml;
|
using System.Xml;
|
||||||
using System.Xml.Schema;
|
using System.Xml.Schema;
|
||||||
|
|
||||||
@@ -560,7 +561,7 @@ namespace SabreTools.Helper
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
FileTools.CleanDirectory(tempDir);
|
CleanDirectory(tempDir);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -618,6 +619,774 @@ namespace SabreTools.Helper
|
|||||||
return success;
|
return success;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Process the DAT and find all matches in input files and folders
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="datFile">DAT to compare against</param>
|
||||||
|
/// <param name="inputs">List of input files/folders to check</param>
|
||||||
|
/// <param name="outDir">Output directory to use to build to</param>
|
||||||
|
/// <param name="tempDir">Temporary directory for archive extraction</param>
|
||||||
|
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
|
||||||
|
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
|
||||||
|
/// <param name="toFolder">True if files should be output to folder, false otherwise</param>
|
||||||
|
/// <param name="delete">True if input files should be deleted, false otherwise</param>
|
||||||
|
/// <param name="tgz">True if output files should be written to TorrentGZ instead of TorrentZip</param>
|
||||||
|
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
|
||||||
|
/// <param name="archiveScanLevel">ArchiveScanLevel representing the archive handling levels</param>
|
||||||
|
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
|
||||||
|
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
|
||||||
|
/// <param name="logger">Logger object for file and console output</param>
|
||||||
|
/// <returns>True if rebuilding was a success, false otherwise</returns>
|
||||||
|
/// <remarks>
|
||||||
|
/// This currently processes files as follows:
|
||||||
|
/// 1) Get all file names from the input files/folders
|
||||||
|
/// 2) Loop through and process each file individually
|
||||||
|
/// a) Hash the file
|
||||||
|
/// b) Check against the DAT for duplicates
|
||||||
|
/// c) Check for headers
|
||||||
|
/// d) Check headerless rom for duplicates
|
||||||
|
///
|
||||||
|
/// This is actually rather slow and inefficient. See below for more correct implemenation
|
||||||
|
/// </remarks>
|
||||||
|
public static bool RebuildToOutput(DatFile datFile, List<string> inputs, string outDir, string tempDir, bool quickScan, bool date,
|
||||||
|
bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst,
|
||||||
|
int maxDegreeOfParallelism, Logger logger)
|
||||||
|
{
|
||||||
|
// First, check that the output directory exists
|
||||||
|
if (!Directory.Exists(outDir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(outDir);
|
||||||
|
outDir = Path.GetFullPath(outDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then create or clean the temp directory
|
||||||
|
if (!Directory.Exists(tempDir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(tempDir);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
CleanDirectory(tempDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool success = true;
|
||||||
|
DatFile matched = new DatFile();
|
||||||
|
|
||||||
|
logger.User("Retrieving list all files from input");
|
||||||
|
DateTime start = DateTime.Now;
|
||||||
|
|
||||||
|
// Create a list of just files from inputs
|
||||||
|
List<string> files = new List<string>();
|
||||||
|
Parallel.ForEach(inputs,
|
||||||
|
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, },
|
||||||
|
input => {
|
||||||
|
if (File.Exists(input))
|
||||||
|
{
|
||||||
|
logger.Verbose("File found: '" + input + "'");
|
||||||
|
files.Add(Path.GetFullPath(input));
|
||||||
|
}
|
||||||
|
else if (Directory.Exists(input))
|
||||||
|
{
|
||||||
|
logger.Verbose("Directory found: '" + input + "'");
|
||||||
|
Parallel.ForEach(Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories),
|
||||||
|
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism, },
|
||||||
|
file =>
|
||||||
|
{
|
||||||
|
logger.Verbose("File found: '" + file + "'");
|
||||||
|
files.Add(Path.GetFullPath(file));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.Error("'" + input + "' is not a file or directory!");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
logger.User("Retrieving complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
|
||||||
|
|
||||||
|
// Then, loop through and check each of the inputs
|
||||||
|
logger.User("Processing files:\n");
|
||||||
|
int cursorTop = Console.CursorTop;
|
||||||
|
for (int i = 0; i < files.Count; i++)
|
||||||
|
{
|
||||||
|
success &= RebuildToOutputHelper(datFile, matched, files[i], i, files.Count, cursorTop, outDir, tempDir, quickScan,
|
||||||
|
date, toFolder, delete, tgz, romba, archiveScanLevel, headerToCheckAgainst, logger);
|
||||||
|
if (tempDir != Path.GetTempPath())
|
||||||
|
{
|
||||||
|
CleanDirectory(tempDir);
|
||||||
|
}
|
||||||
|
if (success && delete)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Delete(files[i]);
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now one final delete of the temp directory
|
||||||
|
while (Directory.Exists(tempDir))
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
if (tempDir != Path.GetTempPath())
|
||||||
|
{
|
||||||
|
Directory.Delete(tempDir, true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now output the stats for the built files
|
||||||
|
logger.ClearBeneath(Constants.HeaderHeight);
|
||||||
|
Console.SetCursorPosition(0, Constants.HeaderHeight + 1);
|
||||||
|
logger.User("Stats of the matched ROMs:");
|
||||||
|
StreamWriter sw = new StreamWriter(new MemoryStream());
|
||||||
|
matched.OutputStats(sw, StatOutputFormat.None, logger, recalculate: true, baddumpCol: true, nodumpCol: true);
|
||||||
|
sw.Dispose();
|
||||||
|
|
||||||
|
// Now output the fixdat based on the original input if asked
|
||||||
|
if (updateDat)
|
||||||
|
{
|
||||||
|
datFile.FileName = "fixDat_" + datFile.FileName;
|
||||||
|
datFile.Name = "fixDat_" + datFile.Name;
|
||||||
|
datFile.Description = "fixDat_" + datFile.Description;
|
||||||
|
datFile.OutputFormat = OutputFormat.Logiqx;
|
||||||
|
datFile.WriteToFile("", logger);
|
||||||
|
}
|
||||||
|
|
||||||
|
return success;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Process an individual file against the DAT for rebuilding
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="datFile">DAT to compare against</param>
|
||||||
|
/// <param name="matched">List of files that were matched by the DAT</param>
|
||||||
|
/// <param name="input">Name of the input file</param>
|
||||||
|
/// <param name="index">Index of the current file</param>
|
||||||
|
/// <param name="total">Total number of files</param>
|
||||||
|
/// <param name="cursorTop">Top cursor position to use</param>
|
||||||
|
/// <param name="outDir">Output directory to use to build to</param>
|
||||||
|
/// <param name="tempDir">Temporary directory for archive extraction</param>
|
||||||
|
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
|
||||||
|
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
|
||||||
|
/// <param name="toFolder">True if files should be output to folder, false otherwise</param>
|
||||||
|
/// <param name="delete">True if input files should be deleted, false otherwise</param>
|
||||||
|
/// <param name="tgz">True if output files should be written to TorrentGZ instead of TorrentZip</param>
|
||||||
|
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
|
||||||
|
/// <param name="archiveScanLevel">ArchiveScanLevel representing the archive handling levels</param>
|
||||||
|
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
|
||||||
|
/// <param name="logger">Logger object for file and console output</param>
|
||||||
|
/// <param name="recurse">True if this is in a recurse step and the file should be deleted, false otherwise (default)</param>
|
||||||
|
/// <returns>True if it was processed properly, false otherwise</returns>
|
||||||
|
private static bool RebuildToOutputHelper(DatFile datFile, DatFile matched, string input, int index, int total, int cursorTop,
|
||||||
|
string outDir, string tempDir, bool quickScan, bool date, bool toFolder, bool delete, bool tgz, bool romba,
|
||||||
|
ArchiveScanLevel archiveScanLevel, string headerToCheckAgainst, Logger logger, bool recurse = false)
|
||||||
|
{
|
||||||
|
bool success = true;
|
||||||
|
|
||||||
|
// Get the full path of the input for movement purposes
|
||||||
|
string percentage = (index == 0 ? "0.00" : Math.Round((100 * ((double)index / total)), 2, MidpointRounding.AwayFromZero).ToString());
|
||||||
|
string statement = percentage + "% - " + input;
|
||||||
|
logger.ClearBeneath(cursorTop + 1);
|
||||||
|
logger.WriteExact(statement, cursorTop, 0);
|
||||||
|
|
||||||
|
// Get if the file should be scanned internally and externally
|
||||||
|
bool shouldExternalScan, shouldInternalScan;
|
||||||
|
ArchiveTools.GetInternalExternalProcess(input, archiveScanLevel, logger, out shouldExternalScan, out shouldInternalScan);
|
||||||
|
|
||||||
|
// Hash and match the external files
|
||||||
|
if (shouldExternalScan)
|
||||||
|
{
|
||||||
|
Rom rom = FileTools.GetFileInfo(input, logger);
|
||||||
|
|
||||||
|
// If we have a blank RomData, it's an error
|
||||||
|
if (rom.Name == null)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to find the matches to the file that was found
|
||||||
|
List<DatItem> foundroms = rom.GetDuplicates(datFile, logger);
|
||||||
|
logger.Verbose("File '" + input + "' had " + foundroms.Count + " matches in the DAT!");
|
||||||
|
foreach (Rom found in foundroms)
|
||||||
|
{
|
||||||
|
logger.Verbose("Matched name: " + found.Name);
|
||||||
|
|
||||||
|
// Add rom to the matched list
|
||||||
|
string key = found.Size + "-" + found.CRC;
|
||||||
|
if (matched.Files.ContainsKey(key))
|
||||||
|
{
|
||||||
|
matched.Files[key].Add(found);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
List<DatItem> temp = new List<DatItem>();
|
||||||
|
temp.Add(found);
|
||||||
|
matched.Files.Add(key, temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (toFolder)
|
||||||
|
{
|
||||||
|
// Copy file to output directory
|
||||||
|
string gamedir = Path.Combine(outDir, found.MachineName);
|
||||||
|
if (!Directory.Exists(gamedir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(gamedir);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Copy(input, Path.Combine(gamedir, Path.GetFileName(found.Name)));
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (tgz)
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteToArchive(input, outDir, found, logger, date: date);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now get the transformed file if it exists
|
||||||
|
SkipperRule rule = Skipper.GetMatchingRule(input, headerToCheckAgainst, logger);
|
||||||
|
|
||||||
|
// If we have have a non-empty rule, apply it
|
||||||
|
if (rule.Tests != null && rule.Tests.Count != 0)
|
||||||
|
{
|
||||||
|
// Otherwise, apply the rule to the file
|
||||||
|
string newinput = input + ".new";
|
||||||
|
rule.TransformFile(input, newinput, logger);
|
||||||
|
Rom drom = FileTools.GetFileInfo(newinput, logger);
|
||||||
|
|
||||||
|
// If we have a blank RomData, it's an error
|
||||||
|
if (String.IsNullOrEmpty(drom.Name))
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to find the matches to the file that was found
|
||||||
|
List<DatItem> founddroms = drom.GetDuplicates(datFile, logger);
|
||||||
|
logger.Verbose("File '" + newinput + "' had " + founddroms.Count + " matches in the DAT!");
|
||||||
|
foreach (Rom found in founddroms)
|
||||||
|
{
|
||||||
|
// Add rom to the matched list
|
||||||
|
string key = found.Size + "-" + found.CRC;
|
||||||
|
if (matched.Files.ContainsKey(key))
|
||||||
|
{
|
||||||
|
matched.Files[key].Add(found);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
List<DatItem> temp = new List<DatItem>();
|
||||||
|
temp.Add(found);
|
||||||
|
matched.Files.Add(key, temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
// First output the headerless rom
|
||||||
|
logger.Verbose("Matched name: " + found.Name);
|
||||||
|
|
||||||
|
if (toFolder)
|
||||||
|
{
|
||||||
|
// Copy file to output directory
|
||||||
|
string gamedir = Path.Combine(outDir, found.MachineName);
|
||||||
|
if (!Directory.Exists(gamedir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(gamedir);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Copy(newinput, Path.Combine(gamedir, Path.GetFileName(found.Name)));
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (tgz)
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteTorrentGZ(newinput, outDir, romba, logger);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteToArchive(newinput, outDir, found, logger, date: date);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then output the headered rom (renamed)
|
||||||
|
Rom newfound = found;
|
||||||
|
newfound.Name = Path.GetFileNameWithoutExtension(newfound.Name) + " (" + rom.CRC + ")" + Path.GetExtension(newfound.Name);
|
||||||
|
newfound.Size = rom.Size;
|
||||||
|
newfound.CRC = rom.CRC;
|
||||||
|
newfound.MD5 = rom.MD5;
|
||||||
|
newfound.SHA1 = rom.SHA1;
|
||||||
|
|
||||||
|
// Add rom to the matched list
|
||||||
|
key = newfound.Size + "-" + newfound.CRC;
|
||||||
|
if (matched.Files.ContainsKey(key))
|
||||||
|
{
|
||||||
|
matched.Files[key].Add(newfound);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
List<DatItem> temp = new List<DatItem>();
|
||||||
|
temp.Add(newfound);
|
||||||
|
matched.Files.Add(key, temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (toFolder)
|
||||||
|
{
|
||||||
|
// Copy file to output directory
|
||||||
|
string gamedir = Path.Combine(outDir, found.MachineName);
|
||||||
|
if (!Directory.Exists(gamedir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(gamedir);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + newfound.Name + "'");
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Copy(input, Path.Combine(gamedir, Path.GetFileName(newfound.Name)));
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.Verbose("Matched name: " + newfound.Name);
|
||||||
|
if (tgz)
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteTorrentGZ(input, outDir, romba, logger);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteToArchive(input, outDir, newfound, logger, date: date);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now remove this temporary file
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Delete(newinput);
|
||||||
|
}
|
||||||
|
catch
|
||||||
|
{
|
||||||
|
// Don't log file deletion errors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we should scan the file as an archive
|
||||||
|
if (shouldInternalScan)
|
||||||
|
{
|
||||||
|
// If external scanning is enabled, use that method instead
|
||||||
|
if (quickScan)
|
||||||
|
{
|
||||||
|
logger.Verbose("Beginning quick scan of contents from '" + input + "'");
|
||||||
|
List<Rom> internalRomData = ArchiveTools.GetArchiveFileInfo(input, logger);
|
||||||
|
logger.Verbose(internalRomData.Count + " entries found in '" + input + "'");
|
||||||
|
|
||||||
|
// If the list is populated, then the file was a filled archive
|
||||||
|
if (internalRomData.Count > 0)
|
||||||
|
{
|
||||||
|
foreach (Rom rom in internalRomData)
|
||||||
|
{
|
||||||
|
// Try to find the matches to the file that was found
|
||||||
|
List<DatItem> foundroms = rom.GetDuplicates(datFile, logger);
|
||||||
|
logger.Verbose("File '" + rom.Name + "' had " + foundroms.Count + " matches in the DAT!");
|
||||||
|
foreach (Rom found in foundroms)
|
||||||
|
{
|
||||||
|
// Add rom to the matched list
|
||||||
|
string key = found.Size + "-" + found.CRC;
|
||||||
|
if (matched.Files.ContainsKey(key))
|
||||||
|
{
|
||||||
|
matched.Files[key].Add(found);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
List<DatItem> temp = new List<DatItem>();
|
||||||
|
temp.Add(found);
|
||||||
|
matched.Files.Add(key, temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (toFolder)
|
||||||
|
{
|
||||||
|
// Copy file to output directory
|
||||||
|
logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + found.Name + "'");
|
||||||
|
string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, tempDir, logger);
|
||||||
|
if (File.Exists(outfile))
|
||||||
|
{
|
||||||
|
string gamedir = Path.Combine(outDir, found.MachineName);
|
||||||
|
if (!Directory.Exists(gamedir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(gamedir);
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Move(outfile, Path.Combine(gamedir, Path.GetFileName(found.Name)));
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Copy file between archives
|
||||||
|
logger.Verbose("Rebuilding file '" + Path.GetFileName(rom.Name) + "' to '" + (tgz ? found.SHA1 : found.Name) + "'");
|
||||||
|
|
||||||
|
if (Build.MonoEnvironment || tgz)
|
||||||
|
{
|
||||||
|
string outfile = ArchiveTools.ExtractSingleItemFromArchive(input, rom.Name, tempDir, logger);
|
||||||
|
if (File.Exists(outfile))
|
||||||
|
{
|
||||||
|
if (tgz)
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteTorrentGZ(outfile, outDir, romba, logger);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ArchiveTools.WriteToArchive(outfile, outDir, found, logger);
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Delete(outfile);
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
ArchiveTools.CopyFileBetweenArchives(input, outDir, rom.Name, found, logger);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Now, if the file is a supported archive type, also run on all files within
|
||||||
|
bool encounteredErrors = ArchiveTools.ExtractArchive(input, tempDir, archiveScanLevel, logger);
|
||||||
|
|
||||||
|
// Remove the current file if we are in recursion so it's not picked up in the next step
|
||||||
|
if (recurse)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
File.Delete(input);
|
||||||
|
}
|
||||||
|
catch (Exception)
|
||||||
|
{
|
||||||
|
// Don't log file deletion errors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no errors were encountered, we loop through the temp directory
|
||||||
|
if (!encounteredErrors)
|
||||||
|
{
|
||||||
|
logger.Verbose("Archive found! Successfully extracted");
|
||||||
|
foreach (string file in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories))
|
||||||
|
{
|
||||||
|
success &= RebuildToOutputHelper(datFile, matched, file, index, total, cursorTop, outDir, tempDir, quickScan,
|
||||||
|
date, toFolder, delete, tgz, romba, archiveScanLevel, headerToCheckAgainst, logger, recurse: true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return success;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Process the DAT and find all matches in input files and folders
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="datFile">DAT to compare against</param>
|
||||||
|
/// <param name="inputs">List of input files/folders to check</param>
|
||||||
|
/// <param name="outDir">Output directory to use to build to</param>
|
||||||
|
/// <param name="tempDir">Temporary directory for archive extraction</param>
|
||||||
|
/// <param name="quickScan">True to enable external scanning of archives, false otherwise</param>
|
||||||
|
/// <param name="date">True if the date from the DAT should be used if available, false otherwise</param>
|
||||||
|
/// <param name="toFolder">True if files should be output to folder, false otherwise</param>
|
||||||
|
/// <param name="delete">True if input files should be deleted, false otherwise</param>
|
||||||
|
/// <param name="tgz">True if output files should be written to TorrentGZ instead of TorrentZip</param>
|
||||||
|
/// <param name="romba">True if files should be output in Romba depot folders, false otherwise</param>
|
||||||
|
/// <param name="archiveScanLevel">ArchiveScanLevel representing the archive handling levels</param>
|
||||||
|
/// <param name="updateDat">True if the updated DAT should be output, false otherwise</param>
|
||||||
|
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
|
||||||
|
/// <param name="logger">Logger object for file and console output</param>
|
||||||
|
/// <returns>True if rebuilding was a success, false otherwise</returns>
|
||||||
|
/// <remarks>
|
||||||
|
/// This implemenation of the code should do the following:
|
||||||
|
/// 1) Get all file names from the input files/folders (parallel)
|
||||||
|
/// 2) Loop through and get the file info from every file (including headerless)
|
||||||
|
/// 3) Find all duplicate files in the input DAT(s)
|
||||||
|
/// 4) Order by output game
|
||||||
|
/// 5) Rebuild all files
|
||||||
|
/// </remarks>
|
||||||
|
public static bool RebuiltToOutputAlternate(DatFile datFile, List<string> inputs, string outDir, string tempDir, bool quickScan, bool date,
|
||||||
|
bool toFolder, bool delete, bool tgz, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat, string headerToCheckAgainst,
|
||||||
|
int maxDegreeOfParallelism, Logger logger)
|
||||||
|
{
|
||||||
|
// First, check that the output directory exists
|
||||||
|
if (!Directory.Exists(outDir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(outDir);
|
||||||
|
outDir = Path.GetFullPath(outDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then create or clean the temp directory
|
||||||
|
if (!Directory.Exists(tempDir))
|
||||||
|
{
|
||||||
|
Directory.CreateDirectory(tempDir);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
CleanDirectory(tempDir);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool success = true;
|
||||||
|
|
||||||
|
#region Find all files
|
||||||
|
|
||||||
|
// Create a list of just files from inputs
|
||||||
|
logger.User("Finding all files...");
|
||||||
|
List<string> files = new List<string>();
|
||||||
|
Parallel.ForEach(inputs,
|
||||||
|
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
|
||||||
|
input =>
|
||||||
|
{
|
||||||
|
if (File.Exists(input))
|
||||||
|
{
|
||||||
|
logger.Verbose("File found: '" + input + "'");
|
||||||
|
lock (files)
|
||||||
|
{
|
||||||
|
files.Add(Path.GetFullPath(input));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (Directory.Exists(input))
|
||||||
|
{
|
||||||
|
logger.Verbose("Directory found: '" + input + "'");
|
||||||
|
|
||||||
|
List<string> infiles = Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories).ToList();
|
||||||
|
Parallel.ForEach(infiles,
|
||||||
|
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
|
||||||
|
file =>
|
||||||
|
{
|
||||||
|
logger.Verbose("File found: '" + input + "'");
|
||||||
|
lock (files)
|
||||||
|
{
|
||||||
|
files.Add(Path.GetFullPath(file));
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.Error("'" + input + "' is not a file or directory!");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
logger.User("Finding files complete!");
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region Get source file information
|
||||||
|
|
||||||
|
// Now loop through all of the files and check them, DFD style
|
||||||
|
logger.User("Getting source file information...");
|
||||||
|
DatFile matchdat = new DatFile
|
||||||
|
{
|
||||||
|
Files = new SortedDictionary<string, List<DatItem>>(),
|
||||||
|
};
|
||||||
|
foreach (string file in files)
|
||||||
|
{
|
||||||
|
// Get if the file should be scanned internally and externally
|
||||||
|
bool shouldExternalScan, shouldInternalScan;
|
||||||
|
ArchiveTools.GetInternalExternalProcess(file, archiveScanLevel, logger, out shouldExternalScan, out shouldInternalScan);
|
||||||
|
|
||||||
|
// Hash and match the external files
|
||||||
|
if (shouldExternalScan)
|
||||||
|
{
|
||||||
|
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we should scan the file as an archive
|
||||||
|
if (shouldInternalScan)
|
||||||
|
{
|
||||||
|
// If external scanning is enabled, use that method instead
|
||||||
|
if (quickScan)
|
||||||
|
{
|
||||||
|
logger.Verbose("Beginning quick scan of contents from '" + file + "'");
|
||||||
|
List<Rom> internalRomData = ArchiveTools.GetArchiveFileInfo(file, logger);
|
||||||
|
logger.Verbose(internalRomData.Count + " entries found in '" + file + "'");
|
||||||
|
|
||||||
|
// Now add all of the roms to the DAT
|
||||||
|
for (int i = 0; i < internalRomData.Count; i++)
|
||||||
|
{
|
||||||
|
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Otherwise, try to extract the file to the temp folder
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Now, if the file is a supported archive type, also run on all files within
|
||||||
|
bool encounteredErrors = ArchiveTools.ExtractArchive(file, tempDir, archiveScanLevel, logger);
|
||||||
|
|
||||||
|
// If we succeeded in extracting, loop through the files
|
||||||
|
if (!encounteredErrors)
|
||||||
|
{
|
||||||
|
List<string> extractedFiles = Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories).ToList();
|
||||||
|
foreach (string extractedFile in extractedFiles)
|
||||||
|
{
|
||||||
|
RebuildToOutputAlternateParseRomHelper(extractedFile, ref matchdat, headerToCheckAgainst, logger);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Otherwise, skip extracting and just get information on the file itself (if we didn't already)
|
||||||
|
else if (!shouldExternalScan)
|
||||||
|
{
|
||||||
|
RebuildToOutputAlternateParseRomHelper(file, ref matchdat, headerToCheckAgainst, logger);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Clean the temp directory for the next round
|
||||||
|
if (Directory.Exists(tempDir))
|
||||||
|
{
|
||||||
|
CleanDirectory(tempDir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
logger.User("Getting source file information complete!");
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region Find all files to rebuild and bucket by game
|
||||||
|
|
||||||
|
// Create a dictionary of from/to Rom mappings
|
||||||
|
Dictionary<DatItem, DatItem> toFromMap = new Dictionary<DatItem, DatItem>();
|
||||||
|
|
||||||
|
// Now populate it
|
||||||
|
foreach (string key in matchdat.Files.Keys)
|
||||||
|
{
|
||||||
|
foreach (DatItem rom in matchdat.Files[key])
|
||||||
|
{
|
||||||
|
List<DatItem> matched = rom.GetDuplicates(datFile, logger, true);
|
||||||
|
foreach (DatItem match in matched)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
toFromMap.Add(match, rom);
|
||||||
|
}
|
||||||
|
catch { }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Then bucket the keys by game for better output
|
||||||
|
SortedDictionary<string, List<DatItem>> keysByGame = DatFile.BucketListByGame(toFromMap.Keys.ToList(), false, true, logger);
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
#region Rebuild all files
|
||||||
|
|
||||||
|
// At this point, we have "toFromMap" which maps output files to input files as well as
|
||||||
|
// as SortedDictionary called keysByGame which is the output files sorted by game in
|
||||||
|
// alphabetical order. We should be able to use these to do everything we need =)
|
||||||
|
|
||||||
|
// Now write out each game sequentially
|
||||||
|
foreach (string key in keysByGame.Keys)
|
||||||
|
{
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
#endregion
|
||||||
|
|
||||||
|
return success;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Wrap adding a file to the dictionary in custom DFD, files that matched a skipper a prefixed with "HEAD::"
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="file">Name of the file to attempt to add</param>
|
||||||
|
/// <param name="matchdat">Reference to the Dat to add to</param>
|
||||||
|
/// <param name="headerToCheckAgainst">Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise</param>
|
||||||
|
/// <param name="logger">Logger object for file and console output</param>
|
||||||
|
/// <returns>True if the file could be added, false otherwise</returns>
|
||||||
|
private static bool RebuildToOutputAlternateParseRomHelper(string file, ref DatFile matchdat, string headerToCheckAgainst, Logger logger)
|
||||||
|
{
|
||||||
|
Rom rom = FileTools.GetFileInfo(file, logger);
|
||||||
|
|
||||||
|
// If we have a blank RomData, it's an error
|
||||||
|
if (rom.Name == null)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Otherwise, set the machine name as the full path to the file
|
||||||
|
rom.MachineName = Path.GetDirectoryName(Path.GetFullPath(file));
|
||||||
|
|
||||||
|
// Add the rom information to the Dat
|
||||||
|
string key = rom.Size + "-" + rom.CRC;
|
||||||
|
if (matchdat.Files.ContainsKey(key))
|
||||||
|
{
|
||||||
|
matchdat.Files[key].Add(rom);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
List<DatItem> temp = new List<DatItem>();
|
||||||
|
temp.Add(rom);
|
||||||
|
matchdat.Files.Add(key, temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Now attempt to see if the file has a header
|
||||||
|
FileStream input = File.OpenRead(file);
|
||||||
|
SkipperRule rule = Skipper.GetMatchingRule(input, headerToCheckAgainst, logger);
|
||||||
|
|
||||||
|
// If there's a match, get the new information from the stream
|
||||||
|
if (rule.Tests != null && rule.Tests.Count != 0)
|
||||||
|
{
|
||||||
|
// Create the input and output streams
|
||||||
|
MemoryStream output = new MemoryStream();
|
||||||
|
|
||||||
|
// Transform the stream and get the information from it
|
||||||
|
rule.TransformStream(input, output, logger, false, true);
|
||||||
|
Rom romNH = FileTools.GetStreamInfo(output, output.Length);
|
||||||
|
romNH.Name = "HEAD::" + rom.Name;
|
||||||
|
romNH.MachineName = rom.MachineName;
|
||||||
|
|
||||||
|
// Add the rom information to the Dat
|
||||||
|
key = romNH.Size + "-" + romNH.CRC;
|
||||||
|
if (matchdat.Files.ContainsKey(key))
|
||||||
|
{
|
||||||
|
matchdat.Files[key].Add(romNH);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
List<DatItem> temp = new List<DatItem>();
|
||||||
|
temp.Add(romNH);
|
||||||
|
matchdat.Files.Add(key, temp);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dispose of the stream
|
||||||
|
output.Dispose();
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dispose of the stream
|
||||||
|
input.Dispose();
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Process the DAT and verify the output directory
|
/// Process the DAT and verify the output directory
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
|||||||
@@ -323,9 +323,7 @@ namespace SabreTools
|
|||||||
}
|
}
|
||||||
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
|
_logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
|
||||||
|
|
||||||
SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date,
|
FileTools.RebuildToOutput(datdata, inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, 4, _logger);
|
||||||
toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, _logger);
|
|
||||||
ss.RebuildToOutput();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -426,9 +426,7 @@ namespace SabreTools
|
|||||||
}
|
}
|
||||||
logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
|
logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
|
||||||
|
|
||||||
SimpleSort ss = new SimpleSort(datdata, inputs, outDir, tempDir, quickScan, date,
|
FileTools.RebuildToOutput(datdata, inputs, outDir, tempDir, quickScan, date, toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, 4, logger);
|
||||||
toFolder, delete, tgz, romba, asl, updateDat, headerToCheckAgainst, logger);
|
|
||||||
ss.RebuildToOutput();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
Reference in New Issue
Block a user