using System;
using System.Collections.Generic;
using System.Linq;
using SabreTools.Helper.Data;
using SabreTools.Helper.Skippers;
using SabreTools.Helper.Tools;
#if MONO
using System.IO;
#else
using Alphaleonis.Win32.Filesystem;
using SearchOption = System.IO.SearchOption;
#endif
namespace SabreTools.Helper.Dats
{
public partial class DatFile
{
#region Rebuilding and Verifying [MODULAR DONE, FOR NOW]
///
/// Process the DAT and find all matches in input files and folders
///
/// List of input files/folders to check
/// Output directory to use to build to
/// Temporary directory for archive extraction
/// True to enable external scanning of archives, false otherwise
/// True if the date from the DAT should be used if available, false otherwise
/// True if input files should be deleted, false otherwise
/// True if the DAT should be used as a filter instead of a template, false otherwise
/// Output format that files should be written to
/// True if files should be output in Romba depot folders, false otherwise
/// ArchiveScanLevel representing the archive handling levels
/// True if the updated DAT should be output, false otherwise
/// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise
/// Logger object for file and console output
/// True if rebuilding was a success, false otherwise
public bool RebuildToOutput(List inputs, string outDir, string tempDir, bool quickScan, bool date,
bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat,
string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger)
{
#region Perform setup
// If the DAT is not populated and inverse is not set, inform the user and quit
if (Count == 0 && !inverse)
{
logger.User("No entries were found to rebuild, exiting...");
return false;
}
// Check that the output directory exists
if (!Directory.Exists(outDir))
{
Directory.CreateDirectory(outDir);
outDir = Path.GetFullPath(outDir);
}
// Check the temp directory
if (String.IsNullOrEmpty(tempDir))
{
tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
}
// Then create or clean the temp directory
if (!Directory.Exists(tempDir))
{
Directory.CreateDirectory(tempDir);
}
else
{
FileTools.CleanDirectory(tempDir);
}
// Preload the Skipper list
int listcount = Skipper.List.Count;
#endregion
bool success = true;
#region Rebuild from sources in order
switch (outputFormat)
{
case OutputFormat.Folder:
logger.User("Rebuilding all files to directory");
break;
case OutputFormat.TapeArchive:
logger.User("Rebuilding all files to TAR");
break;
case OutputFormat.Torrent7Zip:
logger.User("Rebuilding all files to Torrent7Z");
break;
case OutputFormat.TorrentGzip:
logger.User("Rebuilding all files to TorrentGZ");
break;
case OutputFormat.TorrentLrzip:
logger.User("Rebuilding all files to TorrentLRZ");
break;
case OutputFormat.TorrentRar:
logger.User("Rebuilding all files to TorrentRAR");
break;
case OutputFormat.TorrentXZ:
logger.User("Rebuilding all files to TorrentXZ");
break;
case OutputFormat.TorrentZip:
logger.User("Rebuilding all files to TorrentZip");
break;
}
DateTime start = DateTime.Now;
// Now loop through all of the files in all of the inputs
foreach (string input in inputs)
{
// If the input is a file
if (File.Exists(input))
{
logger.Verbose("Checking file: '" + input + "'");
RebuildToOutputHelper(input, outDir, tempDir, quickScan, date, delete, inverse,
outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger);
}
// If the input is a directory
else if (Directory.Exists(input))
{
logger.Verbose("Checking directory: '" + input + "'");
foreach (string file in Directory.EnumerateFiles(input, "*", SearchOption.AllDirectories))
{
logger.Verbose("Checking file: '" + file + "'");
RebuildToOutputHelper(file, outDir, tempDir, quickScan, date, delete, inverse,
outputFormat, romba, archiveScanLevel, updateDat, headerToCheckAgainst, maxDegreeOfParallelism, logger);
}
}
}
logger.User("Rebuilding complete in: " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
#endregion
return success;
}
///
/// Attempt to add a file to the output if it matches
///
/// Name of the file to process
/// Output directory to use to build to
/// Temporary directory for archive extraction
/// True to enable external scanning of archives, false otherwise
/// True if the date from the DAT should be used if available, false otherwise
/// True if input files should be deleted, false otherwise
/// True if the DAT should be used as a filter instead of a template, false otherwise
/// Output format that files should be written to
/// True if files should be output in Romba depot folders, false otherwise
/// ArchiveScanLevel representing the archive handling levels
/// True if the updated DAT should be output, false otherwise
/// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise
/// Logger object for file and console output
private void RebuildToOutputHelper(string file, string outDir, string tempDir, bool quickScan, bool date,
bool delete, bool inverse, OutputFormat outputFormat, bool romba, ArchiveScanLevel archiveScanLevel, bool updateDat,
string headerToCheckAgainst, int maxDegreeOfParallelism, Logger logger)
{
// If we somehow have a null filename, return
if (file == null)
{
return;
}
// Define the temporary directory
string tempSubDir = Path.GetFullPath(Path.Combine(tempDir, Path.GetRandomFileName())) + Path.DirectorySeparatorChar;
// Set the deletion variables
bool usedExternally = false;
bool usedInternally = false;
// Get the required scanning level for the file
bool shouldExternalProcess = false;
bool shouldInternalProcess = false;
ArchiveTools.GetInternalExternalProcess(file, archiveScanLevel, logger, out shouldExternalProcess, out shouldInternalProcess);
// If we're supposed to scan the file externally
if (shouldExternalProcess)
{
Rom rom = FileTools.GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan, header: headerToCheckAgainst);
usedExternally = RebuildToOutputIndividual(rom, file, outDir, tempSubDir, date, inverse, outputFormat,
romba, updateDat, false /* isZip */, headerToCheckAgainst, logger);
}
// If we're supposed to scan the file internally
if (shouldInternalProcess)
{
// If quickscan is set, do so
if (quickScan)
{
List extracted = ArchiveTools.GetArchiveFileInfo(file, logger);
usedInternally = true;
foreach (Rom rom in extracted)
{
usedInternally &= RebuildToOutputIndividual(rom, file, outDir, tempSubDir, date, inverse, outputFormat,
romba, updateDat, true /* isZip */, headerToCheckAgainst, logger);
}
}
// Otherwise, attempt to extract the files to the temporary directory
else
{
bool encounteredErrors = ArchiveTools.ExtractArchive(file, tempSubDir, archiveScanLevel, logger);
// If the file was an archive and was extracted successfully, check it
if (!encounteredErrors)
{
usedInternally = true;
logger.Verbose(Path.GetFileName(file) + " treated like an archive");
List extracted = Directory.EnumerateFiles(tempSubDir, "*", SearchOption.AllDirectories).ToList();
foreach (string entry in extracted)
{
Rom rom = FileTools.GetFileInfo(entry, logger, noMD5: quickScan, noSHA1: quickScan);
usedInternally &= RebuildToOutputIndividual(rom, entry, outDir, tempSubDir, date, inverse, outputFormat,
romba, updateDat, false /* isZip */, headerToCheckAgainst, logger);
}
}
// Otherwise, just get the info on the file itself
else if (File.Exists(file))
{
Rom rom = FileTools.GetFileInfo(file, logger, noMD5: quickScan, noSHA1: quickScan);
usedExternally = RebuildToOutputIndividual(rom, file, outDir, tempSubDir, date, inverse, outputFormat,
romba, updateDat, false /* isZip */, headerToCheckAgainst, logger);
}
}
}
// If we are supposed to delete the file, do so
if (delete && (usedExternally || usedInternally))
{
try
{
logger.Verbose("Attempting to delete input file '" + file + "'");
File.Delete(file);
logger.Verbose("File '" + file + "' deleted");
}
catch (Exception ex)
{
logger.Error("An error occurred while trying to delete '" + file + "' " + ex.ToString());
}
}
// Now delete the temp directory
try
{
Directory.Delete(tempSubDir, true);
}
catch { }
}
///
/// Find duplicates and rebuild individual files to output
///
/// Information for the current file to rebuild from
/// Name of the file to process
/// Output directory to use to build to
/// Temporary directory for archive extraction
/// True if the date from the DAT should be used if available, false otherwise
/// True if the DAT should be used as a filter instead of a template, false otherwise
/// Output format that files should be written to
/// True if files should be output in Romba depot folders, false otherwise
/// True if the updated DAT should be output, false otherwise
/// True if the input file is an archive, false otherwise
/// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise
/// Logger object for file and console output
/// True if the file was able to be rebuilt, false otherwise
private bool RebuildToOutputIndividual(Rom rom, string file, string outDir, string tempDir, bool date,
bool inverse, OutputFormat outputFormat, bool romba, bool updateDat, bool isZip, string headerToCheckAgainst, Logger logger)
{
// Set the output value
bool rebuilt = false;
// Find if the file has duplicates in the DAT
bool hasDuplicates = rom.HasDuplicates(this, logger);
// If it has duplicates and we're not filtering, rebuild it
if (hasDuplicates && !inverse)
{
// Get the list of duplicates to rebuild to
List dupes = rom.GetDuplicates(this, logger, remove: updateDat);
// If we don't have any duplicates, continue
if (dupes.Count == 0)
{
return rebuilt;
}
// If we have an archive input, get the real name of the file to use
if (isZip)
{
// Otherwise, extract the file to the temp folder
file = ArchiveTools.ExtractItem(file, rom.Name, tempDir, logger);
}
// If we couldn't extract the file, then continue,
if (String.IsNullOrEmpty(file))
{
return rebuilt;
}
logger.User("Matches found for '" + Style.GetFileName(file) + "', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (Rom item in dupes)
{
switch (outputFormat)
{
case OutputFormat.Folder:
string outfile = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(item.Machine.Name), item.Name);
// Make sure the output folder is created
Directory.CreateDirectory(Path.GetDirectoryName(outfile));
// Now copy the file over
try
{
File.Copy(file, outfile);
if (date && !String.IsNullOrEmpty(item.Date))
{
File.SetCreationTime(outfile, DateTime.Parse(item.Date));
}
rebuilt &= true;
}
catch
{
rebuilt = false;
}
break;
case OutputFormat.TapeArchive:
rebuilt &= ArchiveTools.WriteTAR(file, outDir, item, logger, date: date);
break;
case OutputFormat.Torrent7Zip:
break;
case OutputFormat.TorrentGzip:
rebuilt &= ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger);
break;
case OutputFormat.TorrentLrzip:
break;
case OutputFormat.TorrentRar:
break;
case OutputFormat.TorrentXZ:
break;
case OutputFormat.TorrentZip:
rebuilt &= ArchiveTools.WriteTorrentZip(file, outDir, item, logger, date: date);
break;
}
}
}
// If we have no duplicates and we're filtering, rebuild it
else if (!hasDuplicates && inverse)
{
string machinename = null;
// If we have an archive input, get the real name of the file to use
if (isZip)
{
// Otherwise, extract the file to the temp folder
machinename = Style.GetFileNameWithoutExtension(file);
file = ArchiveTools.ExtractItem(file, rom.Name, tempDir, logger);
}
// If we couldn't extract the file, then continue,
if (String.IsNullOrEmpty(file))
{
return rebuilt;
}
// Get the item from the current file
Rom item = FileTools.GetFileInfo(file, logger);
item.Machine = new Machine()
{
Name = Style.GetFileNameWithoutExtension(item.Name),
Description = Style.GetFileNameWithoutExtension(item.Name),
};
// If we are coming from an archive, set the correct machine name
if (machinename != null)
{
item.Machine.Name = machinename;
item.Machine.Description = machinename;
}
logger.User("No matches found for '" + Style.GetFileName(file) + "', rebuilding accordingly from inverse flag...");
// Now rebuild to the output file
switch (outputFormat)
{
case OutputFormat.Folder:
string outfile = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(item.Machine.Name), item.Name);
// Make sure the output folder is created
Directory.CreateDirectory(Path.GetDirectoryName(outfile));
// Now copy the file over
try
{
File.Copy(file, outfile);
if (date && !String.IsNullOrEmpty(item.Date))
{
File.SetCreationTime(outfile, DateTime.Parse(item.Date));
}
rebuilt &= true;
}
catch
{
rebuilt &= false;
}
break;
case OutputFormat.TapeArchive:
rebuilt &= ArchiveTools.WriteTAR(file, outDir, item, logger, date: date);
break;
case OutputFormat.Torrent7Zip:
break;
case OutputFormat.TorrentGzip:
rebuilt &= ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger);
break;
case OutputFormat.TorrentLrzip:
break;
case OutputFormat.TorrentRar:
break;
case OutputFormat.TorrentXZ:
break;
case OutputFormat.TorrentZip:
rebuilt &= ArchiveTools.WriteTorrentZip(file, outDir, item, logger, date: date);
break;
}
}
// Now we want to take care of headers, if applicable
if (headerToCheckAgainst != null)
{
// Check to see if we have a matching header first
SkipperRule rule = Skipper.GetMatchingRule(file, Path.GetFileNameWithoutExtension(headerToCheckAgainst), logger);
// If there's a match, create the new file to write
if (rule.Tests != null && rule.Tests.Count != 0)
{
// If the file could be transformed correctly
if (rule.TransformFile(file, file + ".new", logger))
{
// Get the file informations that we will be using
Rom headerless = FileTools.GetFileInfo(file + ".new", logger);
// Find if the file has duplicates in the DAT
hasDuplicates = headerless.HasDuplicates(this, logger);
// If it has duplicates and we're not filtering, rebuild it
if (hasDuplicates && !inverse)
{
// Get the list of duplicates to rebuild to
List dupes = headerless.GetDuplicates(this, logger, remove: updateDat);
// If we don't have any duplicates, continue
if (dupes.Count == 0)
{
return rebuilt;
}
logger.User("Headerless matches found for '" + Style.GetFileName(file) + "', rebuilding accordingly...");
rebuilt = true;
// Now loop through the list and rebuild accordingly
foreach (Rom item in dupes)
{
// Create a headered item to use as well
rom.Machine = item.Machine;
rom.Name += "_" + rom.CRC;
switch (outputFormat)
{
case OutputFormat.Folder:
string outfile = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(item.Machine.Name), item.Name);
string headeredOutfile = Path.Combine(outDir, Style.RemovePathUnsafeCharacters(rom.Machine.Name), rom.Name);
// Make sure the output folder is created
Directory.CreateDirectory(Path.GetDirectoryName(outfile));
// If either copy succeeds, then we want to set rebuilt to true
bool eitherSuccess = false;
// Now copy the files over
try
{
File.Copy(file + ".new", outfile);
if (date && !String.IsNullOrEmpty(item.Date))
{
File.SetCreationTime(outfile, DateTime.Parse(item.Date));
}
eitherSuccess |= true;
}
catch { }
try
{
File.Copy(file, headeredOutfile);
if (date && !String.IsNullOrEmpty(rom.Date))
{
File.SetCreationTime(outfile, DateTime.Parse(rom.Date));
}
eitherSuccess |= true;
}
catch { }
// Now add the success of either rebuild
rebuilt &= eitherSuccess;
break;
case OutputFormat.TapeArchive:
rebuilt &= ArchiveTools.WriteTAR(file + ".new", outDir, item, logger, date: date);
rebuilt &= ArchiveTools.WriteTAR(file, outDir, rom, logger, date: date);
break;
case OutputFormat.Torrent7Zip:
break;
case OutputFormat.TorrentGzip:
rebuilt &= ArchiveTools.WriteTorrentGZ(file + ".new", outDir, romba, logger);
rebuilt &= ArchiveTools.WriteTorrentGZ(file, outDir, romba, logger);
break;
case OutputFormat.TorrentLrzip:
break;
case OutputFormat.TorrentRar:
break;
case OutputFormat.TorrentXZ:
break;
case OutputFormat.TorrentZip:
rebuilt &= ArchiveTools.WriteTorrentZip(file + ".new", outDir, item, logger, date: date);
rebuilt &= ArchiveTools.WriteTorrentZip(file, outDir, rom, logger, date: date);
break;
}
}
}
}
}
}
// And now clear the temp folder to get rid of any transient files if we unzipped
if (isZip)
{
try
{
Directory.Delete(tempDir, true);
}
catch { }
}
return rebuilt;
}
///
/// Process the DAT and verify the output directory
///
/// List of input directories to compare against
/// Temporary directory for archive extraction
/// True if only hashes should be checked, false for full file information
/// True to enable external scanning of archives, false otherwise
/// Populated string representing the name of the skipper to use, a blank string to use the first available checker, null otherwise
/// Logger object for file and console output
/// True if verification was a success, false otherwise
public bool VerifyDirectory(List inputs, string tempDir, bool hashOnly, bool quickScan, string headerToCheckAgainst, Logger logger)
{
// Check the temp directory exists
if (String.IsNullOrEmpty(tempDir))
{
tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
}
// Then create or clean the temp directory
if (!Directory.Exists(tempDir))
{
Directory.CreateDirectory(tempDir);
}
else
{
FileTools.CleanDirectory(tempDir);
}
bool success = true;
/*
We want the cross section of what's the folder and what's in the DAT. Right now, it just has what's in the DAT that's not in the folder
*/
// Then, loop through and check each of the inputs
logger.User("Processing files:\n");
foreach (string input in inputs)
{
PopulateFromDir(input, quickScan /* noMD5 */, quickScan /* noSHA1 */, true /* bare */, false /* archivesAsFiles */,
true /* enableGzip */, false /* addBlanks */, false /* addDate */, tempDir /* tempDir */, false /* copyFiles */,
headerToCheckAgainst, 4 /* maxDegreeOfParallelism */, logger);
}
// Setup the fixdat
DatFile matched = new DatFile(this);
matched.Reset();
matched.FileName = "fixDat_" + matched.FileName;
matched.Name = "fixDat_" + matched.Name;
matched.Description = "fixDat_" + matched.Description;
matched.DatFormat = DatFormat.Logiqx;
// If we are checking hashes only, essentially diff the inputs
bool found = false;
if (hashOnly)
{
// First we need to sort by hash to get duplicates
BucketBySHA1(true, logger, output: false);
// Then follow the same tactics as before
foreach (string key in Keys)
{
List roms = this[key];
foreach (DatItem rom in roms)
{
if (rom.SourceID == 99)
{
found = true;
if (rom.Type == ItemType.Disk || rom.Type == ItemType.Rom)
{
matched.Add(((Disk)rom).SHA1, rom);
}
}
}
}
}
// If we are checking full names, get only files found in directory
else
{
foreach (string key in Keys)
{
List roms = this[key];
List newroms = DatItem.Merge(roms, logger);
foreach (Rom rom in newroms)
{
if (rom.SourceID == 99)
{
found = true;
matched.Add(rom.Size + "-" + rom.CRC, rom);
}
}
}
}
// Now output the fixdat to the main folder
if (found)
{
matched.WriteToFile(String.Empty, logger, stats: true);
}
else
{
logger.User("No fixDat needed");
}
return success;
}
#endregion
}
}