using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using SabreTools.Helper.Data;
using SabreTools.Helper.Tools;
#if MONO
using System.IO;
#else
using Alphaleonis.Win32.Filesystem;
using SearchOption = System.IO.SearchOption;
#endif
using NaturalSort;
namespace SabreTools.Helper.Dats
{
public partial class DatFile
{
#region Converting and Updating [MODULAR DONE]
///
/// Determine if input files should be merged, diffed, or processed invidually
///
/// Names of the input files and/or folders
/// Optional param for output directory
/// True if input files should be merged into a single file, false otherwise
/// Non-zero flag for diffing mode, zero otherwise
/// True if the cascade-diffed files should overwrite their inputs, false otherwise
/// True if the first cascaded diff file should be skipped on output, false otherwise
/// True if the date should not be appended to the default name, false otherwise [OBSOLETE]
/// True to clean the game names to WoD standard, false otherwise (default)
/// True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)
/// Filter object to be passed to the DatItem level
/// Type of the split that should be performed (split, merged, fully merged)
/// True if we are supposed to trim names to NTFS length, false otherwise
/// True if all games should be replaced by '!', false otherwise
/// String representing root directory to compare against for length calculation
/// Integer representing the maximum amount of parallelization to be used
/// Logging object for console and file output
public void DetermineUpdateType(List inputPaths, string outDir, bool merge, DiffMode diff, bool inplace, bool skip,
bool bare, bool clean, bool softlist, Filter filter, SplitType splitType, bool trim, bool single, string root,
int maxDegreeOfParallelism, Logger logger)
{
// If we're in merging or diffing mode, use the full list of inputs
if (merge || diff != 0)
{
// Make sure there are no folders in inputs
List newInputFileNames = FileTools.GetOnlyFilesFromInputs(inputPaths, maxDegreeOfParallelism, logger, appendparent: true);
// If we're in inverse cascade, reverse the list
if ((diff & DiffMode.ReverseCascade) != 0)
{
newInputFileNames.Reverse();
}
// Create a dictionary of all ROMs from the input DATs
List datHeaders = PopulateUserData(newInputFileNames, inplace, clean, softlist,
outDir, filter, splitType, trim, single, root, maxDegreeOfParallelism, logger);
// Modify the Dictionary if necessary and output the results
if (diff != 0 && diff < DiffMode.Cascade)
{
DiffNoCascade(diff, outDir, newInputFileNames, logger);
}
// If we're in cascade and diff, output only cascaded diffs
else if (diff != 0 && diff >= DiffMode.Cascade)
{
DiffCascade(outDir, inplace, newInputFileNames, datHeaders, skip, logger);
}
// Output all entries with user-defined merge
else
{
MergeNoDiff(outDir, newInputFileNames, datHeaders, logger);
}
}
// Otherwise, loop through all of the inputs individually
else
{
Update(inputPaths, outDir, clean, softlist, filter, splitType, trim, single, root, maxDegreeOfParallelism, logger);
}
return;
}
///
/// Populate the user DatData object from the input files
///
/// Type of the split that should be performed (split, merged, fully merged)
/// Filter object to be passed to the DatItem level
/// True if we are supposed to trim names to NTFS length, false otherwise
/// True if all games should be replaced by '!', false otherwise
/// String representing root directory to compare against for length calculation
/// Integer representing the maximum amount of parallelization to be used
/// Logging object for console and file output
/// List of DatData objects representing headers
private List PopulateUserData(List inputs, bool inplace, bool clean, bool softlist, string outDir,
Filter filter, SplitType splitType, bool trim, bool single, string root, int maxDegreeOfParallelism, Logger logger)
{
DatFile[] datHeaders = new DatFile[inputs.Count];
DateTime start = DateTime.Now;
logger.User("Processing individual DATs");
/// BEGIN
Parallel.For(0,
inputs.Count,
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
i =>
{
string input = inputs[i];
logger.User("Adding DAT: " + input.Split('¬')[0]);
datHeaders[i] = new DatFile
{
DatFormat = (DatFormat != 0 ? DatFormat : 0),
MergeRoms = MergeRoms,
};
datHeaders[i].Parse(input.Split('¬')[0], i, 0, filter, splitType, trim, single, root, logger, true, clean, softlist);
});
logger.User("Processing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
logger.User("Populating internal DAT");
for (int i = 0; i < inputs.Count; i++)
{
List keys = datHeaders[i].Keys.ToList();
foreach (string key in keys)
{
AddRange(key, datHeaders[i][key]);
datHeaders[i].Remove(key);
}
datHeaders[i].Delete();
}
/// END
logger.User("Processing and populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
return datHeaders.ToList();
}
///
/// Output non-cascading diffs
///
/// Non-zero flag for diffing mode, zero otherwise
/// Output directory to write the DATs to
/// List of inputs to write out from
/// Logging object for console and file output
public void DiffNoCascade(DiffMode diff, string outDir, List inputs, Logger logger)
{
DateTime start = DateTime.Now;
logger.User("Initializing all output DATs");
// Default vars for use
string post = "";
DatFile outerDiffData = new DatFile();
DatFile dupeData = new DatFile();
// Fill in any information not in the base DAT
if (String.IsNullOrEmpty(_fileName))
{
_fileName = "All DATs";
}
if (String.IsNullOrEmpty(_name))
{
_name = "All DATs";
}
if (String.IsNullOrEmpty(_description))
{
_description = "All DATs";
}
// Don't have External dupes
if ((diff & DiffMode.NoDupes) != 0)
{
post = " (No Duplicates)";
outerDiffData = new DatFile(this);
outerDiffData.FileName += post;
outerDiffData.Name += post;
outerDiffData.Description += post;
outerDiffData.Reset();
}
// Have External dupes
if ((diff & DiffMode.Dupes) != 0)
{
post = " (Duplicates)";
dupeData = new DatFile(this);
dupeData.FileName += post;
dupeData.Name += post;
dupeData.Description += post;
dupeData.Reset();
}
// Create a list of DatData objects representing individual output files
List outDats = new List();
// Loop through each of the inputs and get or create a new DatData object
if ((diff & DiffMode.Individuals) != 0)
{
DatFile[] outDatsArray = new DatFile[inputs.Count];
Parallel.For(0, inputs.Count, j =>
{
string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
DatFile diffData = new DatFile(this);
diffData.FileName += innerpost;
diffData.Name += innerpost;
diffData.Description += innerpost;
diffData.Reset();
outDatsArray[j] = diffData;
});
outDats = outDatsArray.ToList();
}
logger.User("Initializing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Now, loop through the dictionary and populate the correct DATs
start = DateTime.Now;
logger.User("Populating all output DATs");
List keys = Keys.ToList();
foreach (string key in keys)
{
List roms = DatItem.Merge(this[key], logger);
if (roms != null && roms.Count > 0)
{
foreach (DatItem rom in roms)
{
// No duplicates
if ((diff & DiffMode.NoDupes) != 0 || (diff & DiffMode.Individuals) != 0)
{
if ((rom.Dupe & DupeType.Internal) != 0)
{
// Individual DATs that are output
if ((diff & DiffMode.Individuals) != 0)
{
outDats[rom.SystemID].Add(key, rom);
}
// Merged no-duplicates DAT
if ((diff & DiffMode.NoDupes) != 0)
{
DatItem newrom = rom;
newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
outerDiffData.Add(key, newrom);
}
}
}
// Duplicates only
if ((diff & DiffMode.Dupes) != 0)
{
if ((rom.Dupe & DupeType.External) != 0)
{
DatItem newrom = rom;
newrom.Machine.Name += " (" + Path.GetFileNameWithoutExtension(inputs[newrom.SystemID].Split('¬')[0]) + ")";
dupeData.Add(key, newrom);
}
}
}
}
}
logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Finally, loop through and output each of the DATs
start = DateTime.Now;
logger.User("Outputting all created DATs");
// Output the difflist (a-b)+(b-a) diff
if ((diff & DiffMode.NoDupes) != 0)
{
outerDiffData.WriteToFile(outDir, logger);
}
// Output the (ab) diff
if ((diff & DiffMode.Dupes) != 0)
{
dupeData.WriteToFile(outDir, logger);
}
// Output the individual (a-b) DATs
if ((diff & DiffMode.Individuals) != 0)
{
for (int j = 0; j < inputs.Count; j++)
{
// If we have an output directory set, replace the path
string[] split = inputs[j].Split('¬');
string path = outDir + (split[0] == split[1]
? Path.GetFileName(split[0])
: (Path.GetDirectoryName(split[0]).Remove(0, split[1].Length)));
// Try to output the file
outDats[j].WriteToFile(path, logger);
}
}
logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
}
///
/// Output cascading diffs
///
/// Output directory to write the DATs to
/// True if cascaded diffs are outputted in-place, false otherwise
/// List of inputs to write out from
/// Dat headers used optionally
/// True if the first cascaded diff file should be skipped on output, false otherwise
/// Logging object for console and file output
public void DiffCascade(string outDir, bool inplace, List inputs, List datHeaders, bool skip, Logger logger)
{
string post = "";
// Create a list of DatData objects representing output files
List outDats = new List();
// Loop through each of the inputs and get or create a new DatData object
DateTime start = DateTime.Now;
logger.User("Initializing all output DATs");
DatFile[] outDatsArray = new DatFile[inputs.Count];
Parallel.For(0, inputs.Count, j =>
{
string innerpost = " (" + Path.GetFileNameWithoutExtension(inputs[j].Split('¬')[0]) + " Only)";
DatFile diffData;
// If we're in inplace mode, take the appropriate DatData object already stored
if (inplace || !String.IsNullOrEmpty(outDir))
{
diffData = datHeaders[j];
}
else
{
diffData = new DatFile(this);
diffData.FileName += post;
diffData.Name += post;
diffData.Description += post;
}
diffData.Reset();
outDatsArray[j] = diffData;
});
outDats = outDatsArray.ToList();
logger.User("Initializing complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Now, loop through the dictionary and populate the correct DATs
start = DateTime.Now;
logger.User("Populating all output DATs");
List keys = Keys.ToList();
foreach (string key in keys)
{
List roms = DatItem.Merge(this[key], logger);
if (roms != null && roms.Count > 0)
{
foreach (DatItem rom in roms)
{
// There's odd cases where there are items with System ID < 0. Skip them for now
if (rom.SystemID < 0)
{
logger.Warning("Item found with a <0 SystemID: " + rom.Name);
continue;
}
outDats[rom.SystemID].Add(key, rom);
}
}
}
logger.User("Populating complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
// Finally, loop through and output each of the DATs
start = DateTime.Now;
logger.User("Outputting all created DATs");
for (int j = (skip ? 1 : 0); j < inputs.Count; j++)
{
// If we have an output directory set, replace the path
string path = "";
if (inplace)
{
path = Path.GetDirectoryName(inputs[j].Split('¬')[0]);
}
else if (!String.IsNullOrEmpty(outDir))
{
string[] split = inputs[j].Split('¬');
path = outDir + (split[0] == split[1]
? Path.GetFileName(split[0])
: (Path.GetDirectoryName(split[0]).Remove(0, split[1].Length))); ;
}
// Try to output the file
outDats[j].WriteToFile(path, logger);
}
logger.User("Outputting complete in " + DateTime.Now.Subtract(start).ToString(@"hh\:mm\:ss\.fffff"));
}
///
/// Output user defined merge
///
/// Output directory to write the DATs to
/// List of inputs to write out from
/// Dat headers used optionally
/// Logging object for console and file output
public void MergeNoDiff(string outDir, List inputs, List datHeaders, Logger logger)
{
// If we're in SuperDAT mode, prefix all games with their respective DATs
if (Type == "SuperDAT")
{
List keys = Keys.ToList();
foreach (string key in keys)
{
List newroms = new List();
foreach (DatItem rom in this[key])
{
DatItem newrom = rom;
string filename = inputs[newrom.SystemID].Split('¬')[0];
string rootpath = inputs[newrom.SystemID].Split('¬')[1];
rootpath += (rootpath == "" ? "" : Path.DirectorySeparatorChar.ToString());
filename = filename.Remove(0, rootpath.Length);
newrom.Machine.Name = Path.GetDirectoryName(filename) + Path.DirectorySeparatorChar
+ Path.GetFileNameWithoutExtension(filename) + Path.DirectorySeparatorChar
+ newrom.Machine.Name;
newroms.Add(newrom);
}
this[key] = newroms;
}
}
// Try to output the file
WriteToFile(outDir, logger);
}
///
/// Convert, update, and filter a DAT file or set of files using a base
///
/// Names of the input files and/or folders
/// Optional param for output directory
/// True if input files should be merged into a single file, false otherwise
/// Non-zero flag for diffing mode, zero otherwise
/// True if the cascade-diffed files should overwrite their inputs, false otherwise
/// True if the first cascaded diff file should be skipped on output, false otherwise
/// True if the date should not be appended to the default name, false otherwise [OBSOLETE]
/// True to clean the game names to WoD standard, false otherwise (default)
/// True to allow SL DATs to have game names used instead of descriptions, false otherwise (default)
/// Filter object to be passed to the DatItem level
/// Type of the split that should be performed (split, merged, fully merged)
/// True if we are supposed to trim names to NTFS length, false otherwise
/// True if all games should be replaced by '!', false otherwise
/// String representing root directory to compare against for length calculation
/// Integer representing the maximum amount of parallelization to be used
/// Logging object for console and file output
public void Update(List inputFileNames, string outDir, bool clean, bool softlist, Filter filter,
SplitType splitType, bool trim, bool single, string root, int maxDegreeOfParallelism, Logger logger)
{
Parallel.ForEach(inputFileNames,
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
inputFileName =>
{
// Clean the input string
if (inputFileName != "")
{
inputFileName = Path.GetFullPath(inputFileName);
}
if (File.Exists(inputFileName))
{
DatFile innerDatdata = new DatFile(this);
logger.User("Processing \"" + Path.GetFileName(inputFileName) + "\"");
innerDatdata.Parse(inputFileName, 0, 0, filter, splitType, trim, single,
root, logger, true, clean, softlist,
keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
// Try to output the file
innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(inputFileName) : outDir), logger, overwrite: (outDir != ""));
}
else if (Directory.Exists(inputFileName))
{
inputFileName = Path.GetFullPath(inputFileName) + Path.DirectorySeparatorChar;
Parallel.ForEach(Directory.EnumerateFiles(inputFileName, "*", SearchOption.AllDirectories),
new ParallelOptions { MaxDegreeOfParallelism = maxDegreeOfParallelism },
file =>
{
logger.User("Processing \"" + Path.GetFullPath(file).Remove(0, inputFileName.Length) + "\"");
DatFile innerDatdata = new DatFile(this);
innerDatdata.Parse(file, 0, 0, filter, splitType,
trim, single, root, logger, true, clean, softlist,
keepext: ((innerDatdata.DatFormat & DatFormat.TSV) != 0 || (innerDatdata.DatFormat & DatFormat.CSV) != 0));
// Try to output the file
innerDatdata.WriteToFile((outDir == "" ? Path.GetDirectoryName(file) : outDir + Path.GetDirectoryName(file).Remove(0, inputFileName.Length - 1)), logger, overwrite: (outDir != ""));
});
}
else
{
logger.Error("I'm sorry but " + inputFileName + " doesn't exist!");
return;
}
});
}
#endregion
}
}