diff --git a/SabreTools/DATFromDirParallel.cs b/SabreTools/DATFromDirParallel.cs
new file mode 100644
index 00000000..c8511d1d
--- /dev/null
+++ b/SabreTools/DATFromDirParallel.cs
@@ -0,0 +1,494 @@
+using SabreTools.Helper;
+using SharpCompress.Common;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace SabreTools
+{
+ ///
+ /// Create a DAT file from a specified file, directory, or set thereof
+ ///
+ public class DATFromDirParallel
+ {
+ // Path-related variables
+ private string _basePath;
+ private string _tempDir;
+
+ // User specified inputs
+ private List _inputs;
+ private Dat _datdata;
+ private bool _noMD5;
+ private bool _noSHA1;
+ private bool _bare;
+ private bool _archivesAsFiles;
+ private bool _enableGzip;
+
+ // Other required variables
+ private Logger _logger;
+
+ // Public variables
+ public Dat DatData
+ {
+ get { return _datdata; }
+ }
+
+ ///
+ /// Create a new DATFromDir object
+ ///
+ /// A List of Strings representing the files and folders to be DATted
+ /// DatData object representing the requested output DAT
+ /// True if MD5 hashes should be skipped over, false otherwise
+ /// True if SHA-1 hashes should be skipped over, false otherwise
+ /// True if the date should be omitted from the DAT, false otherwise
+ /// True if archives should be treated as files, false otherwise
+ /// True if GZIP archives should be treated as files, false otherwise
+ /// Name of the directory to create a temp folder in (blank is current directory)
+ /// True if the file should not be written out, false otherwise (default)
+ /// Logger object for console and file output
+ public DATFromDirParallel(List inputs, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, string tempDir, Logger logger)
+ {
+ _inputs = inputs;
+ _datdata = datdata;
+ _noMD5 = noMD5;
+ _noSHA1 = noSHA1;
+ _bare = bare;
+ _archivesAsFiles = archivesAsFiles;
+ _enableGzip = enableGzip;
+ _tempDir = tempDir;
+ _logger = logger;
+ }
+
+ ///
+ /// Process the file, folder, or list of some combination into a DAT file
+ ///
+ /// True if the DAT could be created, false otherwise
+ /// Try to get the hashing multithreaded (either on a per-hash or per-file level)
+ public bool Start()
+ {
+ // Double check to see what it needs to be named
+ _basePath = (_inputs.Count > 0 ? (File.Exists(_inputs[0]) ? _inputs[0] : _inputs[0] + Path.DirectorySeparatorChar) : "");
+ _basePath = (_basePath != "" ? Path.GetFullPath(_basePath) : "");
+
+ // If the description is defined but not the name, set the name from the description
+ if (String.IsNullOrEmpty(_datdata.Name) && !String.IsNullOrEmpty(_datdata.Description))
+ {
+ _datdata.Name = _datdata.Description;
+ }
+
+ // If the name is defined but not the description, set the description from the name
+ else if (!String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description))
+ {
+ _datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")");
+ }
+
+ // If neither the name or description are defined, set them from the automatic values
+ else if (String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description))
+ {
+ if (_inputs.Count > 1)
+ {
+ _datdata.Name = Environment.CurrentDirectory.Split(Path.DirectorySeparatorChar).Last();
+ }
+ else
+ {
+ if (_basePath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ _basePath = _basePath.Substring(0, _basePath.Length - 1);
+ }
+ _datdata.Name = _basePath.Split(Path.DirectorySeparatorChar).Last();
+ }
+
+ // If the name is still somehow empty, populate it with defaults
+ _datdata.Name = (String.IsNullOrEmpty(_datdata.Name) ? "Default" : _datdata.Name);
+ _datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")");
+ }
+
+ // Loop over each of the found paths, if any
+ string lastparent = null;
+ foreach (string path in _inputs)
+ {
+ // Set local paths and vars
+ _basePath = (File.Exists(path) ? path : path + Path.DirectorySeparatorChar);
+ _basePath = Path.GetFullPath(_basePath);
+
+ // This is where the main loop would go
+ if (File.Exists(_basePath))
+ {
+ lastparent = ProcessPossibleArchive(_basePath, lastparent);
+ }
+ else if (Directory.Exists(_basePath))
+ {
+ _logger.Log("Folder found: " + _basePath);
+
+ // Process the files in the base folder first
+ Parallel.ForEach(Directory.EnumerateFiles(_basePath, "*", SearchOption.TopDirectoryOnly), item =>
+ {
+ lastparent = ProcessPossibleArchive(item, lastparent);
+ });
+
+ // Then process each of the subfolders themselves
+ string basePathBackup = _basePath;
+ foreach (string item in Directory.EnumerateDirectories(_basePath))
+ {
+ if (_datdata.Type != "SuperDAT")
+ {
+ _basePath = (File.Exists(item) ? item : item + Path.DirectorySeparatorChar);
+ _basePath = Path.GetFullPath(_basePath);
+ }
+
+ bool items = false;
+ Parallel.ForEach(Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories), subitem =>
+ {
+ items = true;
+ lastparent = ProcessPossibleArchive(subitem, lastparent);
+ });
+
+ // In romba mode we ignore empty folders completely
+ if (!_datdata.Romba)
+ {
+ // If there were no subitems, add a "blank" game to to the set (if not in Romba mode)
+ if (!items)
+ {
+ string actualroot = item.Remove(0, basePathBackup.Length);
+ Rom rom = new Rom
+ {
+ Name = "null",
+ Machine = new Machine
+ {
+ Name = (_datdata.Type == "SuperDAT" ?
+ (actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+ Path.DirectorySeparatorChar.ToString() :
+ "") + actualroot :
+ actualroot),
+ },
+ HashData = new Hash
+ {
+ Size = -1,
+ CRC = "null",
+ MD5 = "null",
+ SHA1 = "null",
+ },
+ };
+
+ string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+ if (_datdata.Files.ContainsKey(key))
+ {
+ _datdata.Files[key].Add(rom);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(rom);
+ _datdata.Files.Add(key, temp);
+ }
+ }
+
+ // Now scour subdirectories for empties and add those as well (if not in Romba mode)
+ foreach (string subdir in Directory.EnumerateDirectories(item, "*", SearchOption.AllDirectories))
+ {
+ if (Directory.EnumerateFiles(subdir, "*", SearchOption.AllDirectories).Count() == 0)
+ {
+ string actualroot = subdir.Remove(0, basePathBackup.Length);
+ Rom rom = new Rom
+ {
+ Name = "null",
+ Machine = new Machine
+ {
+ Name = (_datdata.Type == "SuperDAT" ?
+ (actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+ Path.DirectorySeparatorChar.ToString() :
+ "") + actualroot :
+ actualroot),
+ },
+ HashData = new Hash
+ {
+ Size = -1,
+ CRC = "null",
+ MD5 = "null",
+ SHA1 = "null",
+ },
+ };
+
+ string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+ if (_datdata.Files.ContainsKey(key))
+ {
+ _datdata.Files[key].Add(rom);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(rom);
+ _datdata.Files.Add(key, temp);
+ }
+ }
+ }
+ }
+ }
+ _basePath = basePathBackup;
+ }
+ // If this somehow skips past the original sensors
+ else
+ {
+ _logger.Error(path + " is not a valid input!");
+ }
+ }
+
+ // Now output any empties to the stream (if not in Romba mode)
+ if (!_datdata.Romba)
+ {
+ List keys = _datdata.Files.Keys.ToList();
+ foreach (string key in keys)
+ {
+ List roms = _datdata.Files[key];
+ for (int i = 0; i < roms.Count; i++)
+ {
+ Rom rom = roms[i];
+
+ // If we're in a mode that doesn't allow for actual empty folders, add the blank info
+ if (_datdata.OutputFormat != OutputFormat.SabreDat && _datdata.OutputFormat != OutputFormat.MissFile)
+ {
+ rom.Type = ItemType.Rom;
+ rom.Name = "-";
+ rom.HashData.Size = Constants.SizeZero;
+ rom.HashData.CRC = Constants.CRCZero;
+ rom.HashData.MD5 = Constants.MD5Zero;
+ rom.HashData.SHA1 = Constants.SHA1Zero;
+ }
+
+ string inkey = rom.HashData.Size + "-" + rom.HashData.CRC;
+ if (_datdata.Files.ContainsKey(inkey))
+ {
+ _datdata.Files[inkey].Add(rom);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(rom);
+ _datdata.Files.Add(inkey, temp);
+ }
+
+ lastparent = rom.Machine.Name;
+ }
+ }
+
+ // If we had roms but not blanks (and not in Romba mode), create an artifical rom for the purposes of outputting
+ if (lastparent != null && _datdata.Files.Count == 0)
+ {
+ _datdata.Files.Add("temp", new List());
+ }
+ }
+
+ return true;
+ }
+
+ ///
+ /// Check a given file for hashes, based on current settings
+ ///
+ /// Filename of the item to be checked
+ /// Name of the last parent rom to make sure that everything is grouped as well as possible
+ /// New parent to be used
+ private string ProcessPossibleArchive(string item, string lastparent)
+ {
+ // Define the temporary directory
+ string tempdir = (String.IsNullOrEmpty(_tempDir) ? Environment.CurrentDirectory : _tempDir);
+ tempdir += (tempdir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? "" : Path.DirectorySeparatorChar.ToString());
+ tempdir += "__temp__" + Path.DirectorySeparatorChar;
+
+ // Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes)
+ if (_datdata.Romba)
+ {
+ Rom rom = FileTools.GetTorrentGZFileInfo(item, _logger);
+
+ // If the rom is valid, write it out
+ if (rom.Name != null)
+ {
+ string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+ if (_datdata.Files.ContainsKey(key))
+ {
+ _datdata.Files[key].Add(rom);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(rom);
+ _datdata.Files.Add(key, temp);
+ }
+ }
+ else
+ {
+ return string.Empty;
+ }
+
+ _logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine);
+ return rom.Machine.Name;
+ }
+
+ // If both deep hash skip flags are set, do a quickscan
+ if (_noMD5 && _noSHA1)
+ {
+ ArchiveType? type = FileTools.GetCurrentArchiveType(item, _logger);
+
+ // If we have an archive, scan it
+ if (type != null)
+ {
+ List extracted = FileTools.GetArchiveFileInfo(item, _logger);
+ foreach (Rom rom in extracted)
+ {
+ lastparent = ProcessFileHelper(item, rom, _basePath,
+ Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) +
+ Path.GetFileNameWithoutExtension(item)
+ ), _datdata, lastparent);
+ }
+ }
+ // Otherwise, just get the info on the file itself
+ else if (!Directory.Exists(item) && File.Exists(item))
+ {
+ lastparent = ProcessFile(item, _basePath, "", _datdata, lastparent);
+ }
+ }
+ // Otherwise, attempt to extract the files to the temporary directory
+ else
+ {
+ bool encounteredErrors = FileTools.ExtractArchive(item,
+ tempdir,
+ (_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+ (!_archivesAsFiles && _enableGzip ? ArchiveScanLevel.Internal : ArchiveScanLevel.External),
+ (_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+ (_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal),
+ _logger);
+
+ // If the file was an archive and was extracted successfully, check it
+ if (!encounteredErrors)
+ {
+ _logger.Log(Path.GetFileName(item) + " treated like an archive");
+ Parallel.ForEach(Directory.EnumerateFiles(tempdir, "*", SearchOption.AllDirectories), entry =>
+ {
+ string tempbasepath = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar);
+ lastparent = ProcessFile(Path.GetFullPath(entry), Path.GetFullPath(tempdir),
+ (String.IsNullOrEmpty(tempbasepath)
+ ? ""
+ : (tempbasepath.Length < _basePath.Length
+ ? tempbasepath
+ : tempbasepath.Remove(0, _basePath.Length))) +
+ Path.GetFileNameWithoutExtension(item), _datdata, lastparent);
+ });
+
+ // Clear the temp directory
+ if (Directory.Exists(tempdir))
+ {
+ Output.CleanDirectory(tempdir);
+ }
+ }
+ // Otherwise, just get the info on the file itself
+ else if (!Directory.Exists(item) && File.Exists(item))
+ {
+ lastparent = ProcessFile(item, _basePath, "", _datdata, lastparent);
+ }
+ }
+
+ return lastparent;
+ }
+
+ ///
+ /// Process a single file as a file
+ ///
+ /// File to be added
+ /// Path the represents the parent directory
+ /// Parent game to be used
+ /// DatData object with output information
+ /// Last known parent game name
+ /// New last known parent game name
+ private string ProcessFile(string item, string basepath, string parent, Dat datdata, string lastparent)
+ {
+ _logger.Log(Path.GetFileName(item) + " treated like a file");
+ Rom rom = FileTools.GetSingleFileInfo(item, _noMD5, _noSHA1);
+
+ return ProcessFileHelper(item, rom, basepath, parent, datdata, lastparent);
+ }
+
+ ///
+ /// Process a single file as a file (with found Rom data)
+ ///
+ /// File to be added
+ /// Rom data to be used to write to file
+ /// Path the represents the parent directory
+ /// Parent game to be used
+ /// DatData object with output information
+ /// Last known parent game name
+ /// New last known parent game name
+ private string ProcessFileHelper(string item, Rom rom, string basepath, string parent, Dat datdata, string lastparent)
+ {
+ try
+ {
+ if (basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
+ {
+ basepath = basepath.Substring(0, basepath.Length - 1);
+ }
+
+ string actualroot = (item == basepath ? item.Split(Path.DirectorySeparatorChar).Last() : item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0]);
+ if (parent == "")
+ {
+ actualroot = (actualroot == "" && datdata.Type != "SuperDAT" ? basepath.Split(Path.DirectorySeparatorChar).Last() : actualroot);
+ }
+ string actualitem = (item == basepath ? item : item.Remove(0, basepath.Length + 1));
+
+ // If we're in SuperDAT mode, make sure the added item is by itself
+ if (datdata.Type == "SuperDAT")
+ {
+ actualroot += (actualroot != "" ? Path.DirectorySeparatorChar.ToString() : "") +
+ (parent != "" ? parent + Path.DirectorySeparatorChar : "") +
+ Path.GetDirectoryName(actualitem);
+ actualroot = actualroot.TrimEnd(Path.DirectorySeparatorChar);
+ actualitem = Path.GetFileName(actualitem);
+ }
+ else if (parent != "")
+ {
+ actualroot = parent.TrimEnd(Path.DirectorySeparatorChar);
+ }
+
+ // Drag and drop is funny
+ if (actualitem == Path.GetFullPath(actualitem))
+ {
+ actualitem = Path.GetFileName(actualitem);
+ }
+
+ _logger.Log("Actual item added: " + actualitem);
+
+ // Update rom information
+ rom.Machine = new Machine
+ {
+ Name = (datdata.Type == "SuperDAT" ?
+ (actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ?
+ Path.DirectorySeparatorChar.ToString() :
+ "") + actualroot :
+ actualroot),
+ };
+ rom.Machine.Name = rom.Machine.Name.Replace(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString(), Path.DirectorySeparatorChar.ToString());
+ rom.Name = actualitem;
+
+ string key = rom.HashData.Size + "-" + rom.HashData.CRC;
+ if (_datdata.Files.ContainsKey(key))
+ {
+ _datdata.Files[key].Add(rom);
+ }
+ else
+ {
+ List temp = new List();
+ temp.Add(rom);
+ _datdata.Files.Add(key, temp);
+ }
+ _logger.User("File added: " + actualitem + Environment.NewLine);
+
+ return rom.Machine.Name;
+ }
+ catch (IOException ex)
+ {
+ _logger.Error(ex.ToString());
+ return null;
+ }
+ }
+ }
+}
diff --git a/SabreTools/SabreTools.csproj b/SabreTools/SabreTools.csproj
index 2b623ac2..d9efb479 100644
--- a/SabreTools/SabreTools.csproj
+++ b/SabreTools/SabreTools.csproj
@@ -101,6 +101,7 @@
+