using SabreTools.Helper; using SharpCompress.Common; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; namespace SabreTools { /// /// Create a DAT file from a specified file, directory, or set thereof /// public class DATFromDir { // Path-related variables private string _basePath; private string _tempDir; // User specified inputs private List _inputs; private Dat _datdata; private bool _noMD5; private bool _noSHA1; private bool _bare; private bool _archivesAsFiles; private bool _enableGzip; private bool _addBlanks; private bool _addDate; private bool _nowrite; // Other required variables private Logger _logger; // Public variables public Dat DatData { get { return _datdata; } } /// /// Create a new DATFromDir object /// /// A List of Strings representing the files and folders to be DATted /// DatData object representing the requested output DAT /// True if MD5 hashes should be skipped over, false otherwise /// True if SHA-1 hashes should be skipped over, false otherwise /// True if the date should be omitted from the DAT, false otherwise /// True if archives should be treated as files, false otherwise /// True if GZIP archives should be treated as files, false otherwise /// True if blank items should be created for empty folders, false otherwise /// True if dates should be archived for all files, false otherwise /// Name of the directory to create a temp folder in (blank is current directory) /// True if the file should not be written out, false otherwise (default) /// Logger object for console and file output public DATFromDir(List inputs, Dat datdata, bool noMD5, bool noSHA1, bool bare, bool archivesAsFiles, bool enableGzip, bool addBlanks, bool addDate, string tempDir, Logger logger, bool nowrite = false) { _inputs = inputs; _datdata = datdata; _noMD5 = noMD5; _noSHA1 = noSHA1; _bare = bare; _archivesAsFiles = archivesAsFiles; _enableGzip = enableGzip; _tempDir = tempDir; _logger = logger; _addBlanks = addBlanks; _addDate = addDate; _nowrite = nowrite; } /// /// Process the file, folder, or list of some combination into a DAT file /// /// True if the DAT could be created, false otherwise /// Try to get the hashing multithreaded (either on a per-hash or per-file level) public bool Start() { // Double check to see what it needs to be named _basePath = (_inputs.Count > 0 ? (File.Exists(_inputs[0]) ? _inputs[0] : _inputs[0] + Path.DirectorySeparatorChar) : ""); _basePath = (_basePath != "" ? Path.GetFullPath(_basePath) : ""); // If the description is defined but not the name, set the name from the description if (String.IsNullOrEmpty(_datdata.Name) && !String.IsNullOrEmpty(_datdata.Description)) { _datdata.Name = _datdata.Description; } // If the name is defined but not the description, set the description from the name else if (!String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description)) { _datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")"); } // If neither the name or description are defined, set them from the automatic values else if (String.IsNullOrEmpty(_datdata.Name) && String.IsNullOrEmpty(_datdata.Description)) { if (_inputs.Count > 1) { _datdata.Name = Environment.CurrentDirectory.Split(Path.DirectorySeparatorChar).Last(); } else { if (_basePath.EndsWith(Path.DirectorySeparatorChar.ToString())) { _basePath = _basePath.Substring(0, _basePath.Length - 1); } _datdata.Name = _basePath.Split(Path.DirectorySeparatorChar).Last(); } // If the name is still somehow empty, populate it with defaults _datdata.Name = (String.IsNullOrEmpty(_datdata.Name) ? "Default" : _datdata.Name); _datdata.Description = _datdata.Name + (_bare ? "" : " (" + _datdata.Date + ")"); } StreamWriter sw; if (_nowrite) { sw = new StreamWriter(new MemoryStream()); } else { // Create and open the output file for writing FileStream fs = File.Create(Style.CreateOutfileNames(Environment.CurrentDirectory, _datdata)[_datdata.OutputFormat]); sw = new StreamWriter(fs, Encoding.UTF8); sw.AutoFlush = true; } // Write out the initial file header DatTools.WriteHeader(sw, _datdata.OutputFormat, _datdata, _logger); // Loop over each of the found paths, if any string lastparent = null; foreach (string path in _inputs) { // Set local paths and vars _basePath = (File.Exists(path) ? path : path + Path.DirectorySeparatorChar); _basePath = Path.GetFullPath(_basePath); // This is where the main loop would go if (File.Exists(_basePath)) { lastparent = ProcessPossibleArchive(_basePath, sw, lastparent); } else if (Directory.Exists(_basePath)) { _logger.Log("Folder found: " + _basePath); // Process the files in the base folder first foreach (string item in Directory.EnumerateFiles(_basePath, "*", SearchOption.TopDirectoryOnly)) { lastparent = ProcessPossibleArchive(item, sw, lastparent); } // Then process each of the subfolders themselves string basePathBackup = _basePath; foreach (string item in Directory.EnumerateDirectories(_basePath)) { if (_datdata.Type != "SuperDAT") { _basePath = (File.Exists(item) ? item : item + Path.DirectorySeparatorChar); _basePath = Path.GetFullPath(_basePath); } bool items = false; foreach (string subitem in Directory.EnumerateFiles(item, "*", SearchOption.AllDirectories)) { items = true; lastparent = ProcessPossibleArchive(subitem, sw, lastparent); } // Now find all folders that are empty, if we are supposed to if (!_datdata.Romba && _addBlanks) { // If there were no subitems, add a "blank" game to to the set (if not in Romba mode) if (!items) { string actualroot = item.Remove(0, basePathBackup.Length); Rom rom = new Rom { Name = "null", Machine = new Machine { Name = (_datdata.Type == "SuperDAT" ? (actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ? Path.DirectorySeparatorChar.ToString() : "") + actualroot : actualroot), }, HashData = new Hash { Size = -1, CRC = "null", MD5 = "null", SHA1 = "null", }, }; string key = rom.HashData.Size + "-" + rom.HashData.CRC; if (_datdata.Files.ContainsKey(key)) { _datdata.Files[key].Add(rom); } else { List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } } // Now scour subdirectories for empties and add those as well (if not in Romba mode) foreach (string subdir in Directory.EnumerateDirectories(item, "*", SearchOption.AllDirectories)) { if (Directory.EnumerateFiles(subdir, "*", SearchOption.AllDirectories).Count() == 0) { string actualroot = subdir.Remove(0, basePathBackup.Length); Rom rom = new Rom { Name = "null", Machine = new Machine { Name = (_datdata.Type == "SuperDAT" ? (actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ? Path.DirectorySeparatorChar.ToString() : "") + actualroot : actualroot), }, HashData = new Hash { Size = -1, CRC = "null", MD5 = "null", SHA1 = "null", }, }; string key = rom.HashData.Size + "-" + rom.HashData.CRC; if (_datdata.Files.ContainsKey(key)) { _datdata.Files[key].Add(rom); } else { List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } } } } } _basePath = basePathBackup; } // If this somehow skips past the original sensors else { _logger.Error(path + " is not a valid input!"); } } // Now find all folders that are empty, if we are supposed to if (!_datdata.Romba && _addBlanks) { List keys = _datdata.Files.Keys.ToList(); foreach (string key in keys) { List roms = _datdata.Files[key]; for (int i = 0; i < roms.Count; i++) { Rom rom = roms[i]; // If we're in a mode that doesn't allow for actual empty folders, add the blank info if ((_datdata.OutputFormat & OutputFormat.SabreDat) == 0 && (_datdata.OutputFormat & OutputFormat.MissFile) == 0) { rom.Type = ItemType.Rom; rom.Name = "-"; rom.HashData.Size = Constants.SizeZero; rom.HashData.CRC = Constants.CRCZero; rom.HashData.MD5 = Constants.MD5Zero; rom.HashData.SHA1 = Constants.SHA1Zero; } if (_nowrite) { string inkey = rom.HashData.Size + "-" + rom.HashData.CRC; if (_datdata.Files.ContainsKey(inkey)) { _datdata.Files[inkey].Add(rom); } else { List temp = new List(); temp.Add(rom); _datdata.Files.Add(inkey, temp); } } else { // If we have a different game and we're not at the start of the list, output the end of last item int last = 0; if (lastparent != null && lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant()) { DatTools.WriteEndGame(sw, _datdata.OutputFormat, rom, new List(), new List(), lastparent, 0, out last, _logger); } // If we have a new game, output the beginning of the new item if (lastparent == null || lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant()) { DatTools.WriteStartGame(sw, _datdata.OutputFormat, rom, new List(), lastparent, 0, last, _logger); } // Write out the rom data if ((_datdata.OutputFormat & OutputFormat.SabreDat) == 0 && (_datdata.OutputFormat & OutputFormat.MissFile) == 0) { DatTools.WriteRomData(sw, _datdata.OutputFormat, rom, lastparent, _datdata, 0, _logger); } } lastparent = rom.Machine.Name; } } // If we had roms but not blanks (and not in Romba mode), create an artifical rom for the purposes of outputting if (lastparent != null && _datdata.Files.Count == 0) { _datdata.Files.Add("temp", new List()); } } // Now write the final piece and close the output stream DatTools.WriteFooter(sw, _datdata.OutputFormat, _datdata, 0, _logger); sw.Close(); return true; } /// /// Check a given file for hashes, based on current settings /// /// Filename of the item to be checked /// StreamWriter representing the output file /// Name of the last parent rom to make sure that everything is grouped as well as possible /// New parent to be used private string ProcessPossibleArchive(string item, StreamWriter sw, string lastparent) { // Define the temporary directory string tempdir = (String.IsNullOrEmpty(_tempDir) ? Environment.CurrentDirectory : _tempDir); tempdir += (tempdir.EndsWith(Path.DirectorySeparatorChar.ToString()) ? "" : Path.DirectorySeparatorChar.ToString()); tempdir += "__temp__" + Path.DirectorySeparatorChar; // Special case for if we are in Romba mode (all names are supposed to be SHA-1 hashes) if (_datdata.Romba) { Rom rom = FileTools.GetTorrentGZFileInfo(item, _logger); // If the rom is valid, write it out if (rom.Name != null) { int last = 0; if (_nowrite) { string key = rom.HashData.Size + "-" + rom.HashData.CRC; if (_datdata.Files.ContainsKey(key)) { _datdata.Files[key].Add(rom); } else { List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } } else { DatTools.WriteStartGame(sw, _datdata.OutputFormat, rom, new List(), "", 0, 0, _logger); DatTools.WriteRomData(sw, _datdata.OutputFormat, rom, "", _datdata, 0, _logger); DatTools.WriteEndGame(sw, _datdata.OutputFormat, rom, new List(), new List(), "", 0, out last, _logger); } } else { return string.Empty; } _logger.User("File added: " + Path.GetFileNameWithoutExtension(item) + Environment.NewLine); return rom.Machine.Name; } // If both deep hash skip flags are set, do a quickscan if (_noMD5 && _noSHA1) { ArchiveType? type = FileTools.GetCurrentArchiveType(item, _logger); // If we have an archive, scan it if (type != null) { List extracted = FileTools.GetArchiveFileInfo(item, _logger); foreach (Rom rom in extracted) { lastparent = ProcessFileHelper(item, rom, sw, _basePath, Path.Combine((Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, _basePath.Length) + Path.GetFileNameWithoutExtension(item) ), _datdata, lastparent); } } // Otherwise, just get the info on the file itself else if (!Directory.Exists(item) && File.Exists(item)) { lastparent = ProcessFile(item, sw, _basePath, "", _datdata, lastparent); } } // Otherwise, attempt to extract the files to the temporary directory else { bool encounteredErrors = FileTools.ExtractArchive(item, tempdir, (_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal), (!_archivesAsFiles && _enableGzip ? ArchiveScanLevel.Internal : ArchiveScanLevel.External), (_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal), (_archivesAsFiles ? ArchiveScanLevel.External : ArchiveScanLevel.Internal), _logger); // If the file was an archive and was extracted successfully, check it if (!encounteredErrors) { _logger.Log(Path.GetFileName(item) + " treated like an archive"); foreach (string entry in Directory.EnumerateFiles(tempdir, "*", SearchOption.AllDirectories)) { string tempbasepath = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar); lastparent = ProcessFile(Path.GetFullPath(entry), sw, Path.GetFullPath(tempdir), (String.IsNullOrEmpty(tempbasepath) ? "" : (tempbasepath.Length < _basePath.Length ? tempbasepath : tempbasepath.Remove(0, _basePath.Length))) + Path.GetFileNameWithoutExtension(item), _datdata, lastparent); } // Clear the temp directory if (Directory.Exists(tempdir)) { FileTools.CleanDirectory(tempdir); } } // Otherwise, just get the info on the file itself else if (!Directory.Exists(item) && File.Exists(item)) { lastparent = ProcessFile(item, sw, _basePath, "", _datdata, lastparent); } } return lastparent; } /// /// Process a single file as a file /// /// File to be added /// StreamWriter representing the output file /// Path the represents the parent directory /// Parent game to be used /// DatData object with output information /// Last known parent game name /// New last known parent game name private string ProcessFile(string item, StreamWriter sw, string basepath, string parent, Dat datdata, string lastparent) { _logger.Log(Path.GetFileName(item) + " treated like a file"); Rom rom = FileTools.GetSingleFileInfo(item, noMD5: _noMD5, noSHA1: _noSHA1, date: _addDate); return ProcessFileHelper(item, rom, sw, basepath, parent, datdata, lastparent); } /// /// Process a single file as a file (with found Rom data) /// /// File to be added /// Rom data to be used to write to file /// StreamWriter representing the output file /// Path the represents the parent directory /// Parent game to be used /// DatData object with output information /// Last known parent game name /// New last known parent game name private string ProcessFileHelper(string item, Rom rom, StreamWriter sw, string basepath, string parent, Dat datdata, string lastparent) { try { if (basepath.EndsWith(Path.DirectorySeparatorChar.ToString())) { basepath = basepath.Substring(0, basepath.Length - 1); } string actualroot = (item == basepath ? item.Split(Path.DirectorySeparatorChar).Last() : item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0]); if (parent == "") { actualroot = (actualroot == "" && datdata.Type != "SuperDAT" ? basepath.Split(Path.DirectorySeparatorChar).Last() : actualroot); } string actualitem = (item == basepath ? item : item.Remove(0, basepath.Length + 1)); // If we're in SuperDAT mode, make sure the added item is by itself if (datdata.Type == "SuperDAT") { actualroot += (actualroot != "" ? Path.DirectorySeparatorChar.ToString() : "") + (parent != "" ? parent + Path.DirectorySeparatorChar : "") + Path.GetDirectoryName(actualitem); actualroot = actualroot.TrimEnd(Path.DirectorySeparatorChar); actualitem = Path.GetFileName(actualitem); } else if (parent != "") { actualroot = parent.TrimEnd(Path.DirectorySeparatorChar); } // Drag and drop is funny if (actualitem == Path.GetFullPath(actualitem)) { actualitem = Path.GetFileName(actualitem); } _logger.Log("Actual item added: " + actualitem); // Update rom information rom.Machine = new Machine { Name = (datdata.Type == "SuperDAT" ? (actualroot != "" && !actualroot.StartsWith(Path.DirectorySeparatorChar.ToString()) ? Path.DirectorySeparatorChar.ToString() : "") + actualroot : actualroot), }; rom.Machine.Name = rom.Machine.Name.Replace(Path.DirectorySeparatorChar.ToString() + Path.DirectorySeparatorChar.ToString(), Path.DirectorySeparatorChar.ToString()); rom.Name = actualitem; if (_nowrite) { string key = rom.HashData.Size + "-" + rom.HashData.CRC; if (_datdata.Files.ContainsKey(key)) { _datdata.Files[key].Add(rom); } else { List temp = new List(); temp.Add(rom); _datdata.Files.Add(key, temp); } } else { // If we have a different game and we're not at the start of the list, output the end of last item int last = 0; if (lastparent != null && lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant()) { DatTools.WriteEndGame(sw, datdata.OutputFormat, rom, new List(), new List(), lastparent, 0, out last, _logger); } // If we have a new game, output the beginning of the new item if (lastparent == null || lastparent.ToLowerInvariant() != rom.Machine.Name.ToLowerInvariant()) { DatTools.WriteStartGame(sw, datdata.OutputFormat, rom, new List(), lastparent, 0, last, _logger); } // Write out the rom data DatTools.WriteRomData(sw, datdata.OutputFormat, rom, lastparent, datdata, 0, _logger); } _logger.User("File added: " + actualitem + Environment.NewLine); return rom.Machine.Name; } catch (IOException ex) { _logger.Error(ex.ToString()); return null; } } } }