using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; using SabreTools.Core; using SabreTools.DatItems; using SabreTools.FileTypes; using SabreTools.IO; // This file represents all methods related to creating a DatFile // from a set of files and directories namespace SabreTools.DatFiles { // TODO: See if any of the methods can be broken up a bit more neatly public abstract partial class DatFile { /// /// Create a new Dat from a directory /// /// Base folder to be used in creating the DAT /// TreatAsFiles representing CHD and Archive scanning /// Type of files that should be skipped /// True if blank items should be created for empty folders, false otherwise /// Hashes to include in the information public bool PopulateFromDir( string basePath, TreatAsFile asFiles = 0x00, SkipFileType skipFileType = SkipFileType.None, bool addBlanks = false, Hash hashes = Hash.Standard) { // Clean the temp directory path Globals.TempDir = DirectoryExtensions.Ensure(Globals.TempDir, temp: true); // Set the progress variables long totalSize = 0; long currentSize = 0; // Process the input if (Directory.Exists(basePath)) { logger.Verbose($"Folder found: {basePath}"); // Get a list of all files to process List files = Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories).ToList(); // Loop through and add the file sizes Parallel.ForEach(files, Globals.ParallelOptions, item => { Interlocked.Add(ref totalSize, new FileInfo(item).Length); }); // Process the files in the main folder or any subfolder logger.User(totalSize, currentSize); foreach (string item in files) { CheckFileForHashes(item, basePath, asFiles, skipFileType, addBlanks, hashes); currentSize += new FileInfo(item).Length; logger.User(totalSize, currentSize, item); } // Now find all folders that are empty, if we are supposed to if (addBlanks) ProcessDirectoryBlanks(basePath); } else if (File.Exists(basePath)) { logger.Verbose($"File found: {basePath}"); totalSize = new FileInfo(basePath).Length; logger.User(totalSize, currentSize); string parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath)); CheckFileForHashes(basePath, parentPath, asFiles, skipFileType, addBlanks, hashes); logger.User(totalSize, totalSize, basePath); } // Now that we're done, delete the temp folder (if it's not the default) logger.User("Cleaning temp folder"); if (Globals.TempDir != Path.GetTempPath()) { if (Directory.Exists(Globals.TempDir)) Directory.Delete(Globals.TempDir, true); } return true; } /// /// Check a given file for hashes, based on current settings /// /// Filename of the item to be checked /// Base folder to be used in creating the DAT /// TreatAsFiles representing CHD and Archive scanning /// Type of files that should be skipped /// True if blank items should be created for empty folders, false otherwise /// Hashes to include in the information private void CheckFileForHashes(string item, string basePath, TreatAsFile asFiles, SkipFileType skipFileType, bool addBlanks, Hash hashes) { // If we're in depot mode, process it separately if (CheckDepotFile(item)) return; // Initialize possible archive variables BaseArchive archive = BaseArchive.Create(item); // Process archives according to flags if (archive != null) { // Set the archive flags archive.AvailableHashes = hashes; // Skip if we're treating archives as files and skipping files if (asFiles.HasFlag(TreatAsFile.Archive) && skipFileType == SkipFileType.File) { return; } // Skip if we're skipping archives else if (skipFileType == SkipFileType.Archive) { return; } // Process as archive if we're not treating archives as files else if (!asFiles.HasFlag(TreatAsFile.Archive)) { var extracted = archive.GetChildren(); // If we have internal items to process, do so if (extracted != null) ProcessArchive(item, basePath, extracted); // Now find all folders that are empty, if we are supposed to if (addBlanks) ProcessArchiveBlanks(item, basePath, archive); } // Process as file if we're treating archives as files else { ProcessFile(item, basePath, hashes, asFiles); } } // Process non-archives according to flags else { // Skip if we're skipping files if (skipFileType == SkipFileType.File) return; // Process as file else ProcessFile(item, basePath, hashes, asFiles); } } /// /// Check an item as if it's supposed to be in a depot /// /// Filename of the item to be checked /// True if we checked a depot file, false otherwise private bool CheckDepotFile(string item) { // If we're not in Depot mode, return false if (Header.OutputDepot?.IsActive != true) return false; // Check the file as if it were in a depot GZipArchive gzarc = new GZipArchive(item); BaseFile baseFile = gzarc.GetTorrentGZFileInfo(); // If the rom is valid, add it if (baseFile != null && baseFile.Filename != null) { // Add the list if it doesn't exist already Rom rom = new Rom(baseFile); Items.Add(rom.GetKey(Field.DatItem_CRC), rom); logger.Verbose($"File added: {Path.GetFileNameWithoutExtension(item)}"); } else { logger.Verbose($"File not added: {Path.GetFileNameWithoutExtension(item)}"); return true; } return true; } /// /// Process a single file as an archive /// /// File to be added /// Path the represents the parent directory /// List of BaseFiles representing the internal files private void ProcessArchive(string item, string basePath, List extracted) { // Get the parent path for all items string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item); // First take care of the found items Parallel.ForEach(extracted, Globals.ParallelOptions, baseFile => { DatItem datItem = DatItem.Create(baseFile); ProcessFileHelper(item, datItem, basePath, parent); }); } /// /// Process blank folders in an archive /// /// File containing the blanks /// Path the represents the parent directory /// BaseArchive to get blanks from private void ProcessArchiveBlanks(string item, string basePath, BaseArchive archive) { List empties = new List(); // Get the parent path for all items string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath.Length) + Path.GetFileNameWithoutExtension(item); // Now get all blank folders from the archive if (archive != null) empties = archive.GetEmptyFolders(); // Add add all of the found empties to the DAT Parallel.ForEach(empties, Globals.ParallelOptions, empty => { Rom emptyRom = new Rom(Path.Combine(empty, "_"), item); ProcessFileHelper(item, emptyRom, basePath, parent); }); } /// /// Process blank folders in a directory /// /// Path the represents the parent directory private void ProcessDirectoryBlanks(string basePath) { // If we're in depot mode, we don't process blanks if (Header.OutputDepot?.IsActive == true) return; List empties = DirectoryExtensions.ListEmpty(basePath); Parallel.ForEach(empties, Globals.ParallelOptions, dir => { // Get the full path for the directory string fulldir = Path.GetFullPath(dir); // Set the temporary variables string gamename = string.Empty; string romname = string.Empty; // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom if (Header.Type == "SuperDAT") { gamename = fulldir.Remove(0, basePath.Length + 1); romname = "_"; } // Otherwise, we want just the top level folder as the game, and the file as everything else else { gamename = fulldir.Remove(0, basePath.Length + 1).Split(Path.DirectorySeparatorChar)[0]; romname = Path.Combine(fulldir.Remove(0, basePath.Length + 1 + gamename.Length), "_"); } // Sanitize the names gamename = gamename.Trim(Path.DirectorySeparatorChar); romname = romname.Trim(Path.DirectorySeparatorChar); logger.Verbose($"Adding blank empty folder: {gamename}"); Items["null"].Add(new Rom(romname, gamename)); }); } /// /// Process a single file as a file /// /// File to be added /// Path the represents the parent directory /// Hashes to include in the information /// TreatAsFiles representing CHD and Archive scanning private void ProcessFile(string item, string basePath, Hash hashes, TreatAsFile asFiles) { logger.Verbose($"'{Path.GetFileName(item)}' treated like a file"); BaseFile baseFile = BaseFile.GetInfo(item, header: Header.HeaderSkipper, hashes: hashes, asFiles: asFiles); DatItem datItem = DatItem.Create(baseFile); ProcessFileHelper(item, datItem, basePath, string.Empty); } /// /// Process a single file as a file (with found Rom data) /// /// File to be added /// Rom data to be used to write to file /// Path the represents the parent directory /// Parent game to be used private void ProcessFileHelper(string item, DatItem datItem, string basepath, string parent) { // If we didn't get an accepted parsed type somehow, cancel out List parsed = new List { ItemType.Disk, ItemType.Media, ItemType.Rom }; if (!parsed.Contains(datItem.ItemType)) return; try { // If the basepath doesn't end with a directory separator, add it if (!basepath.EndsWith(Path.DirectorySeparatorChar.ToString())) basepath += Path.DirectorySeparatorChar.ToString(); // Make sure we have the full item path item = Path.GetFullPath(item); // Process the item to sanitize names based on input SetDatItemInfo(datItem, item, parent, basepath); // Add the file information to the DAT string key = datItem.GetKey(Field.DatItem_CRC); Items.Add(key, datItem); logger.Verbose($"File added: {datItem.GetName() ?? string.Empty}"); } catch (IOException ex) { logger.Error(ex); return; } } /// /// Set proper Game and Rom names from user inputs /// /// DatItem representing the input file /// Item name to use /// Parent name to use /// Base path to use private void SetDatItemInfo(DatItem datItem, string item, string parent, string basepath) { // Get the data to be added as game and item names string machineName, itemName; // If the parent is blank, then we have a non-archive file if (string.IsNullOrWhiteSpace(parent)) { // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom if (Header.Type == "SuperDAT") { machineName = Path.GetDirectoryName(item.Remove(0, basepath.Length)); itemName = Path.GetFileName(item); } // Otherwise, we want just the top level folder as the game, and the file as everything else else { machineName = item.Remove(0, basepath.Length).Split(Path.DirectorySeparatorChar)[0]; itemName = item.Remove(0, (Path.Combine(basepath, machineName).Length)); } } // Otherwise, we assume that we have an archive else { // If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?) if (Header.Type == "SuperDAT") { machineName = parent; itemName = datItem.GetName(); } // Otherwise, we want the archive name as the game, and the file as everything else else { machineName = parent; itemName = datItem.GetName(); } } // Sanitize the names machineName = machineName.Trim(Path.DirectorySeparatorChar); itemName = itemName?.Trim(Path.DirectorySeparatorChar) ?? string.Empty; if (!string.IsNullOrWhiteSpace(machineName) && string.IsNullOrWhiteSpace(itemName)) { itemName = machineName; machineName = "Default"; } // Update machine information datItem.Machine.Name = machineName; datItem.Machine.Description = machineName; // If we have a Disk, then the ".chd" extension needs to be removed if (datItem.ItemType == ItemType.Disk && itemName.EndsWith(".chd")) { itemName = itemName.Substring(0, itemName.Length - 4); } // If we have a Media, then the extension needs to be removed else if (datItem.ItemType == ItemType.Media) { if (itemName.EndsWith(".dicf")) itemName = itemName.Substring(0, itemName.Length - 5); else if (itemName.EndsWith(".aaru")) itemName = itemName.Substring(0, itemName.Length - 5); else if (itemName.EndsWith(".aaruformat")) itemName = itemName.Substring(0, itemName.Length - 11); else if (itemName.EndsWith(".aaruf")) itemName = itemName.Substring(0, itemName.Length - 6); else if (itemName.EndsWith(".aif")) itemName = itemName.Substring(0, itemName.Length - 4); } // Set the item name back datItem.SetFields(new Dictionary { [Field.DatItem_Name] = itemName }); } } }