using System.Collections.Generic; using System.IO; using System.Threading; #if NET40_OR_GREATER || NETCOREAPP using System.Threading.Tasks; #endif using SabreTools.Core.Tools; using SabreTools.DatFiles; using SabreTools.DatItems; using SabreTools.DatItems.Formats; using SabreTools.FileTypes; using SabreTools.FileTypes.Archives; using SabreTools.Hashing; using SabreTools.IO.Extensions; using SabreTools.IO.Logging; namespace SabreTools.DatTools { /// /// This file represents all methods related to populating a DatFile /// from a set of files and directories /// public class DatFromDir { #region Fields /// /// Hashes to include in the information /// private readonly HashType[] _hashes; /// /// Type of files that should be skipped /// private readonly SkipFileType _skipFileType; /// /// Indicates if blank items should be created for empty folders /// private readonly bool _addBlanks; #endregion #region Logging /// /// Logging object /// private static readonly Logger _staticLogger = new(); #endregion #region Constructors public DatFromDir(HashType[]? hashes, SkipFileType skipFileType, bool addBlanks) { _hashes = hashes ?? [HashType.CRC32, HashType.MD5, HashType.SHA1]; _skipFileType = skipFileType; _addBlanks = addBlanks; } #endregion /// /// Create a new Dat from a directory /// /// Current DatFile object to add to /// Base folder to be used in creating the DAT /// TreatAsFile representing CHD and Archive scanning public bool PopulateFromDir(DatFile datFile, string basePath, TreatAsFile asFile = 0x00) { // Set the progress variables long totalSize = 0; long currentSize = 0; InternalStopwatch watch = new($"Populating DAT from {basePath}"); // Process the input if (Directory.Exists(basePath)) { _staticLogger.Verbose($"Folder found: {basePath}"); // Get a list of all files to process #if NET20 || NET35 List files = [.. Directory.GetFiles(basePath, "*")]; #else List files = [.. Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories)]; #endif // Loop through and add the file sizes #if NET452_OR_GREATER || NETCOREAPP Parallel.ForEach(files, Core.Globals.ParallelOptions, item => #elif NET40_OR_GREATER Parallel.ForEach(files, item => #else foreach (var item in files) #endif { Interlocked.Add(ref totalSize, new FileInfo(item).Length); #if NET40_OR_GREATER || NETCOREAPP }); #else } #endif // Process the files in the main folder or any subfolder _staticLogger.User(totalSize, currentSize); foreach (string item in files) { currentSize += new FileInfo(item).Length; CheckFileForHashes(datFile, item, basePath, asFile); _staticLogger.User(totalSize, currentSize, item); } // Now find all folders that are empty, if we are supposed to if (_addBlanks) ProcessDirectoryBlanks(datFile, basePath); } else if (System.IO.File.Exists(basePath)) { _staticLogger.Verbose($"File found: {basePath}"); totalSize = new FileInfo(basePath).Length; _staticLogger.User(totalSize, currentSize); string? parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath)); CheckFileForHashes(datFile, basePath, parentPath, asFile); _staticLogger.User(totalSize, totalSize, basePath); } watch.Stop(); return true; } /// /// Check a given file for hashes, based on current settings /// /// Current DatFile object to add to /// Filename of the item to be checked /// Base folder to be used in creating the DAT /// TreatAsFile representing CHD and Archive scanning private void CheckFileForHashes(DatFile datFile, string item, string? basePath, TreatAsFile asFile) { // If we're in depot mode, process it separately if (CheckDepotFile(datFile, item)) return; // Initialize possible archive variables BaseArchive? archive = FileTypeTool.CreateArchiveType(item); // Process archives according to flags if (archive != null) { // Set the archive flags archive.SetHashTypes(_hashes); // Skip if we're treating archives as files and skipping files #if NET20 || NET35 if ((asFile & TreatAsFile.Archive) != 0 && _skipFileType == SkipFileType.File) #else if (asFile.HasFlag(TreatAsFile.Archive) && _skipFileType == SkipFileType.File) #endif { return; } // Skip if we're skipping archives else if (_skipFileType == SkipFileType.Archive) { return; } // Process as archive if we're not treating archives as files #if NET20 || NET35 else if ((asFile & TreatAsFile.Archive) == 0) #else else if (!asFile.HasFlag(TreatAsFile.Archive)) #endif { var extracted = archive.GetChildren(); // If we have internal items to process, do so if (extracted != null) ProcessArchive(datFile, item, basePath, extracted); // Now find all folders that are empty, if we are supposed to if (_addBlanks) ProcessArchiveBlanks(datFile, item, basePath, archive); } // Process as file if we're treating archives as files else { ProcessFile(datFile, item, basePath, asFile); } } // Process non-archives according to flags else { // Skip if we're skipping files if (_skipFileType == SkipFileType.File) return; // Process as file else ProcessFile(datFile, item, basePath, asFile); } } /// /// Check an item as if it's supposed to be in a depot /// /// Current DatFile object to add to /// Filename of the item to be checked /// True if we checked a depot file, false otherwise private static bool CheckDepotFile(DatFile datFile, string item) { // If we're not in Depot mode, return false if (datFile.Header.GetFieldValue(DatHeader.OutputDepotKey)?.IsActive != true) return false; // Check the file as if it were in a depot GZipArchive gzarc = new(item); BaseFile? baseFile = gzarc.GetTorrentGZFileInfo(); // If the rom is valid, add it if (baseFile != null && baseFile.Filename != null) { // Add the list if it doesn't exist already Rom rom = baseFile.ConvertToRom(); datFile.AddItem(rom, statsOnly: false); _staticLogger.Verbose($"File added: {Path.GetFileNameWithoutExtension(item)}"); } else { _staticLogger.Verbose($"File not added: {Path.GetFileNameWithoutExtension(item)}"); return true; } return true; } /// /// Process a single file as an archive /// /// Current DatFile object to add to /// File to be added /// Path the represents the parent directory /// List of BaseFiles representing the internal files private static void ProcessArchive(DatFile datFile, string item, string? basePath, List extracted) { // Get the parent path for all items string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath?.Length ?? 0) + Path.GetFileNameWithoutExtension(item); // First take care of the found items #if NET452_OR_GREATER || NETCOREAPP Parallel.ForEach(extracted, Core.Globals.ParallelOptions, baseFile => #elif NET40_OR_GREATER Parallel.ForEach(extracted, baseFile => #else foreach (var baseFile in extracted) #endif { DatItem? datItem = DatItemTool.CreateDatItem(baseFile); if (datItem == null) #if NET40_OR_GREATER || NETCOREAPP return; #else continue; #endif ProcessFileHelper(datFile, item, datItem, basePath, parent); #if NET40_OR_GREATER || NETCOREAPP }); #else } #endif } /// /// Process blank folders in an archive /// /// Current DatFile object to add to /// File containing the blanks /// Path the represents the parent directory /// BaseArchive to get blanks from private static void ProcessArchiveBlanks(DatFile datFile, string item, string? basePath, BaseArchive archive) { List empties = []; // Get the parent path for all items string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath?.Length ?? 0) + Path.GetFileNameWithoutExtension(item); // Now get all blank folders from the archive if (archive != null) empties = archive.GetEmptyFolders(); // Add add all of the found empties to the DAT #if NET452_OR_GREATER || NETCOREAPP Parallel.ForEach(empties, Core.Globals.ParallelOptions, empty => #elif NET40_OR_GREATER Parallel.ForEach(empties, empty => #else foreach (var empty in empties) #endif { var emptyMachine = new Machine(); emptyMachine.SetFieldValue(Models.Metadata.Machine.NameKey, item); var emptyRom = new Rom(); emptyRom.SetName(Path.Combine(empty, "_")); emptyRom.SetFieldValue(DatItem.MachineKey, emptyMachine); ProcessFileHelper(datFile, item, emptyRom, basePath, parent); #if NET40_OR_GREATER || NETCOREAPP }); #else } #endif } /// /// Process blank folders in a directory /// /// Current DatFile object to add to /// Path the represents the parent directory private static void ProcessDirectoryBlanks(DatFile datFile, string? basePath) { // If we're in depot mode, we don't process blanks if (datFile.Header.GetFieldValue(DatHeader.OutputDepotKey)?.IsActive == true) return; List empties = basePath.ListEmpty() ?? []; #if NET452_OR_GREATER || NETCOREAPP Parallel.ForEach(empties, Core.Globals.ParallelOptions, dir => #elif NET40_OR_GREATER Parallel.ForEach(empties, dir => #else foreach (var dir in empties) #endif { // Get the full path for the directory string fulldir = Path.GetFullPath(dir); // Set the temporary variables string gamename = string.Empty; string romname = string.Empty; // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom if (datFile.Header.GetStringFieldValue(Models.Metadata.Header.TypeKey) == "SuperDAT") { if (basePath != null) gamename = fulldir.Remove(0, basePath.Length + 1); else gamename = fulldir; romname = "_"; } // Otherwise, we want just the top level folder as the game, and the file as everything else else { if (basePath != null) { gamename = fulldir.Remove(0, basePath.Length + 1).Split(Path.DirectorySeparatorChar)[0]; romname = Path.Combine(fulldir.Remove(0, basePath.Length + 1 + gamename.Length), "_"); } else { gamename = fulldir; romname = Path.Combine(fulldir, "_"); } } // Sanitize the names gamename = gamename.Trim(Path.DirectorySeparatorChar); romname = romname.Trim(Path.DirectorySeparatorChar); _staticLogger.Verbose($"Adding blank empty folder: {gamename}"); var blankMachine = new Machine(); blankMachine.SetFieldValue(Models.Metadata.Machine.NameKey, gamename); var blankRom = new Blank(); blankRom.SetName(romname); blankRom.SetFieldValue(DatItem.MachineKey, blankMachine); datFile.AddItem(blankRom, statsOnly: false); #if NET40_OR_GREATER || NETCOREAPP }); #else } #endif } /// /// Process a single file as a file /// /// Current DatFile object to add to /// File to be added /// Path the represents the parent directory /// TreatAsFile representing CHD and Archive scanning private void ProcessFile(DatFile datFile, string item, string? basePath, TreatAsFile asFile) { _staticLogger.Verbose($"'{Path.GetFileName(item)}' treated like a file"); var header = datFile.Header.GetStringFieldValue(Models.Metadata.Header.HeaderKey); BaseFile? baseFile = FileTypeTool.GetInfo(item, _hashes, header); DatItem? datItem = DatItemTool.CreateDatItem(baseFile, asFile); if (datItem != null) ProcessFileHelper(datFile, item, datItem, basePath, string.Empty); } /// /// Process a single file as a file (with found Rom data) /// /// Current DatFile object to add to /// File to be added /// Rom data to be used to write to file /// Path the represents the parent directory /// Parent game to be used private static void ProcessFileHelper(DatFile datFile, string item, DatItem datItem, string? basepath, string parent) { // If we didn't get an accepted parsed type somehow, cancel out List parsed = [ItemType.Disk, ItemType.File, ItemType.Media, ItemType.Rom]; if (!parsed.Contains(datItem.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue())) return; try { // If the basepath doesn't end with a directory separator, add it if (basepath != null && !basepath.EndsWith(Path.DirectorySeparatorChar.ToString())) basepath += Path.DirectorySeparatorChar.ToString(); // Make sure we have the full item path item = Path.GetFullPath(item); // Process the item to sanitize names based on input SetDatItemInfo(datFile, datItem, item, parent, basepath); // Add the file information to the DAT datFile.AddItem(datItem, statsOnly: false); _staticLogger.Verbose($"File added: {datItem.GetName() ?? string.Empty}"); } catch (IOException ex) { _staticLogger.Error(ex); return; } } /// /// Set proper Game and Rom names from user inputs /// /// Current DatFile object to add to /// DatItem representing the input file /// Item name to use /// Parent name to use /// Base path to use private static void SetDatItemInfo(DatFile datFile, DatItem datItem, string item, string parent, string? basepath) { // Get the data to be added as game and item names string? machineName, itemName; // If the parent is blank, then we have a non-archive file if (string.IsNullOrEmpty(parent)) { // If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom if (datFile.Header.GetStringFieldValue(Models.Metadata.Header.TypeKey) == "SuperDAT") { machineName = Path.GetDirectoryName(item.Remove(0, basepath?.Length ?? 0)); itemName = Path.GetFileName(item); } // Otherwise, we want just the top level folder as the game, and the file as everything else else { machineName = item.Remove(0, basepath?.Length ?? 0).Split(Path.DirectorySeparatorChar)[0]; if (basepath != null) itemName = item.Remove(0, (Path.Combine(basepath, machineName).Length)); else itemName = item.Remove(0, (machineName.Length)); } } // Otherwise, we assume that we have an archive else { // If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?) if (datFile.Header.GetStringFieldValue(Models.Metadata.Header.TypeKey) == "SuperDAT") { machineName = parent; itemName = datItem.GetName(); } // Otherwise, we want the archive name as the game, and the file as everything else else { machineName = parent; itemName = datItem.GetName(); } } // Sanitize the names machineName = machineName?.Trim(Path.DirectorySeparatorChar); itemName = itemName?.Trim(Path.DirectorySeparatorChar) ?? string.Empty; if (!string.IsNullOrEmpty(machineName) && string.IsNullOrEmpty(itemName)) { itemName = machineName; machineName = "Default"; } // Update machine information datItem.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.DescriptionKey, machineName); datItem.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.NameKey, machineName); // If we have a Disk, then the ".chd" extension needs to be removed if (datItem is Disk && itemName!.EndsWith(".chd")) { itemName = itemName.Substring(0, itemName.Length - 4); } // If we have a Media, then the extension needs to be removed else if (datItem is Media) { if (itemName!.EndsWith(".dicf")) itemName = itemName.Substring(0, itemName.Length - 5); else if (itemName.EndsWith(".aaru")) itemName = itemName.Substring(0, itemName.Length - 5); else if (itemName.EndsWith(".aaruformat")) itemName = itemName.Substring(0, itemName.Length - 11); else if (itemName.EndsWith(".aaruf")) itemName = itemName.Substring(0, itemName.Length - 6); else if (itemName.EndsWith(".aif")) itemName = itemName.Substring(0, itemName.Length - 4); } // Set the item name back datItem.SetName(itemName); } } }