using System.Collections.Generic;
using System.IO;
using System.Threading;
#if NET40_OR_GREATER || NETCOREAPP
using System.Threading.Tasks;
#endif
using SabreTools.Core.Tools;
using SabreTools.DatFiles;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
using SabreTools.FileTypes;
using SabreTools.FileTypes.Archives;
using SabreTools.Hashing;
using SabreTools.IO.Extensions;
using SabreTools.IO.Logging;
namespace SabreTools.DatTools
{
///
/// This file represents all methods related to populating a DatFile
/// from a set of files and directories
///
public class DatFromDir
{
#region Fields
///
/// Hashes to include in the information
///
private readonly HashType[] _hashes;
///
/// Type of files that should be skipped
///
private readonly SkipFileType _skipFileType;
///
/// Indicates if blank items should be created for empty folders
///
private readonly bool _addBlanks;
#endregion
#region Logging
///
/// Logging object
///
private static readonly Logger _staticLogger = new();
#endregion
#region Constructors
public DatFromDir(HashType[]? hashes, SkipFileType skipFileType, bool addBlanks)
{
_hashes = hashes ?? [HashType.CRC32, HashType.MD5, HashType.SHA1];
_skipFileType = skipFileType;
_addBlanks = addBlanks;
}
#endregion
///
/// Create a new Dat from a directory
///
/// Current DatFile object to add to
/// Base folder to be used in creating the DAT
/// TreatAsFile representing CHD and Archive scanning
public bool PopulateFromDir(DatFile datFile, string basePath, TreatAsFile asFile = 0x00)
{
// Set the progress variables
long totalSize = 0;
long currentSize = 0;
InternalStopwatch watch = new($"Populating DAT from {basePath}");
// Process the input
if (Directory.Exists(basePath))
{
_staticLogger.Verbose($"Folder found: {basePath}");
// Get a list of all files to process
#if NET20 || NET35
List files = [.. Directory.GetFiles(basePath, "*")];
#else
List files = [.. Directory.EnumerateFiles(basePath, "*", SearchOption.AllDirectories)];
#endif
// Loop through and add the file sizes
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(files, Core.Globals.ParallelOptions, item =>
#elif NET40_OR_GREATER
Parallel.ForEach(files, item =>
#else
foreach (var item in files)
#endif
{
Interlocked.Add(ref totalSize, new FileInfo(item).Length);
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
// Process the files in the main folder or any subfolder
_staticLogger.User(totalSize, currentSize);
foreach (string item in files)
{
currentSize += new FileInfo(item).Length;
CheckFileForHashes(datFile, item, basePath, asFile);
_staticLogger.User(totalSize, currentSize, item);
}
// Now find all folders that are empty, if we are supposed to
if (_addBlanks)
ProcessDirectoryBlanks(datFile, basePath);
}
else if (System.IO.File.Exists(basePath))
{
_staticLogger.Verbose($"File found: {basePath}");
totalSize = new FileInfo(basePath).Length;
_staticLogger.User(totalSize, currentSize);
string? parentPath = Path.GetDirectoryName(Path.GetDirectoryName(basePath));
CheckFileForHashes(datFile, basePath, parentPath, asFile);
_staticLogger.User(totalSize, totalSize, basePath);
}
watch.Stop();
return true;
}
///
/// Check a given file for hashes, based on current settings
///
/// Current DatFile object to add to
/// Filename of the item to be checked
/// Base folder to be used in creating the DAT
/// TreatAsFile representing CHD and Archive scanning
private void CheckFileForHashes(DatFile datFile, string item, string? basePath, TreatAsFile asFile)
{
// If we're in depot mode, process it separately
if (CheckDepotFile(datFile, item))
return;
// Initialize possible archive variables
BaseArchive? archive = FileTypeTool.CreateArchiveType(item);
// Process archives according to flags
if (archive != null)
{
// Set the archive flags
archive.SetHashTypes(_hashes);
// Skip if we're treating archives as files and skipping files
#if NET20 || NET35
if ((asFile & TreatAsFile.Archive) != 0 && _skipFileType == SkipFileType.File)
#else
if (asFile.HasFlag(TreatAsFile.Archive) && _skipFileType == SkipFileType.File)
#endif
{
return;
}
// Skip if we're skipping archives
else if (_skipFileType == SkipFileType.Archive)
{
return;
}
// Process as archive if we're not treating archives as files
#if NET20 || NET35
else if ((asFile & TreatAsFile.Archive) == 0)
#else
else if (!asFile.HasFlag(TreatAsFile.Archive))
#endif
{
var extracted = archive.GetChildren();
// If we have internal items to process, do so
if (extracted != null)
ProcessArchive(datFile, item, basePath, extracted);
// Now find all folders that are empty, if we are supposed to
if (_addBlanks)
ProcessArchiveBlanks(datFile, item, basePath, archive);
}
// Process as file if we're treating archives as files
else
{
ProcessFile(datFile, item, basePath, asFile);
}
}
// Process non-archives according to flags
else
{
// Skip if we're skipping files
if (_skipFileType == SkipFileType.File)
return;
// Process as file
else
ProcessFile(datFile, item, basePath, asFile);
}
}
///
/// Check an item as if it's supposed to be in a depot
///
/// Current DatFile object to add to
/// Filename of the item to be checked
/// True if we checked a depot file, false otherwise
private static bool CheckDepotFile(DatFile datFile, string item)
{
// If we're not in Depot mode, return false
if (datFile.Modifiers.OutputDepot?.IsActive != true)
return false;
// Check the file as if it were in a depot
GZipArchive gzarc = new(item);
BaseFile? baseFile = gzarc.GetTorrentGZFileInfo();
// If the rom is valid, add it
if (baseFile != null && baseFile.Filename != null)
{
// Add the list if it doesn't exist already
Rom rom = baseFile.ConvertToRom();
datFile.AddItem(rom, statsOnly: false);
_staticLogger.Verbose($"File added: {Path.GetFileNameWithoutExtension(item)}");
}
else
{
_staticLogger.Verbose($"File not added: {Path.GetFileNameWithoutExtension(item)}");
return true;
}
return true;
}
///
/// Process a single file as an archive
///
/// Current DatFile object to add to
/// File to be added
/// Path the represents the parent directory
/// List of BaseFiles representing the internal files
private static void ProcessArchive(DatFile datFile, string item, string? basePath, List extracted)
{
// Get the parent path for all items
string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath?.Length ?? 0) + Path.GetFileNameWithoutExtension(item);
// First take care of the found items
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(extracted, Core.Globals.ParallelOptions, baseFile =>
#elif NET40_OR_GREATER
Parallel.ForEach(extracted, baseFile =>
#else
foreach (var baseFile in extracted)
#endif
{
DatItem? datItem = DatItemTool.CreateDatItem(baseFile);
if (datItem == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
ProcessFileHelper(datFile, item, datItem, basePath, parent);
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
}
///
/// Process blank folders in an archive
///
/// Current DatFile object to add to
/// File containing the blanks
/// Path the represents the parent directory
/// BaseArchive to get blanks from
private static void ProcessArchiveBlanks(DatFile datFile, string item, string? basePath, BaseArchive archive)
{
List empties = [];
// Get the parent path for all items
string parent = (Path.GetDirectoryName(Path.GetFullPath(item)) + Path.DirectorySeparatorChar).Remove(0, basePath?.Length ?? 0) + Path.GetFileNameWithoutExtension(item);
// Now get all blank folders from the archive
if (archive != null)
empties = archive.GetEmptyFolders();
// Add add all of the found empties to the DAT
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(empties, Core.Globals.ParallelOptions, empty =>
#elif NET40_OR_GREATER
Parallel.ForEach(empties, empty =>
#else
foreach (var empty in empties)
#endif
{
var emptyMachine = new Machine();
emptyMachine.SetFieldValue(Models.Metadata.Machine.NameKey, item);
var emptyRom = new Rom();
emptyRom.SetName(Path.Combine(empty, "_"));
emptyRom.SetFieldValue(DatItem.MachineKey, emptyMachine);
ProcessFileHelper(datFile, item, emptyRom, basePath, parent);
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
}
///
/// Process blank folders in a directory
///
/// Current DatFile object to add to
/// Path the represents the parent directory
private static void ProcessDirectoryBlanks(DatFile datFile, string? basePath)
{
// If we're in depot mode, we don't process blanks
if (datFile.Modifiers.OutputDepot?.IsActive == true)
return;
List empties = basePath.ListEmpty() ?? [];
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(empties, Core.Globals.ParallelOptions, dir =>
#elif NET40_OR_GREATER
Parallel.ForEach(empties, dir =>
#else
foreach (var dir in empties)
#endif
{
// Get the full path for the directory
string fulldir = Path.GetFullPath(dir);
// Set the temporary variables
string gamename = string.Empty;
string romname = string.Empty;
// If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
if (datFile.Header.GetStringFieldValue(Models.Metadata.Header.TypeKey) == "SuperDAT")
{
if (basePath != null)
gamename = fulldir.Remove(0, basePath.Length + 1);
else
gamename = fulldir;
romname = "_";
}
// Otherwise, we want just the top level folder as the game, and the file as everything else
else
{
if (basePath != null)
{
gamename = fulldir.Remove(0, basePath.Length + 1).Split(Path.DirectorySeparatorChar)[0];
romname = Path.Combine(fulldir.Remove(0, basePath.Length + 1 + gamename.Length), "_");
}
else
{
gamename = fulldir;
romname = Path.Combine(fulldir, "_");
}
}
// Sanitize the names
gamename = gamename.Trim(Path.DirectorySeparatorChar);
romname = romname.Trim(Path.DirectorySeparatorChar);
_staticLogger.Verbose($"Adding blank empty folder: {gamename}");
var blankMachine = new Machine();
blankMachine.SetFieldValue(Models.Metadata.Machine.NameKey, gamename);
var blankRom = new Blank();
blankRom.SetName(romname);
blankRom.SetFieldValue(DatItem.MachineKey, blankMachine);
datFile.AddItem(blankRom, statsOnly: false);
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
}
///
/// Process a single file as a file
///
/// Current DatFile object to add to
/// File to be added
/// Path the represents the parent directory
/// TreatAsFile representing CHD and Archive scanning
private void ProcessFile(DatFile datFile, string item, string? basePath, TreatAsFile asFile)
{
_staticLogger.Verbose($"'{Path.GetFileName(item)}' treated like a file");
var header = datFile.Header.GetStringFieldValue(Models.Metadata.Header.HeaderKey);
BaseFile? baseFile = FileTypeTool.GetInfo(item, _hashes, header);
DatItem? datItem = DatItemTool.CreateDatItem(baseFile, asFile);
if (datItem != null)
ProcessFileHelper(datFile, item, datItem, basePath, string.Empty);
}
///
/// Process a single file as a file (with found Rom data)
///
/// Current DatFile object to add to
/// File to be added
/// Rom data to be used to write to file
/// Path the represents the parent directory
/// Parent game to be used
private static void ProcessFileHelper(DatFile datFile, string item, DatItem datItem, string? basepath, string parent)
{
// If we didn't get an accepted parsed type somehow, cancel out
List parsed = [ItemType.Disk, ItemType.File, ItemType.Media, ItemType.Rom];
if (!parsed.Contains(datItem.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue()))
return;
try
{
// If the basepath doesn't end with a directory separator, add it
if (basepath != null && !basepath.EndsWith(Path.DirectorySeparatorChar.ToString()))
basepath += Path.DirectorySeparatorChar.ToString();
// Make sure we have the full item path
item = Path.GetFullPath(item);
// Process the item to sanitize names based on input
SetDatItemInfo(datFile, datItem, item, parent, basepath);
// Add the file information to the DAT
datFile.AddItem(datItem, statsOnly: false);
_staticLogger.Verbose($"File added: {datItem.GetName() ?? string.Empty}");
}
catch (IOException ex)
{
_staticLogger.Error(ex);
return;
}
}
///
/// Set proper Game and Rom names from user inputs
///
/// Current DatFile object to add to
/// DatItem representing the input file
/// Item name to use
/// Parent name to use
/// Base path to use
private static void SetDatItemInfo(DatFile datFile, DatItem datItem, string item, string parent, string? basepath)
{
// Get the data to be added as game and item names
string? machineName, itemName;
// If the parent is blank, then we have a non-archive file
if (string.IsNullOrEmpty(parent))
{
// If we have a SuperDAT, we want anything that's not the base path as the game, and the file as the rom
if (datFile.Header.GetStringFieldValue(Models.Metadata.Header.TypeKey) == "SuperDAT")
{
machineName = Path.GetDirectoryName(item.Remove(0, basepath?.Length ?? 0));
itemName = Path.GetFileName(item);
}
// Otherwise, we want just the top level folder as the game, and the file as everything else
else
{
machineName = item.Remove(0, basepath?.Length ?? 0).Split(Path.DirectorySeparatorChar)[0];
if (basepath != null)
itemName = item.Remove(0, (Path.Combine(basepath, machineName).Length));
else
itemName = item.Remove(0, (machineName.Length));
}
}
// Otherwise, we assume that we have an archive
else
{
// If we have a SuperDAT, we want the archive name as the game, and the file as everything else (?)
if (datFile.Header.GetStringFieldValue(Models.Metadata.Header.TypeKey) == "SuperDAT")
{
machineName = parent;
itemName = datItem.GetName();
}
// Otherwise, we want the archive name as the game, and the file as everything else
else
{
machineName = parent;
itemName = datItem.GetName();
}
}
// Sanitize the names
machineName = machineName?.Trim(Path.DirectorySeparatorChar);
itemName = itemName?.Trim(Path.DirectorySeparatorChar) ?? string.Empty;
if (!string.IsNullOrEmpty(machineName) && string.IsNullOrEmpty(itemName))
{
itemName = machineName;
machineName = "Default";
}
// Update machine information
datItem.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.DescriptionKey, machineName);
datItem.GetFieldValue(DatItem.MachineKey)!.SetFieldValue(Models.Metadata.Machine.NameKey, machineName);
// If we have a Disk, then the ".chd" extension needs to be removed
if (datItem is Disk && itemName!.EndsWith(".chd"))
{
itemName = itemName.Substring(0, itemName.Length - 4);
}
// If we have a Media, then the extension needs to be removed
else if (datItem is Media)
{
if (itemName!.EndsWith(".dicf"))
itemName = itemName.Substring(0, itemName.Length - 5);
else if (itemName.EndsWith(".aaru"))
itemName = itemName.Substring(0, itemName.Length - 5);
else if (itemName.EndsWith(".aaruformat"))
itemName = itemName.Substring(0, itemName.Length - 11);
else if (itemName.EndsWith(".aaruf"))
itemName = itemName.Substring(0, itemName.Length - 6);
else if (itemName.EndsWith(".aif"))
itemName = itemName.Substring(0, itemName.Length - 4);
}
// Set the item name back
datItem.SetName(itemName);
}
}
}