Files
SabreTools/SabreTools.DatFiles/DatFile.cs
Matt Nadareski b37aed389e Add nullable context to SabreTools.DatItems
This change also starts migrating the internals of the DatItem formats to the new internal models. Right now, it's basically just acting like a wrapper around those models.
2023-08-14 13:17:51 -04:00

610 lines
25 KiB
C#

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Xml.Serialization;
using Newtonsoft.Json;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatFiles.Formats;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
using SabreTools.Logging;
namespace SabreTools.DatFiles
{
/// <summary>
/// Represents a format-agnostic DAT
/// </summary>
[JsonObject("datfile"), XmlRoot("datfile")]
public abstract class DatFile
{
#region Fields
/// <summary>
/// Header values
/// </summary>
[JsonProperty("header"), XmlElement("header")]
public DatHeader Header { get; set; } = new DatHeader();
/// <summary>
/// DatItems and related statistics
/// </summary>
[JsonProperty("items"), XmlElement("items")]
public ItemDictionary Items { get; set; } = new ItemDictionary();
#endregion
#region Logging
/// <summary>
/// Logging object
/// </summary>
[JsonIgnore, XmlIgnore]
protected Logger logger;
#endregion
#region Constructors
/// <summary>
/// Create a new DatFile from an existing one
/// </summary>
/// <param name="datFile">DatFile to get the values from</param>
public DatFile(DatFile? datFile)
{
logger = new Logger(this);
if (datFile != null)
{
Header = datFile.Header;
Items = datFile.Items;
}
}
/// <summary>
/// Create a specific type of DatFile to be used based on a format and a base DAT
/// </summary>
/// <param name="datFormat">Format of the DAT to be created</param>
/// <param name="baseDat">DatFile containing the information to use in specific operations</param>
/// <param name="quotes">For relevant types, assume the usage of quotes</param>
/// <returns>DatFile of the specific internal type that corresponds to the inputs</returns>
public static DatFile Create(DatFormat? datFormat = null, DatFile? baseDat = null, bool quotes = true)
{
return datFormat switch
{
DatFormat.ArchiveDotOrg => new ArchiveDotOrg(baseDat),
DatFormat.AttractMode => new AttractMode(baseDat),
DatFormat.ClrMamePro => new ClrMamePro(baseDat, quotes),
DatFormat.CSV => new SeparatedValue(baseDat, ','),
DatFormat.DOSCenter => new DosCenter(baseDat),
DatFormat.EverdriveSMDB => new EverdriveSMDB(baseDat),
DatFormat.Listrom => new Listrom(baseDat),
DatFormat.Listxml => new Listxml(baseDat),
DatFormat.Logiqx => new Logiqx(baseDat, false),
DatFormat.LogiqxDeprecated => new Logiqx(baseDat, true),
DatFormat.MissFile => new Missfile(baseDat),
DatFormat.OfflineList => new OfflineList(baseDat),
DatFormat.OpenMSX => new OpenMSX(baseDat),
DatFormat.RedumpMD5 => new Hashfile(baseDat, Hash.MD5),
DatFormat.RedumpSFV => new Hashfile(baseDat, Hash.CRC),
DatFormat.RedumpSHA1 => new Hashfile(baseDat, Hash.SHA1),
DatFormat.RedumpSHA256 => new Hashfile(baseDat, Hash.SHA256),
DatFormat.RedumpSHA384 => new Hashfile(baseDat, Hash.SHA384),
DatFormat.RedumpSHA512 => new Hashfile(baseDat, Hash.SHA512),
DatFormat.RedumpSpamSum => new Hashfile(baseDat, Hash.SpamSum),
DatFormat.RomCenter => new RomCenter(baseDat),
DatFormat.SabreJSON => new SabreJSON(baseDat),
DatFormat.SabreXML => new SabreXML(baseDat),
DatFormat.SoftwareList => new Formats.SoftwareList(baseDat),
DatFormat.SSV => new SeparatedValue(baseDat, ';'),
DatFormat.TSV => new SeparatedValue(baseDat, '\t'),
// We use new-style Logiqx as a backup for generic DatFile
_ => new Logiqx(baseDat, false),
};
}
/// <summary>
/// Create a new DatFile from an existing DatHeader
/// </summary>
/// <param name="datHeader">DatHeader to get the values from</param>
public static DatFile Create(DatHeader datHeader)
{
DatFile datFile = Create(datHeader.DatFormat);
datFile.Header = (DatHeader)datHeader.Clone();
return datFile;
}
/// <summary>
/// Fill the header values based on existing Header and path
/// </summary>
/// <param name="path">Path used for creating a name, if necessary</param>
/// <param name="bare">True if the date should be omitted from name and description, false otherwise</param>
public void FillHeaderFromPath(string path, bool bare)
{
// If the description is defined but not the name, set the name from the description
if (string.IsNullOrWhiteSpace(Header.Name) && !string.IsNullOrWhiteSpace(Header.Description))
{
Header.Name = Header.Description;
}
// If the name is defined but not the description, set the description from the name
else if (!string.IsNullOrWhiteSpace(Header.Name) && string.IsNullOrWhiteSpace(Header.Description))
{
Header.Description = Header.Name + (bare ? string.Empty : $" ({Header.Date})");
}
// If neither the name or description are defined, set them from the automatic values
else if (string.IsNullOrWhiteSpace(Header.Name) && string.IsNullOrWhiteSpace(Header.Description))
{
string[] splitpath = path.TrimEnd(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar);
Header.Name = splitpath.Last();
Header.Description = Header.Name + (bare ? string.Empty : $" ({Header.Date})");
}
}
#endregion
#region Parsing
/// <summary>
/// Parse DatFile and return all found games and roms within
/// </summary>
/// <param name="filename">Name of the file to be parsed</param>
/// <param name="indexId">Index ID for the DAT</param>
/// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
public abstract void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false);
/// <summary>
/// Add a rom to the Dat after checking
/// </summary>
/// <param name="item">Item data to check against</param>
/// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
/// <returns>The key for the item</returns>
protected string ParseAddHelper(DatItem item, bool statsOnly)
{
string key;
// If we have a Disk, Media, or Rom, clean the hash data
if (item.ItemType == ItemType.Disk && item is Disk disk)
{
// If the file has aboslutely no hashes, skip and log
if (disk.ItemStatus != ItemStatus.Nodump
&& string.IsNullOrWhiteSpace(disk.MD5)
&& string.IsNullOrWhiteSpace(disk.SHA1))
{
logger.Verbose($"Incomplete entry for '{disk.Name}' will be output as nodump");
disk.ItemStatus = ItemStatus.Nodump;
}
item = disk;
}
if (item.ItemType == ItemType.Media && item is Media media)
{
// If the file has aboslutely no hashes, skip and log
if (string.IsNullOrWhiteSpace(media.MD5)
&& string.IsNullOrWhiteSpace(media.SHA1)
&& string.IsNullOrWhiteSpace(media.SHA256)
&& string.IsNullOrWhiteSpace(media.SpamSum))
{
logger.Verbose($"Incomplete entry for '{media.Name}' will be output as nodump");
}
item = media;
}
else if (item.ItemType == ItemType.Rom && item is Rom rom)
{
// If we have the case where there is SHA-1 and nothing else, we don't fill in any other part of the data
if (rom.Size == null && !rom.HasHashes())
{
// No-op, just catch it so it doesn't go further
logger.Verbose($"{Header.FileName}: Entry with only SHA-1 found - '{rom.Name}'");
}
// If we have a rom and it's missing size AND the hashes match a 0-byte file, fill in the rest of the info
else if ((rom.Size == 0 || rom.Size == null)
&& (string.IsNullOrWhiteSpace(rom.CRC) || rom.HasZeroHash()))
{
// TODO: All instances of Hash.DeepHashes should be made into 0x0 eventually
rom.Size = Constants.SizeZero;
rom.CRC = Constants.CRCZero;
rom.MD5 = Constants.MD5Zero;
rom.SHA1 = Constants.SHA1Zero;
rom.SHA256 = null; // Constants.SHA256Zero;
rom.SHA384 = null; // Constants.SHA384Zero;
rom.SHA512 = null; // Constants.SHA512Zero;
rom.SpamSum = null; // Constants.SpamSumZero;
}
// If the file has no size and it's not the above case, skip and log
else if (rom.ItemStatus != ItemStatus.Nodump && (rom.Size == 0 || rom.Size == null))
{
logger.Verbose($"{Header.FileName}: Incomplete entry for '{rom.Name}' will be output as nodump");
rom.ItemStatus = ItemStatus.Nodump;
}
// If the file has a size but aboslutely no hashes, skip and log
else if (rom.ItemStatus != ItemStatus.Nodump
&& rom.Size != null && rom.Size > 0
&& !rom.HasHashes())
{
logger.Verbose($"{Header.FileName}: Incomplete entry for '{rom.Name}' will be output as nodump");
rom.ItemStatus = ItemStatus.Nodump;
}
item = rom;
}
// Get the key and add the file
key = item.GetKey(ItemKey.Machine);
// If only adding statistics, we add an empty key for games and then just item stats
if (statsOnly)
{
Items.EnsureKey(key);
Items.AddItemStatistics(item);
}
else
{
Items.Add(key, item);
}
return key;
}
/// <summary>
/// Get a sanitized Date from an input string
/// </summary>
/// <param name="input">String to get value from</param>
/// <returns>Date as a string, if possible</returns>
protected static string? CleanDate(string? input)
{
// Null in, null out
if (string.IsNullOrWhiteSpace(input))
return null;
string date = string.Empty;
if (input != null)
{
if (DateTime.TryParse(input, out DateTime dateTime))
date = dateTime.ToString();
else
date = input;
}
return date;
}
/// <summary>
/// Clean a hash string from a Listrom DAT
/// </summary>
/// <param name="hash">Hash string to sanitize</param>
/// <returns>Cleaned string</returns>
protected static string CleanListromHashData(string hash)
{
if (hash.StartsWith("CRC"))
return hash.Substring(4, 8).ToLowerInvariant();
else if (hash.StartsWith("SHA1"))
return hash.Substring(5, 40).ToLowerInvariant();
return hash;
}
#endregion
#region Writing
/// <summary>
/// Create and open an output file for writing direct from a dictionary
/// </summary>
/// <param name="outfile">Name of the file to write to</param>
/// <param name="ignoreblanks">True if blank roms should be skipped on output, false otherwise (default)</param>
/// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
/// <returns>True if the DAT was written correctly, false otherwise</returns>
public abstract bool WriteToFile(string outfile, bool ignoreblanks = false, bool throwOnError = false);
/// <summary>
/// Create a prefix or postfix from inputs
/// </summary>
/// <param name="item">DatItem to create a prefix/postfix for</param>
/// <param name="prefix">True for prefix, false for postfix</param>
/// <returns>Sanitized string representing the postfix or prefix</returns>
protected string CreatePrefixPostfix(DatItem item, bool prefix)
{
// Initialize strings
string fix,
game = item.Machine?.Name ?? string.Empty,
name = item.GetName() ?? item.ItemType.ToString(),
crc = string.Empty,
md5 = string.Empty,
sha1 = string.Empty,
sha256 = string.Empty,
sha384 = string.Empty,
sha512 = string.Empty,
size = string.Empty,
spamsum = string.Empty;
// If we have a prefix
if (prefix)
fix = Header.Prefix + (Header.Quotes ? "\"" : string.Empty);
// If we have a postfix
else
fix = (Header.Quotes ? "\"" : string.Empty) + Header.Postfix;
// Ensure we have the proper values for replacement
if (item.ItemType == ItemType.Disk && item is Disk disk)
{
md5 = disk.MD5 ?? string.Empty;
sha1 = disk.SHA1 ?? string.Empty;
}
else if (item.ItemType == ItemType.Media && item is Media media)
{
md5 = media.MD5 ?? string.Empty;
sha1 = media.SHA1 ?? string.Empty;
sha256 = media.SHA256 ?? string.Empty;
spamsum = media.SpamSum ?? string.Empty;
}
else if (item.ItemType == ItemType.Rom && item is Rom rom)
{
crc = rom.CRC ?? string.Empty;
md5 = rom.MD5 ?? string.Empty;
sha1 = rom.SHA1 ?? string.Empty;
sha256 = rom.SHA256 ?? string.Empty;
sha384 = rom.SHA384 ?? string.Empty;
sha512 = rom.SHA512 ?? string.Empty;
size = rom.Size?.ToString() ?? string.Empty;
spamsum = rom.SpamSum ?? string.Empty;
}
// Now do bulk replacement where possible
fix = fix
.Replace("%game%", game)
.Replace("%machine%", game)
.Replace("%name%", name)
.Replace("%manufacturer%", item.Machine?.Manufacturer ?? string.Empty)
.Replace("%publisher%", item.Machine?.Publisher ?? string.Empty)
.Replace("%category%", item.Machine?.Category ?? string.Empty)
.Replace("%crc%", crc)
.Replace("%md5%", md5)
.Replace("%sha1%", sha1)
.Replace("%sha256%", sha256)
.Replace("%sha384%", sha384)
.Replace("%sha512%", sha512)
.Replace("%size%", size)
.Replace("%spamsum%", spamsum);
return fix;
}
/// <summary>
/// Process an item and correctly set the item name
/// </summary>
/// <param name="item">DatItem to update</param>
/// <param name="forceRemoveQuotes">True if the Quotes flag should be ignored, false otherwise</param>
/// <param name="forceRomName">True if the UseRomName should be always on (default), false otherwise</param>
protected void ProcessItemName(DatItem item, bool forceRemoveQuotes, bool forceRomName = true)
{
// Backup relevant values and set new ones accordingly
bool quotesBackup = Header.Quotes;
bool useRomNameBackup = Header.UseRomName;
if (forceRemoveQuotes)
Header.Quotes = false;
if (forceRomName)
Header.UseRomName = true;
// Get the name to update
string? name = (Header.UseRomName ? item.GetName() : item.Machine?.Name) ?? string.Empty;
// Create the proper Prefix and Postfix
string pre = CreatePrefixPostfix(item, true);
string post = CreatePrefixPostfix(item, false);
// If we're in Depot mode, take care of that instead
if (Header.OutputDepot?.IsActive == true)
{
if (item.ItemType == ItemType.Disk && item is Disk disk)
{
// We can only write out if there's a SHA-1
if (!string.IsNullOrWhiteSpace(disk.SHA1))
{
name = Utilities.GetDepotPath(disk.SHA1, Header.OutputDepot.Depth)?.Replace('\\', '/');
item.SetName($"{pre}{name}{post}");
}
}
else if (item.ItemType == ItemType.Media && item is Media media)
{
// We can only write out if there's a SHA-1
if (!string.IsNullOrWhiteSpace(media.SHA1))
{
name = Utilities.GetDepotPath(media.SHA1, Header.OutputDepot.Depth)?.Replace('\\', '/');
item.SetName($"{pre}{name}{post}");
}
}
else if (item.ItemType == ItemType.Rom && item is Rom rom)
{
// We can only write out if there's a SHA-1
if (!string.IsNullOrWhiteSpace(rom.SHA1))
{
name = Utilities.GetDepotPath(rom.SHA1, Header.OutputDepot.Depth)?.Replace('\\', '/');
item.SetName($"{pre}{name}{post}");
}
}
return;
}
if (!string.IsNullOrWhiteSpace(Header.ReplaceExtension) || Header.RemoveExtension)
{
if (Header.RemoveExtension)
Header.ReplaceExtension = string.Empty;
string? dir = Path.GetDirectoryName(name);
if (dir != null)
{
dir = dir.TrimStart(Path.DirectorySeparatorChar);
name = Path.Combine(dir, Path.GetFileNameWithoutExtension(name) + Header.ReplaceExtension);
}
}
if (!string.IsNullOrWhiteSpace(Header.AddExtension))
name += Header.AddExtension;
if (Header.UseRomName && Header.GameName)
name = Path.Combine(item.Machine?.Name ?? string.Empty, name);
// Now assign back the formatted name
name = $"{pre}{name}{post}";
if (Header.UseRomName)
item.SetName(name);
else if (item.Machine != null)
item.Machine.Name = name;
// Restore all relevant values
if (forceRemoveQuotes)
Header.Quotes = quotesBackup;
if (forceRomName)
Header.UseRomName = useRomNameBackup;
}
/// <summary>
/// Process any DatItems that are "null", usually created from directory population
/// </summary>
/// <param name="datItem">DatItem to check for "null" status</param>
/// <returns>Cleaned DatItem</returns>
protected DatItem ProcessNullifiedItem(DatItem datItem)
{
// If we don't have a Rom, we can ignore it
if (datItem.ItemType != ItemType.Rom)
return datItem;
// Cast for easier parsing
if (datItem is not Rom rom)
return datItem;
// If the Rom has "null" characteristics, ensure all fields
if (rom.Size == null && rom.CRC == "null")
{
logger.Verbose($"Empty folder found: {datItem.Machine?.Name}");
rom.Name = (rom.Name == "null" ? "-" : rom.Name);
rom.Size = Constants.SizeZero;
rom.CRC = rom.CRC == "null" ? Constants.CRCZero : null;
rom.MD5 = rom.MD5 == "null" ? Constants.MD5Zero : null;
rom.SHA1 = rom.SHA1 == "null" ? Constants.SHA1Zero : null;
rom.SHA256 = rom.SHA256 == "null" ? Constants.SHA256Zero : null;
rom.SHA384 = rom.SHA384 == "null" ? Constants.SHA384Zero : null;
rom.SHA512 = rom.SHA512 == "null" ? Constants.SHA512Zero : null;
rom.SpamSum = rom.SpamSum == "null" ? Constants.SpamSumZero : null;
}
return rom;
}
/// <summary>
/// Get supported types for write
/// </summary>
/// <returns>List of supported types for writing</returns>
protected virtual ItemType[] GetSupportedTypes()
{
return Enum.GetValues(typeof(ItemType)) as ItemType[] ?? Array.Empty<ItemType>();
}
/// <summary>
/// Return list of required fields missing from a DatItem
/// </summary>
/// <returns>List of missing required fields, null or empty if none were found</returns>
protected virtual List<DatItemField>? GetMissingRequiredFields(DatItem datItem) => null;
/// <summary>
/// Get if a machine contains any writable items
/// </summary>
/// <param name="datItems">DatItems to check</param>
/// <returns>True if the machine contains at least one writable item, false otherwise</returns>
/// <remarks>Empty machines are kept with this</remarks>
protected bool ContainsWritable(ConcurrentList<DatItem> datItems)
{
// Empty machines are considered writable
if (datItems == null || datItems.Count == 0)
return true;
foreach (DatItem datItem in datItems)
{
if (GetSupportedTypes().Contains(datItem.ItemType))
return true;
}
return false;
}
/// <summary>
/// Get if an item should be ignored on write
/// </summary>
/// <param name="datItem">DatItem to check</param>
/// <param name="ignoreBlanks">True if blank roms should be skipped on output, false otherwise</param>
/// <returns>True if the item should be skipped on write, false otherwise</returns>
protected bool ShouldIgnore(DatItem? datItem, bool ignoreBlanks)
{
// If this is invoked with a null DatItem, we ignore
if (datItem == null)
{
logger?.Verbose($"Item was skipped because it was null");
return true;
}
// If the item is supposed to be removed, we ignore
if (datItem.Remove)
{
string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
logger?.Verbose($"Item '{itemString}' was skipped because it was marked for removal");
return true;
}
// If we have the Blank dat item, we ignore
if (datItem.ItemType == ItemType.Blank)
{
string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
logger?.Verbose($"Item '{itemString}' was skipped because it was of type 'Blank'");
return true;
}
// If we're ignoring blanks and we have a Rom
if (ignoreBlanks && datItem.ItemType == ItemType.Rom && datItem is Rom rom)
{
// If we have a 0-size or blank rom, then we ignore
if (rom.Size == 0 || rom.Size == null)
{
string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
logger?.Verbose($"Item '{itemString}' was skipped because it had an invalid size");
return true;
}
}
// If we have an item type not in the list of supported values
if (!GetSupportedTypes().Contains(datItem.ItemType))
{
string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
logger?.Verbose($"Item '{itemString}' was skipped because it was not supported in {Header?.DatFormat}");
return true;
}
// If we have an item with missing required fields
List<DatItemField>? missingFields = GetMissingRequiredFields(datItem);
if (missingFields != null && missingFields.Count != 0)
{
string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
logger?.Verbose($"Item '{itemString}' was skipped because it was missing required fields for {Header?.DatFormat}: {string.Join(", ", missingFields)}");
return true;
}
return false;
}
#endregion
}
}