SabreTools/SabreTools.DatFiles/DatFile.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using System.Xml.Serialization;
using Newtonsoft.Json;
using SabreTools.Core;
using SabreTools.Core.Filter;
using SabreTools.Core.Tools;
using SabreTools.DatFiles.Formats;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
using SabreTools.Hashing;
using SabreTools.Logging;
using SabreTools.Serialization.Interfaces;

namespace SabreTools.DatFiles
{
    /// <summary>
    /// Represents a format-agnostic DAT
    /// </summary>
    [JsonObject("datfile"), XmlRoot("datfile")]
    public abstract partial class DatFile
    {
        #region Fields

        /// <summary>
        /// Header values
        /// </summary>
        [JsonProperty("header"), XmlElement("header")]
        public DatHeader Header { get; private set; } = new DatHeader();

        /// <summary>
        /// DatItems and related statistics
        /// </summary>
        [JsonProperty("items"), XmlElement("items")]
        public ItemDictionary Items { get; private set; } = [];

        /// <summary>
        /// DatItems and related statistics
        /// </summary>
        [JsonProperty("items"), XmlElement("items")]
        public ItemDictionaryDB ItemsDB { get; private set; } = new ItemDictionaryDB();

        #endregion

        #region Logging

        /// <summary>
        /// Logging object
        /// </summary>
        [JsonIgnore, XmlIgnore]
        protected Logger logger;

        #endregion

        #region Constructors

        /// <summary>
        /// Create a new DatFile from an existing one
        /// </summary>
        /// <param name="datFile">DatFile to get the values from</param>
        public DatFile(DatFile? datFile)
        {
            logger = new Logger(this);
            if (datFile != null)
            {
                Header = datFile.Header;
                Items = datFile.Items;
                ItemsDB = datFile.ItemsDB;
            }
        }

        /// <summary>
        /// Create a specific type of DatFile to be used based on a format and a base DAT
        /// </summary>
        /// <param name="datFormat">Format of the DAT to be created</param>
        /// <param name="baseDat">DatFile containing the information to use in specific operations</param>
        /// <param name="quotes">For relevant types, assume the usage of quotes</param>
        /// <returns>DatFile of the specific internal type that corresponds to the inputs</returns>
        public static DatFile Create(DatFormat? datFormat = null, DatFile? baseDat = null, bool quotes = true)
        {
            return datFormat switch
            {
                DatFormat.ArchiveDotOrg => new ArchiveDotOrg(baseDat),
                DatFormat.AttractMode => new AttractMode(baseDat),
                DatFormat.ClrMamePro => new ClrMamePro(baseDat, quotes),
                DatFormat.CSV => new CommaSeparatedValue(baseDat),
                DatFormat.DOSCenter => new DosCenter(baseDat),
                DatFormat.EverdriveSMDB => new EverdriveSMDB(baseDat),
                DatFormat.Listrom => new Listrom(baseDat),
                DatFormat.Listxml => new Listxml(baseDat),
                DatFormat.Logiqx => new Logiqx(baseDat, false),
                DatFormat.LogiqxDeprecated => new Logiqx(baseDat, true),
                DatFormat.MissFile => new Missfile(baseDat),
                DatFormat.OfflineList => new OfflineList(baseDat),
                DatFormat.OpenMSX => new OpenMSX(baseDat),
                DatFormat.RedumpMD5 => new Md5File(baseDat),
                DatFormat.RedumpSFV => new SfvFile(baseDat),
                DatFormat.RedumpSHA1 => new Sha1File(baseDat),
                DatFormat.RedumpSHA256 => new Sha256File(baseDat),
                DatFormat.RedumpSHA384 => new Sha384File(baseDat),
                DatFormat.RedumpSHA512 => new Sha512File(baseDat),
                DatFormat.RedumpSpamSum => new SpamSumFile(baseDat),
                DatFormat.RomCenter => new RomCenter(baseDat),
                DatFormat.SabreJSON => new SabreJSON(baseDat),
                DatFormat.SabreXML => new SabreXML(baseDat),
                DatFormat.SoftwareList => new Formats.SoftwareList(baseDat),
                DatFormat.SSV => new SemicolonSeparatedValue(baseDat),
                DatFormat.TSV => new TabSeparatedValue(baseDat),

                // We use new-style Logiqx as a backup for generic DatFile
                _ => new Logiqx(baseDat, false),
            };
        }

        /// <summary>
        /// Create a new DatFile from an existing DatHeader
        /// </summary>
        /// <param name="datHeader">DatHeader to get the values from</param>
        public static DatFile Create(DatHeader datHeader)
        {
            DatFile datFile = Create(datHeader.GetFieldValue<DatFormat>(DatHeader.DatFormatKey));
            datFile.Header = (DatHeader)datHeader.Clone();
            return datFile;
        }

        /// <summary>
        /// Fill the header values based on existing Header and path
        /// </summary>
        /// <param name="path">Path used for creating a name, if necessary</param>
        /// <param name="bare">True if the date should be omitted from name and description, false otherwise</param>
        public void FillHeaderFromPath(string path, bool bare)
        {
            // If the description is defined but not the name, set the name from the description
            if (string.IsNullOrEmpty(Header.GetStringFieldValue(Models.Metadata.Header.NameKey)) && !string.IsNullOrEmpty(Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey)))
            {
                Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey));
            }

            // If the name is defined but not the description, set the description from the name
            else if (!string.IsNullOrEmpty(Header.GetStringFieldValue(Models.Metadata.Header.NameKey)) && string.IsNullOrEmpty(Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey)))
            {
                Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + (bare ? string.Empty : $" ({Header.GetStringFieldValue(Models.Metadata.Header.DateKey)})"));
            }

            // If neither the name or description are defined, set them from the automatic values
            else if (string.IsNullOrEmpty(Header.GetStringFieldValue(Models.Metadata.Header.NameKey)) && string.IsNullOrEmpty(Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey)))
            {
                string[] splitpath = path.TrimEnd(Path.DirectorySeparatorChar).Split(Path.DirectorySeparatorChar);
                Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, splitpath.Last());
                Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + (bare ? string.Empty : $" ({Header.GetStringFieldValue(Models.Metadata.Header.DateKey)})"));
            }
        }

        #endregion

        #region Accessors

        /// <summary>
        /// Reset the internal item dictionary
        /// </summary>
        public void ResetDictionary()
        {
            Items = [];
            ItemsDB = new ItemDictionaryDB();
        }

        #endregion

        #region Filtering

        /// <summary>
        /// Execute all filters in a filter runner on the items in the dictionary
        /// </summary>
        /// <param name="filterRunner">Preconfigured filter runner to use</param>
        public void ExecuteFilters(FilterRunner filterRunner)
        {
            Items.ExecuteFilters(filterRunner);
            ItemsDB.ExecuteFilters(filterRunner);
        }

        #endregion

        #region Parsing

        /// <summary>
        /// Parse DatFile and return all found games and roms within
        /// </summary>
        /// <param name="filename">Name of the file to be parsed</param>
        /// <param name="indexId">Index ID for the DAT</param>
        /// <param name="keep">True if full pathnames are to be kept, false otherwise (default)</param>
        /// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
        /// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
        public abstract void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false);

        /// <summary>
        /// Add a DatItem to the dictionary after checking
        /// </summary>
        /// <param name="item">Item data to check against</param>
        /// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
        /// <returns>The key for the item</returns>
        protected string ParseAddHelper(DatItem item, bool statsOnly)
            => Items.AddItem(item, statsOnly);

        /// <summary>
        /// Add a DatItem to the dictionary after checking
        /// </summary>
        /// <param name="item">Item data to check against</param>
        /// <param name="machineIndex">Index of the machine to map the DatItem to</param>
        /// <param name="statsOnly">True to only add item statistics while parsing, false otherwise</param>
        /// <returns>The key for the item</returns>
        protected long ParseAddHelper(DatItem item, long machineIndex, bool statsOnly)
            => ItemsDB.AddItem(item, machineIndex, statsOnly);

        #endregion

        #region Removal

        /// <summary>
        /// Remove fields indicated by the three input lists
        /// </summary>
        public void ApplyRemovals(List<string> headerFieldNames, List<string> machineFieldNames, Dictionary<string, List<string>> itemFieldNames)
        {
            // Remove DatHeader fields
            if (headerFieldNames.Count > 0)
                RemoveHeaderFields(headerFieldNames);

            // Remove DatItem and Machine fields
            if (machineFieldNames.Count > 0 || itemFieldNames.Count > 0)
            {
                ApplyRemovalsItemDictionary(machineFieldNames, itemFieldNames);
                ApplyRemovalsItemDictionaryDB(machineFieldNames, itemFieldNames);
            }
        }

        /// <summary>
        /// Apply removals to the item dictionary
        /// </summary>
        private void ApplyRemovalsItemDictionary(List<string> machineFieldNames, Dictionary<string, List<string>> itemFieldNames)
        {
#if NET452_OR_GREATER || NETCOREAPP
            Parallel.ForEach(Items.Keys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
            Parallel.ForEach(Items.Keys, key =>
#else
            foreach (var key in Items.Keys)
#endif
            {
                ConcurrentList<DatItem>? items = Items[key];
                if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
                    return;
#else
                    continue;
#endif

                for (int j = 0; j < items.Count; j++)
                {
                    RemoveFields(items[j], machineFieldNames, itemFieldNames);
                }
#if NET40_OR_GREATER || NETCOREAPP
            });
#else
            }
#endif
        }

        /// <summary>
        /// Apply removals to the item dictionary
        /// </summary>
        private void ApplyRemovalsItemDictionaryDB(List<string> machineFieldNames, Dictionary<string, List<string>> itemFieldNames)
        {
#if NET452_OR_GREATER || NETCOREAPP
            Parallel.ForEach(ItemsDB.SortedKeys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
            Parallel.ForEach(ItemsDB.SortedKeys, key =>
#else
            foreach (var key in ItemsDB.SortedKeys)
#endif
            {
                var items = ItemsDB.GetDatItemsForBucket(key);
                if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
                    return;
#else
                    continue;
#endif

                for (int j = 0; j < items.Length; j++)
                {
                    RemoveFields(items[j].Item2, machineFieldNames, itemFieldNames);
                }
#if NET40_OR_GREATER || NETCOREAPP
            });
#else
            }
#endif
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        private void RemoveHeaderFields(List<string> headerFieldNames)
        {
            // If we have an invalid input, return
            if (Header == null || !headerFieldNames.Any())
                return;

            foreach (var fieldName in headerFieldNames)
            {
                Header.RemoveField(fieldName);
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        private void RemoveFields(Machine? machine, List<string> machineFieldNames)
        {
            // If we have an invalid input, return
            if (machine == null || !machineFieldNames.Any())
                return;

            foreach (var fieldName in machineFieldNames)
            {
                machine.RemoveField(fieldName);
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="datItem">DatItem to remove fields from</param>
        private void RemoveFields(DatItem? datItem, List<string> machineFieldNames, Dictionary<string, List<string>> itemFieldNames)
        {
            if (datItem == null)
                return;

            #region Common

            // Handle Machine fields
            if (machineFieldNames.Any() && datItem.GetFieldValue<Machine>(DatItem.MachineKey) != null)
                RemoveFields(datItem.GetFieldValue<Machine>(DatItem.MachineKey), machineFieldNames);

            // If there are no field names, return
            if (itemFieldNames == null || !itemFieldNames.Any())
                return;

            // If there are no field names for this type or generic, return
            string? itemType = datItem.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>().AsStringValue();
            if (itemType == null || (!itemFieldNames.ContainsKey(itemType) && !itemFieldNames.ContainsKey("item")))
                return;

            // Get the combined list of fields to remove
            var fieldNames = new List<string>();
            if (itemFieldNames.ContainsKey(itemType))
                fieldNames.AddRange(itemFieldNames[itemType]);
            if (itemFieldNames.ContainsKey("item"))
                fieldNames.AddRange(itemFieldNames["item"]);
            fieldNames = fieldNames.Distinct().ToList();

            // If the field specifically contains Name, set it separately
            if (fieldNames.Contains(Models.Metadata.Rom.NameKey))
                datItem.SetName(null);

            #endregion

            #region Item-Specific

            // Handle unnested removals first
            foreach (var datItemField in fieldNames)
            {
                datItem.RemoveField(datItemField);
            }

            // Handle nested removals
            switch (datItem)
            {
                case Adjuster adjuster: RemoveFields(adjuster, itemFieldNames); break;
                case Configuration configuration: RemoveFields(configuration, itemFieldNames); break;
                case ConfSetting confSetting: RemoveFields(confSetting, itemFieldNames); break;
                case Device device: RemoveFields(device, itemFieldNames); break;
                case DipSwitch dipSwitch: RemoveFields(dipSwitch, itemFieldNames); break;
                case DipValue dipValue: RemoveFields(dipValue, itemFieldNames); break;
                case Disk disk: RemoveFields(disk, itemFieldNames); break;
                case Input input: RemoveFields(input, itemFieldNames); break;
                case Part part: RemoveFields(part, itemFieldNames); break;
                case Port port: RemoveFields(port, itemFieldNames); break;
                case Rom rom: RemoveFields(rom, itemFieldNames); break;
                case Slot slot: RemoveFields(slot, itemFieldNames); break;
            }

            #endregion
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="adjuster">Adjuster to remove fields from</param>
        private void RemoveFields(Adjuster adjuster, Dictionary<string, List<string>> itemFieldNames)
        {
            if (!adjuster.ConditionsSpecified)
                return;

            foreach (Condition subCondition in adjuster.GetFieldValue<Condition[]?>(Models.Metadata.Adjuster.ConditionKey)!)
            {
                RemoveFields(subCondition, [], itemFieldNames);
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="configuration">Configuration to remove fields from</param>
        private void RemoveFields(Configuration configuration, Dictionary<string, List<string>> itemFieldNames)
        {
            if (configuration.ConditionsSpecified)
            {
                foreach (Condition subCondition in configuration.GetFieldValue<Condition[]?>(Models.Metadata.Configuration.ConditionKey)!)
                {
                    RemoveFields(subCondition, [], itemFieldNames);
                }
            }

            if (configuration.LocationsSpecified)
            {
                foreach (ConfLocation subLocation in configuration.GetFieldValue<ConfLocation[]?>(Models.Metadata.Configuration.ConfLocationKey)!)
                {
                    RemoveFields(subLocation, [], itemFieldNames);
                }
            }

            if (configuration.SettingsSpecified)
            {
                foreach (ConfSetting subSetting in configuration.GetFieldValue<ConfSetting[]?>(Models.Metadata.Configuration.ConfSettingKey)!)
                {
                    RemoveFields(subSetting as DatItem, [], itemFieldNames);
                }
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="confsetting">ConfSetting to remove fields from</param>
        private void RemoveFields(ConfSetting confsetting, Dictionary<string, List<string>> itemFieldNames)
        {
            if (confsetting.ConditionsSpecified)
            {
                foreach (Condition subCondition in confsetting.GetFieldValue<Condition[]?>(Models.Metadata.ConfSetting.ConditionKey)!)
                {
                    RemoveFields(subCondition, [], itemFieldNames);
                }
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="device">Device to remove fields from</param>
        private void RemoveFields(Device device, Dictionary<string, List<string>> itemFieldNames)
        {
            if (device.ExtensionsSpecified)
            {
                foreach (Extension subExtension in device.GetFieldValue<Extension[]?>(Models.Metadata.Device.ExtensionKey)!)
                {
                    RemoveFields(subExtension, [], itemFieldNames);
                }
            }

            if (device.InstancesSpecified)
            {
                foreach (Instance subInstance in device.GetFieldValue<Instance[]?>(Models.Metadata.Device.InstanceKey)!)
                {
                    RemoveFields(subInstance, [], itemFieldNames);
                }
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="dipSwitch">DipSwitch to remove fields from</param>
        private void RemoveFields(DipSwitch dipSwitch, Dictionary<string, List<string>> itemFieldNames)
        {
            if (dipSwitch.ConditionsSpecified)
            {
                foreach (Condition subCondition in dipSwitch.GetFieldValue<Condition[]?>(Models.Metadata.DipSwitch.ConditionKey)!)
                {
                    RemoveFields(subCondition, [], itemFieldNames);
                }
            }

            if (dipSwitch.LocationsSpecified)
            {
                foreach (DipLocation subLocation in dipSwitch.GetFieldValue<DipLocation[]?>(Models.Metadata.DipSwitch.DipLocationKey)!)
                {
                    RemoveFields(subLocation, [], itemFieldNames);
                }
            }

            if (dipSwitch.ValuesSpecified)
            {
                foreach (DipValue subValue in dipSwitch.GetFieldValue<DipValue[]?>(Models.Metadata.DipSwitch.DipValueKey)!)
                {
                    RemoveFields(subValue as DatItem, [], itemFieldNames);
                }
            }

            if (dipSwitch.PartSpecified)
                RemoveFields(dipSwitch.GetFieldValue<Part?>(DipSwitch.PartKey)! as DatItem, [], itemFieldNames);
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="dipValue">DipValue to remove fields from</param>
        private void RemoveFields(DipValue dipValue, Dictionary<string, List<string>> itemFieldNames)
        {
            if (dipValue.ConditionsSpecified)
            {
                foreach (Condition subCondition in dipValue.GetFieldValue<Condition[]?>(Models.Metadata.DipValue.ConditionKey)!)
                {
                    RemoveFields(subCondition, [], itemFieldNames);
                }
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="disk">Disk to remove fields from</param>
        private void RemoveFields(Disk disk, Dictionary<string, List<string>> itemFieldNames)
        {
            if (disk.DiskAreaSpecified)
                RemoveFields(disk.GetFieldValue<DiskArea?>(Disk.DiskAreaKey)! as DatItem, [], itemFieldNames);

            if (disk.PartSpecified)
                RemoveFields(disk.GetFieldValue<Part?>(Disk.PartKey)! as DatItem, [], itemFieldNames);
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="input">Input to remove fields from</param>
        private void RemoveFields(Input input, Dictionary<string, List<string>> itemFieldNames)
        {
            if (input.ControlsSpecified)
            {
                foreach (Control subControl in input.GetFieldValue<Control[]?>(Models.Metadata.Input.ControlKey)!)
                {
                    RemoveFields(subControl, [], itemFieldNames);
                }
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="part">Part to remove fields from</param>
        private void RemoveFields(Part part, Dictionary<string, List<string>> itemFieldNames)
        {
            if (part.FeaturesSpecified)
            {
                foreach (PartFeature subPartFeature in part.GetFieldValue<PartFeature[]?>(Models.Metadata.Part.FeatureKey)!)
                {
                    RemoveFields(subPartFeature, [], itemFieldNames);
                }
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="port">Port to remove fields from</param>
        private void RemoveFields(Port port, Dictionary<string, List<string>> itemFieldNames)
        {
            if (port.AnalogsSpecified)
            {
                foreach (Analog subAnalog in port.GetFieldValue<Analog[]?>(Models.Metadata.Port.AnalogKey)!)
                {
                    RemoveFields(subAnalog, [], itemFieldNames);
                }
            }
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="rom">Rom to remove fields from</param>
        private void RemoveFields(Rom rom, Dictionary<string, List<string>> itemFieldNames)
        {
            if (rom.DataAreaSpecified)
                RemoveFields(rom.GetFieldValue<DataArea?>(Rom.DataAreaKey)!, [], itemFieldNames);

            if (rom.PartSpecified)
                RemoveFields(rom.GetFieldValue<Part?>(Rom.PartKey)! as DatItem, [], itemFieldNames);
        }

        /// <summary>
        /// Remove fields with given values
        /// </summary>
        /// <param name="slot">Slot to remove fields from</param>
        private void RemoveFields(Slot slot, Dictionary<string, List<string>> itemFieldNames)
        {
            if (slot.SlotOptionsSpecified)
            {
                foreach (SlotOption subSlotOption in slot.GetFieldValue<SlotOption[]?>(Models.Metadata.Slot.SlotOptionKey)!)
                {
                    RemoveFields(subSlotOption, [], itemFieldNames);
                }
            }
        }

        #endregion

        #region Writing

        /// <summary>
        /// Create and open an output file for writing direct from a dictionary
        /// </summary>
        /// <param name="outfile">Name of the file to write to</param>
        /// <param name="ignoreblanks">True if blank roms should be skipped on output, false otherwise (default)</param>
        /// <param name="throwOnError">True if the error that is thrown should be thrown back to the caller, false otherwise</param>
        /// <returns>True if the DAT was written correctly, false otherwise</returns>
        public abstract bool WriteToFile(string outfile, bool ignoreblanks = false, bool throwOnError = false);

        /// <summary>
        /// Create a prefix or postfix from inputs
        /// </summary>
        /// <param name="item">DatItem to create a prefix/postfix for</param>
        /// <param name="prefix">True for prefix, false for postfix</param>
        /// <returns>Sanitized string representing the postfix or prefix</returns>
        protected string CreatePrefixPostfix(DatItem item, bool prefix)
        {
            // Initialize strings
            string fix,
                game = item.GetFieldValue<Machine>(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey) ?? string.Empty,
                name = item.GetName() ?? item.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>().AsStringValue() ?? string.Empty,
                crc = string.Empty,
                md5 = string.Empty,
                sha1 = string.Empty,
                sha256 = string.Empty,
                sha384 = string.Empty,
                sha512 = string.Empty,
                size = string.Empty,
                spamsum = string.Empty;

            // If we have a prefix
            if (prefix)
                fix = Header.GetStringFieldValue(DatHeader.PrefixKey) + (Header.GetBoolFieldValue(DatHeader.QuotesKey) == true ? "\"" : string.Empty);

            // If we have a postfix
            else
                fix = (Header.GetBoolFieldValue(DatHeader.QuotesKey) == true ? "\"" : string.Empty) + Header.GetStringFieldValue(DatHeader.PostfixKey);

            // Ensure we have the proper values for replacement
            if (item is Disk disk)
            {
                md5 = disk.GetStringFieldValue(Models.Metadata.Disk.MD5Key) ?? string.Empty;
                sha1 = disk.GetStringFieldValue(Models.Metadata.Disk.SHA1Key) ?? string.Empty;
            }
            else if (item is Media media)
            {
                md5 = media.GetStringFieldValue(Models.Metadata.Media.MD5Key) ?? string.Empty;
                sha1 = media.GetStringFieldValue(Models.Metadata.Media.SHA1Key) ?? string.Empty;
                sha256 = media.GetStringFieldValue(Models.Metadata.Media.SHA256Key) ?? string.Empty;
                spamsum = media.GetStringFieldValue(Models.Metadata.Media.SpamSumKey) ?? string.Empty;
            }
            else if (item is Rom rom)
            {
                crc = rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey) ?? string.Empty;
                md5 = rom.GetStringFieldValue(Models.Metadata.Rom.MD5Key) ?? string.Empty;
                sha1 = rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key) ?? string.Empty;
                sha256 = rom.GetStringFieldValue(Models.Metadata.Rom.SHA256Key) ?? string.Empty;
                sha384 = rom.GetStringFieldValue(Models.Metadata.Rom.SHA384Key) ?? string.Empty;
                sha512 = rom.GetStringFieldValue(Models.Metadata.Rom.SHA512Key) ?? string.Empty;
                size = rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey).ToString() ?? string.Empty;
                spamsum = rom.GetStringFieldValue(Models.Metadata.Rom.SpamSumKey) ?? string.Empty;
            }

            // Now do bulk replacement where possible
            var machine = item.GetFieldValue<Machine>(DatItem.MachineKey);
            fix = fix
                .Replace("%game%", game)
                .Replace("%machine%", game)
                .Replace("%name%", name)
                .Replace("%manufacturer%", machine!.GetStringFieldValue(Models.Metadata.Machine.ManufacturerKey) ?? string.Empty)
                .Replace("%publisher%", machine!.GetStringFieldValue(Models.Metadata.Machine.PublisherKey) ?? string.Empty)
                .Replace("%category%", machine!.GetStringFieldValue(Models.Metadata.Machine.CategoryKey) ?? string.Empty)
                .Replace("%crc%", crc)
                .Replace("%md5%", md5)
                .Replace("%sha1%", sha1)
                .Replace("%sha256%", sha256)
                .Replace("%sha384%", sha384)
                .Replace("%sha512%", sha512)
                .Replace("%size%", size)
                .Replace("%spamsum%", spamsum);

            return fix;
        }

        /// <summary>
        /// Process an item and correctly set the item name
        /// </summary>
        /// <param name="item">DatItem to update</param>
        /// <param name="forceRemoveQuotes">True if the Quotes flag should be ignored, false otherwise</param>
        /// <param name="forceRomName">True if the UseRomName should be always on (default), false otherwise</param>
        protected void ProcessItemName(DatItem item, bool forceRemoveQuotes, bool forceRomName = true)
        {
            // Backup relevant values and set new ones accordingly
            bool? quotesBackup = Header.GetBoolFieldValue(DatHeader.QuotesKey);
            bool? useRomNameBackup = Header.GetBoolFieldValue(DatHeader.UseRomNameKey);
            if (forceRemoveQuotes)
                Header.SetFieldValue<bool>(DatHeader.QuotesKey, false);
            if (forceRomName)
                Header.SetFieldValue<bool>(DatHeader.UseRomNameKey, true);

            // Get the name to update
            string? name = (Header.GetBoolFieldValue(DatHeader.UseRomNameKey) == true
                ? item.GetName()
                : item.GetFieldValue<Machine>(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)) ?? string.Empty;

            // Create the proper Prefix and Postfix
            string pre = CreatePrefixPostfix(item, true);
            string post = CreatePrefixPostfix(item, false);

            // If we're in Depot mode, take care of that instead
            if (Header.GetFieldValue<DepotInformation?>(DatHeader.OutputDepotKey)?.IsActive == true)
            {
                if (item is Disk disk)
                {
                    // We can only write out if there's a SHA-1
                    if (!string.IsNullOrEmpty(disk.GetStringFieldValue(Models.Metadata.Disk.SHA1Key)))
                    {
                        name = Utilities.GetDepotPath(disk.GetStringFieldValue(Models.Metadata.Disk.SHA1Key), Header.GetFieldValue<DepotInformation?>(DatHeader.OutputDepotKey)!.Depth)?.Replace('\\', '/');
                        item.SetName($"{pre}{name}{post}");
                    }
                }
                else if (item is Media media)
                {
                    // We can only write out if there's a SHA-1
                    if (!string.IsNullOrEmpty(media.GetStringFieldValue(Models.Metadata.Media.SHA1Key)))
                    {
                        name = Utilities.GetDepotPath(media.GetStringFieldValue(Models.Metadata.Media.SHA1Key), Header.GetFieldValue<DepotInformation?>(DatHeader.OutputDepotKey)!.Depth)?.Replace('\\', '/');
                        item.SetName($"{pre}{name}{post}");
                    }
                }
                else if (item is Rom rom)
                {
                    // We can only write out if there's a SHA-1
                    if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key)))
                    {
                        name = Utilities.GetDepotPath(rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key), Header.GetFieldValue<DepotInformation?>(DatHeader.OutputDepotKey)!.Depth)?.Replace('\\', '/');
                        item.SetName($"{pre}{name}{post}");
                    }
                }

                return;
            }

            if (!string.IsNullOrEmpty(Header.GetStringFieldValue(DatHeader.ReplaceExtensionKey)) || Header.GetBoolFieldValue(DatHeader.RemoveExtensionKey) == true)
            {
                if (Header.GetBoolFieldValue(DatHeader.RemoveExtensionKey) == true)
                    Header.SetFieldValue<string?>(DatHeader.ReplaceExtensionKey, string.Empty);

                string? dir = Path.GetDirectoryName(name);
                if (dir != null)
                {
                    dir = dir.TrimStart(Path.DirectorySeparatorChar);
                    name = Path.Combine(dir, Path.GetFileNameWithoutExtension(name) + Header.GetStringFieldValue(DatHeader.ReplaceExtensionKey));
                }
            }

            if (!string.IsNullOrEmpty(Header.GetStringFieldValue(DatHeader.AddExtensionKey)))
                name += Header.GetStringFieldValue(DatHeader.AddExtensionKey);

            if (Header.GetBoolFieldValue(DatHeader.UseRomNameKey) == true && Header.GetBoolFieldValue(DatHeader.GameNameKey) == true)
                name = Path.Combine(item.GetFieldValue<Machine>(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey) ?? string.Empty, name);

            // Now assign back the formatted name
            name = $"{pre}{name}{post}";
            if (Header.GetBoolFieldValue(DatHeader.UseRomNameKey) == true)
                item.SetName(name);
            else if (item.GetFieldValue<Machine>(DatItem.MachineKey) != null)
                item.GetFieldValue<Machine>(DatItem.MachineKey)!.SetFieldValue<string?>(Models.Metadata.Machine.NameKey, name);

            // Restore all relevant values
            if (forceRemoveQuotes)
                Header.SetFieldValue<bool?>(DatHeader.QuotesKey, quotesBackup);

            if (forceRomName)
                Header.SetFieldValue<bool?>(DatHeader.UseRomNameKey, useRomNameBackup);
        }

        /// <summary>
        /// Process any DatItems that are "null", usually created from directory population
        /// </summary>
        /// <param name="datItem">DatItem to check for "null" status</param>
        /// <returns>Cleaned DatItem</returns>
        protected DatItem ProcessNullifiedItem(DatItem datItem)
        {
            // If we don't have a Rom, we can ignore it
            if (datItem is not Rom rom)
                return datItem;

            // If the Rom has "null" characteristics, ensure all fields
            if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) == null && rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey) == "null")
            {
                logger.Verbose($"Empty folder found: {datItem.GetFieldValue<Machine>(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)}");

                rom.SetName(rom.GetName() == "null" ? "-" : rom.GetName());
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SizeKey, Constants.SizeZero.ToString());
                rom.SetFieldValue<string?>(Models.Metadata.Rom.CRCKey, rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey) == "null" ? Constants.CRCZero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.MD5Key, rom.GetStringFieldValue(Models.Metadata.Rom.MD5Key) == "null" ? Constants.MD5Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA1Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key) == "null" ? Constants.SHA1Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA256Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA256Key) == "null" ? Constants.SHA256Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA384Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA384Key) == "null" ? Constants.SHA384Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA512Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA512Key) == "null" ? Constants.SHA512Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SpamSumKey, rom.GetStringFieldValue(Models.Metadata.Rom.SpamSumKey) == "null" ? Constants.SpamSumZero : null);
            }

            return rom;
        }

        /// <summary>
        /// Process any DatItems that are "null", usually created from directory population
        /// </summary>
        /// <param name="datItem">DatItem to check for "null" status</param>
        /// <returns>Cleaned DatItem</returns>
        protected (long, DatItem) ProcessNullifiedItem((long, DatItem) datItem)
        {
            // If we don't have a Rom, we can ignore it
            if (datItem.Item2 is not Rom rom)
                return datItem;

            // If the Rom has "null" characteristics, ensure all fields
            if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) == null && rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey) == "null")
            {
                logger.Verbose($"Empty folder found: {datItem.Item2.GetFieldValue<Machine>(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)}");

                rom.SetName(rom.GetName() == "null" ? "-" : rom.GetName());
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SizeKey, Constants.SizeZero.ToString());
                rom.SetFieldValue<string?>(Models.Metadata.Rom.CRCKey, rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey) == "null" ? Constants.CRCZero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.MD5Key, rom.GetStringFieldValue(Models.Metadata.Rom.MD5Key) == "null" ? Constants.MD5Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA1Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key) == "null" ? Constants.SHA1Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA256Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA256Key) == "null" ? Constants.SHA256Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA384Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA384Key) == "null" ? Constants.SHA384Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SHA512Key, rom.GetStringFieldValue(Models.Metadata.Rom.SHA512Key) == "null" ? Constants.SHA512Zero : null);
                rom.SetFieldValue<string?>(Models.Metadata.Rom.SpamSumKey, rom.GetStringFieldValue(Models.Metadata.Rom.SpamSumKey) == "null" ? Constants.SpamSumZero : null);
            }

            return (datItem.Item1, rom);
        }

        /// <summary>
        /// Get supported types for write
        /// </summary>
        /// <returns>List of supported types for writing</returns>
        protected virtual ItemType[] GetSupportedTypes()
        {
            return Enum.GetValues(typeof(ItemType)) as ItemType[] ?? [];
        }

        /// <summary>
        /// Return list of required fields missing from a DatItem
        /// </summary>
        /// <returns>List of missing required fields, null or empty if none were found</returns>
        protected virtual List<string>? GetMissingRequiredFields(DatItem datItem) => null;

        /// <summary>
        /// Get if a machine contains any writable items
        /// </summary>
        /// <param name="datItems">DatItems to check</param>
        /// <returns>True if the machine contains at least one writable item, false otherwise</returns>
        /// <remarks>Empty machines are kept with this</remarks>
        protected bool ContainsWritable(ConcurrentList<DatItem> datItems)
        {
            // Empty machines are considered writable
            if (datItems == null || datItems.Count == 0)
                return true;

            foreach (DatItem datItem in datItems)
            {
                if (GetSupportedTypes().Contains(datItem.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>()))
                    return true;
            }

            return false;
        }

        /// <summary>
        /// Get if a machine contains any writable items
        /// </summary>
        /// <param name="datItems">DatItems to check</param>
        /// <returns>True if the machine contains at least one writable item, false otherwise</returns>
        /// <remarks>Empty machines are kept with this</remarks>
        protected bool ContainsWritable((long, DatItem)[]? datItems)
        {
            // Empty machines are considered writable
            if (datItems == null || datItems.Length == 0)
                return true;

            foreach ((long, DatItem) datItem in datItems)
            {
                if (GetSupportedTypes().Contains(datItem.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>()))
                    return true;
            }

            return false;
        }

        /// <summary>
        /// Get if an item should be ignored on write
        /// </summary>
        /// <param name="datItem">DatItem to check</param>
        /// <param name="ignoreBlanks">True if blank roms should be skipped on output, false otherwise</param>
        /// <returns>True if the item should be skipped on write, false otherwise</returns>
        protected bool ShouldIgnore(DatItem? datItem, bool ignoreBlanks)
        {
            // If this is invoked with a null DatItem, we ignore
            if (datItem == null)
            {
                logger?.Verbose($"Item was skipped because it was null");
                return true;
            }

            // If the item is supposed to be removed, we ignore
            if (datItem.GetBoolFieldValue(DatItem.RemoveKey) == true)
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                logger?.Verbose($"Item '{itemString}' was skipped because it was marked for removal");
                return true;
            }

            // If we have the Blank dat item, we ignore
            if (datItem is Blank)
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                logger?.Verbose($"Item '{itemString}' was skipped because it was of type 'Blank'");
                return true;
            }

            // If we're ignoring blanks and we have a Rom
            if (ignoreBlanks && datItem is Rom rom)
            {
                // If we have a 0-size or blank rom, then we ignore
                long? size = rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey);
                if (size == 0 || size == null)
                {
                    string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                    logger?.Verbose($"Item '{itemString}' was skipped because it had an invalid size");
                    return true;
                }
            }

            // If we have an item type not in the list of supported values
            if (!GetSupportedTypes().Contains(datItem.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>()))
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                logger?.Verbose($"Item '{itemString}' was skipped because it was not supported in {Header?.GetFieldValue<DatFormat>(DatHeader.DatFormatKey)}");
                return true;
            }

            // If we have an item with missing required fields
            List<string>? missingFields = GetMissingRequiredFields(datItem);
            if (missingFields != null && missingFields.Count != 0)
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
#if NET20 || NET35
                logger?.Verbose($"Item '{itemString}' was skipped because it was missing required fields for {Header?.GetFieldValue<DatFormat>(DatHeader.DatFormatKey)}: {string.Join(", ", [.. missingFields])}");
#else
                logger?.Verbose($"Item '{itemString}' was skipped because it was missing required fields for {Header?.GetFieldValue<DatFormat>(DatHeader.DatFormatKey)}: {string.Join(", ", missingFields)}");
#endif
                return true;
            }

            return false;
        }

        /// <summary>
        /// Get if an item should be ignored on write
        /// </summary>
        /// <param name="datItem">DatItem to check</param>
        /// <param name="ignoreBlanks">True if blank roms should be skipped on output, false otherwise</param>
        /// <returns>True if the item should be skipped on write, false otherwise</returns>
        protected bool ShouldIgnore((long, DatItem?) datItem, bool ignoreBlanks)
        {
            // If this is invoked with a null DatItem, we ignore
            if (datItem.Item1 < 0 || datItem.Item2 == null)
            {
                logger?.Verbose($"Item was skipped because it was null");
                return true;
            }

            // If the item is supposed to be removed, we ignore
            if (datItem.Item2.GetBoolFieldValue(DatItem.RemoveKey) == true)
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                logger?.Verbose($"Item '{itemString}' was skipped because it was marked for removal");
                return true;
            }

            // If we have the Blank dat item, we ignore
            if (datItem.Item2 is Blank)
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                logger?.Verbose($"Item '{itemString}' was skipped because it was of type 'Blank'");
                return true;
            }

            // If we're ignoring blanks and we have a Rom
            if (ignoreBlanks && datItem.Item2 is Rom rom)
            {
                // If we have a 0-size or blank rom, then we ignore
                long? size = rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey);
                if (size == 0 || size == null)
                {
                    string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                    logger?.Verbose($"Item '{itemString}' was skipped because it had an invalid size");
                    return true;
                }
            }

            // If we have an item type not in the list of supported values
            if (!GetSupportedTypes().Contains(datItem.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>()))
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
                logger?.Verbose($"Item '{itemString}' was skipped because it was not supported in {Header?.GetFieldValue<DatFormat>(DatHeader.DatFormatKey)}");
                return true;
            }

            // If we have an item with missing required fields
            List<string>? missingFields = GetMissingRequiredFields(datItem.Item2);
            if (missingFields != null && missingFields.Count != 0)
            {
                string itemString = JsonConvert.SerializeObject(datItem, Formatting.None);
#if NET20 || NET35
                logger?.Verbose($"Item '{itemString}' was skipped because it was missing required fields for {Header?.GetFieldValue<DatFormat>(DatHeader.DatFormatKey)}: {string.Join(", ", [.. missingFields])}");
#else
                logger?.Verbose($"Item '{itemString}' was skipped because it was missing required fields for {Header?.GetFieldValue<DatFormat>(DatHeader.DatFormatKey)}: {string.Join(", ", missingFields)}");
#endif
                return true;
            }

            return false;
        }

        #endregion
    }

    /// <summary>
    /// Represents a DAT that can be serialized
    /// </summary>
    /// <typeparam name="T">Base internal model for the DAT type</typeparam>
    /// <typeparam name="U">IFileSerializer type to use for conversion</typeparam>
    /// <typeparam name="V">IModelSerializer for cross-model serialization</typeparam>
    public abstract class SerializableDatFile<T, U, V> : DatFile
        where U : IFileSerializer<T>
        where V : IModelSerializer<T, Models.Metadata.MetadataFile>
    {
        /// <inheritdoc/>
        protected SerializableDatFile(DatFile? datFile) : base(datFile) { }

        /// <inheritdoc/>
        public override void ParseFile(string filename, int indexId, bool keep, bool statsOnly = false, bool throwOnError = false)
        {
            try
            {
                // Deserialize the input file in two steps
                var specificFormat = Activator.CreateInstance<U>().Deserialize(filename);
                var internalFormat = Activator.CreateInstance<V>().Serialize(specificFormat);

                // Convert to the internal format
                ConvertMetadata(internalFormat, filename, indexId, keep, statsOnly);
            }
            catch (Exception ex) when (!throwOnError)
            {
                string message = $"'{filename}' - An error occurred during parsing";
                logger.Error(ex, message);
            }
        }

        /// <inheritdoc/>
        public override bool WriteToFile(string outfile, bool ignoreblanks = false, bool throwOnError = false)
        {
            try
            {
                logger.User($"Writing to '{outfile}'...");

                // Serialize the input file in two steps
                var internalFormat = ConvertMetadata(ignoreblanks);
                var specificFormat = Activator.CreateInstance<V>().Deserialize(internalFormat);
                if (!Activator.CreateInstance<U>().Serialize(specificFormat, outfile))
                {
                    logger.Warning($"File '{outfile}' could not be written! See the log for more details.");
                    return false;
                }
            }
            catch (Exception ex) when (!throwOnError)
            {
                logger.Error(ex);
                return false;
            }

            logger.User($"'{outfile}' written!{Environment.NewLine}");
            return true;
        }
    }
}