using System; using System.IO; using System.Linq; using System.Xml.Serialization; using SabreTools.Core; using SabreTools.Core.Tools; using SabreTools.DatItems.Formats; using SabreTools.FileTypes; using SabreTools.Logging; using NaturalSort; using Newtonsoft.Json; using Newtonsoft.Json.Converters; namespace SabreTools.DatItems { /// /// Base class for all items included in a set /// /// /// TODO: Can this be made into a `record` type for easier comparison? /// [JsonObject("datitem"), XmlRoot("datitem")] [XmlInclude(typeof(Adjuster))] [XmlInclude(typeof(Analog))] [XmlInclude(typeof(Archive))] [XmlInclude(typeof(BiosSet))] [XmlInclude(typeof(Blank))] [XmlInclude(typeof(Chip))] [XmlInclude(typeof(Condition))] [XmlInclude(typeof(Configuration))] [XmlInclude(typeof(Control))] [XmlInclude(typeof(DataArea))] [XmlInclude(typeof(Device))] [XmlInclude(typeof(DeviceReference))] [XmlInclude(typeof(DipSwitch))] [XmlInclude(typeof(Disk))] [XmlInclude(typeof(DiskArea))] [XmlInclude(typeof(Display))] [XmlInclude(typeof(Driver))] [XmlInclude(typeof(Extension))] [XmlInclude(typeof(Feature))] [XmlInclude(typeof(Info))] [XmlInclude(typeof(Input))] [XmlInclude(typeof(Instance))] [XmlInclude(typeof(Location))] [XmlInclude(typeof(Media))] [XmlInclude(typeof(Part))] [XmlInclude(typeof(PartFeature))] [XmlInclude(typeof(Port))] [XmlInclude(typeof(RamOption))] [XmlInclude(typeof(Release))] [XmlInclude(typeof(Rom))] [XmlInclude(typeof(Sample))] [XmlInclude(typeof(Setting))] [XmlInclude(typeof(SharedFeature))] [XmlInclude(typeof(Slot))] [XmlInclude(typeof(SlotOption))] [XmlInclude(typeof(SoftwareList))] [XmlInclude(typeof(Sound))] public abstract class DatItem : IEquatable, IComparable, ICloneable { #region Fields #region Common Fields /// /// Item type for outputting /// [JsonProperty("itemtype")] [JsonConverter(typeof(StringEnumConverter))] [XmlElement("itemtype")] public ItemType ItemType { get; set; } /// /// Duplicate type when compared to another item /// [JsonIgnore, XmlIgnore] public DupeType DupeType { get; set; } #endregion #region Machine Fields /// /// Machine values /// [JsonIgnore, XmlIgnore] public Machine Machine { get; set; } = new Machine(); #endregion #region Metadata information /// /// Source information /// [JsonIgnore, XmlIgnore] public Source Source { get; set; } = new Source(); /// /// Flag if item should be removed /// [JsonIgnore, XmlIgnore] public bool Remove { get; set; } #endregion // Metadata information #endregion #region Logging /// /// Logging object /// [JsonIgnore, XmlIgnore] protected Logger logger; /// /// Static logger for static methods /// [JsonIgnore, XmlIgnore] protected static Logger staticLogger = new(); #endregion #region Instance Methods #region Accessors /// /// Gets the name to use for a DatItem /// /// Name if available, null otherwise public virtual string GetName() => null; /// /// Sets the name to use for a DatItem /// /// Name to set for the item public virtual void SetName(string name) { } #endregion #region Constructors /// /// Constructor /// public DatItem() { logger = new Logger(this); } /// /// Create a specific type of DatItem to be used based on an ItemType /// /// Type of the DatItem to be created /// DatItem of the specific internal type that corresponds to the inputs public static DatItem Create(ItemType? itemType) { return itemType switch { ItemType.Adjuster => new Adjuster(), ItemType.Analog => new Analog(), ItemType.Archive => new Archive(), ItemType.BiosSet => new BiosSet(), ItemType.Blank => new Blank(), ItemType.Chip => new Chip(), ItemType.Condition => new Condition(), ItemType.Configuration => new Configuration(), ItemType.Device => new Device(), ItemType.DeviceReference => new DeviceReference(), ItemType.DipSwitch => new DipSwitch(), ItemType.Disk => new Disk(), ItemType.Display => new Display(), ItemType.Driver => new Driver(), ItemType.Extension => new Extension(), ItemType.Feature => new Feature(), ItemType.File => new Formats.File(), ItemType.Info => new Info(), ItemType.Instance => new Instance(), ItemType.Location => new Location(), ItemType.Media => new Media(), ItemType.PartFeature => new PartFeature(), ItemType.Port => new Port(), ItemType.RamOption => new RamOption(), ItemType.Release => new Release(), ItemType.ReleaseDetails => new ReleaseDetails(), ItemType.Rom => new Rom(), ItemType.Sample => new Sample(), ItemType.Serials => new Serials(), ItemType.SharedFeature => new SharedFeature(), ItemType.Slot => new Slot(), ItemType.SlotOption => new SlotOption(), ItemType.SoftwareList => new SoftwareList(), ItemType.Sound => new Sound(), ItemType.SourceDetails => new SourceDetails(), _ => new Rom(), }; } /// /// Create a specific type of DatItem to be used based on a BaseFile /// /// BaseFile containing information to be created /// DatItem of the specific internal type that corresponds to the inputs public static DatItem Create(BaseFile baseFile) { return baseFile.Type switch { FileType.AaruFormat => new Media(baseFile), FileType.CHD => new Disk(baseFile), FileType.GZipArchive => new Rom(baseFile), FileType.LRZipArchive => new Rom(baseFile), FileType.LZ4Archive => new Rom(baseFile), FileType.None => new Rom(baseFile), FileType.RarArchive => new Rom(baseFile), FileType.SevenZipArchive => new Rom(baseFile), FileType.TapeArchive => new Rom(baseFile), FileType.XZArchive => new Rom(baseFile), FileType.ZipArchive => new Rom(baseFile), FileType.ZPAQArchive => new Rom(baseFile), FileType.ZstdArchive => new Rom(baseFile), _ => null, }; } #endregion #region Cloning Methods /// /// Clone the DatItem /// /// Clone of the DatItem public abstract object Clone(); /// /// Copy all machine information over in one shot /// /// Existing item to copy information from public void CopyMachineInformation(DatItem item) { Machine = (Machine)item.Machine.Clone(); } /// /// Copy all machine information over in one shot /// /// Existing machine to copy information from public void CopyMachineInformation(Machine machine) { Machine = (Machine)machine.Clone(); } #endregion #region Comparision Methods /// public int CompareTo(DatItem other) { try { if (GetName() == other.GetName()) return Equals(other) ? 0 : 1; return string.Compare(GetName(), other.GetName()); } catch { return 1; } } /// /// Determine if an item is a duplicate using partial matching logic /// /// DatItem to use as a baseline /// True if the items are duplicates, false otherwise public abstract bool Equals(DatItem other); /// /// Return the duplicate status of two items /// /// DatItem to check against /// The DupeType corresponding to the relationship between the two public DupeType GetDuplicateStatus(DatItem lastItem) { DupeType output = 0x00; // If we don't have a duplicate at all, return none if (!Equals(lastItem)) return output; // If the duplicate is external already or should be, set it if (lastItem.DupeType.HasFlag(DupeType.External) || lastItem.Source.Index != Source.Index) { if (lastItem.Machine.Name == Machine.Name && lastItem.GetName() == GetName()) output = DupeType.External | DupeType.All; else output = DupeType.External | DupeType.Hash; } // Otherwise, it's considered an internal dupe else { if (lastItem.Machine.Name == Machine.Name && lastItem.GetName() == GetName()) output = DupeType.Internal | DupeType.All; else output = DupeType.Internal | DupeType.Hash; } return output; } #endregion #region Filtering /// /// Clean a CRC32 string and pad to the correct size /// /// Hash string to sanitize /// Cleaned string protected static string CleanCRC32(string hash) { return CleanHashData(hash, Constants.CRCLength); } /// /// Clean a MD5 string and pad to the correct size /// /// Hash string to sanitize /// Cleaned string protected static string CleanMD5(string hash) { return CleanHashData(hash, Constants.MD5Length); } /// /// Clean a SHA1 string and pad to the correct size /// /// Hash string to sanitize /// Cleaned string protected static string CleanSHA1(string hash) { return CleanHashData(hash, Constants.SHA1Length); } /// /// Clean a SHA256 string and pad to the correct size /// /// Hash string to sanitize /// Cleaned string protected static string CleanSHA256(string hash) { return CleanHashData(hash, Constants.SHA256Length); } /// /// Clean a SHA384 string and pad to the correct size /// /// Hash string to sanitize /// Cleaned string protected static string CleanSHA384(string hash) { return CleanHashData(hash, Constants.SHA384Length); } /// /// Clean a SHA512 string and pad to the correct size /// /// Hash string to sanitize /// Cleaned string protected static string CleanSHA512(string hash) { return CleanHashData(hash, Constants.SHA512Length); } /// /// Clean a hash string and pad to the correct size /// /// Hash string to sanitize /// Amount of characters to pad to /// Cleaned string private static string CleanHashData(string hash, int padding) { // If we have a known blank hash, return blank if (string.IsNullOrWhiteSpace(hash) || hash == "-" || hash == "_") return string.Empty; // Check to see if it's a "hex" hash hash = hash.Trim().Replace("0x", string.Empty); // If we have a blank hash now, return blank if (string.IsNullOrWhiteSpace(hash)) return string.Empty; // If the hash shorter than the required length, pad it if (hash.Length < padding) hash = hash.PadLeft(padding, '0'); // If the hash is longer than the required length, it's invalid else if (hash.Length > padding) return string.Empty; // Now normalize the hash hash = hash.ToLowerInvariant(); // Otherwise, make sure that every character is a proper match for (int i = 0; i < hash.Length; i++) { if ((hash[i] < '0' || hash[i] > '9') && (hash[i] < 'a' || hash[i] > 'f')) { hash = string.Empty; break; } } return hash; } #endregion #region Sorting and Merging /// /// Get the dictionary key that should be used for a given item and bucketing type /// /// ItemKey value representing what key to get /// True if the key should be lowercased (default), false otherwise /// True if games should only be compared on game and file name, false if system and source are counted /// String representing the key to be used for the DatItem public virtual string GetKey(ItemKey bucketedBy, bool lower = true, bool norename = true) { // Set the output key as the default blank string string key = string.Empty; // Now determine what the key should be based on the bucketedBy value switch (bucketedBy) { case ItemKey.CRC: key = Constants.CRCZero; break; case ItemKey.Machine: key = (norename ? string.Empty : Source.Index.ToString().PadLeft(10, '0') + "-") + (string.IsNullOrWhiteSpace(Machine.Name) ? "Default" : Machine.Name); if (lower) key = key.ToLowerInvariant(); key ??= "null"; break; case ItemKey.MD5: key = Constants.MD5Zero; break; case ItemKey.SHA1: key = Constants.SHA1Zero; break; case ItemKey.SHA256: key = Constants.SHA256Zero; break; case ItemKey.SHA384: key = Constants.SHA384Zero; break; case ItemKey.SHA512: key = Constants.SHA512Zero; break; case ItemKey.SpamSum: key = Constants.SpamSumZero; break; } // Double and triple check the key for corner cases key ??= string.Empty; return key; } #endregion #endregion // Instance Methods #region Static Methods #region Sorting and Merging /// /// Determine if two hashes are equal for the purposes of merging /// /// First hash to compare /// Second hash to compare /// True if either is empty OR the hashes exactly match, false otherwise public static bool ConditionalHashEquals(byte[] firstHash, byte[] secondHash) { // If either hash is empty, we say they're equal for merging if (firstHash.IsNullOrEmpty() || secondHash.IsNullOrEmpty()) return true; // If they're different sizes, they can't match if (firstHash.Length != secondHash.Length) return false; // Otherwise, they need to match exactly return Enumerable.SequenceEqual(firstHash, secondHash); } /// /// Merge an arbitrary set of ROMs based on the supplied information /// /// List of File objects representing the roms to be merged /// A List of DatItem objects representing the merged roms public static ConcurrentList Merge(ConcurrentList infiles) { // Check for null or blank roms first if (infiles == null || infiles.Count == 0) return new ConcurrentList(); // Create output list ConcurrentList outfiles = new(); // Then deduplicate them by checking to see if data matches previous saved roms int nodumpCount = 0; for (int f = 0; f < infiles.Count; f++) { DatItem file = infiles[f]; // If we somehow have a null item, skip if (file == null) continue; // If we don't have a Disk, File, Media, or Rom, we skip checking for duplicates if (file.ItemType != ItemType.Disk && file.ItemType != ItemType.File && file.ItemType != ItemType.Media && file.ItemType != ItemType.Rom) { continue; } // If it's a nodump, add and skip if (file.ItemType == ItemType.Rom && (file as Rom).ItemStatus == ItemStatus.Nodump) { outfiles.Add(file); nodumpCount++; continue; } else if (file.ItemType == ItemType.Disk && (file as Disk).ItemStatus == ItemStatus.Nodump) { outfiles.Add(file); nodumpCount++; continue; } // If it's the first non-nodump rom in the list, don't touch it else if (outfiles.Count == 0 || outfiles.Count == nodumpCount) { outfiles.Add(file); continue; } // Check if the rom is a duplicate DupeType dupetype = 0x00; DatItem saveditem = new Blank(); int pos = -1; for (int i = 0; i < outfiles.Count; i++) { DatItem lastrom = outfiles[i]; // Get the duplicate status dupetype = file.GetDuplicateStatus(lastrom); // If it's a duplicate, skip adding it to the output but add any missing information if (dupetype != 0x00) { saveditem = lastrom; pos = i; // Disks, Media, and Roms have more information to fill if (file.ItemType == ItemType.Disk) (saveditem as Disk).FillMissingInformation(file as Disk); else if (file.ItemType == ItemType.File) (saveditem as Formats.File).FillMissingInformation(file as Formats.File); else if (file.ItemType == ItemType.Media) (saveditem as Media).FillMissingInformation(file as Media); else if (file.ItemType == ItemType.Rom) (saveditem as Rom).FillMissingInformation(file as Rom); saveditem.DupeType = dupetype; // If the current system has a lower ID than the previous, set the system accordingly if (file.Source.Index < saveditem.Source.Index) { saveditem.Source = file.Source.Clone() as Source; saveditem.CopyMachineInformation(file); saveditem.SetName(file.GetName()); } // If the current machine is a child of the new machine, use the new machine instead if (saveditem.Machine.CloneOf == file.Machine.Name || saveditem.Machine.RomOf == file.Machine.Name) { saveditem.CopyMachineInformation(file); saveditem.SetName(file.GetName()); } break; } } // If no duplicate is found, add it to the list if (dupetype == 0x00) { outfiles.Add(file); } // Otherwise, if a new rom information is found, add that else { outfiles.RemoveAt(pos); outfiles.Insert(pos, saveditem); } } // Then return the result return outfiles; } /// /// Resolve name duplicates in an arbitrary set of ROMs based on the supplied information /// /// List of File objects representing the roms to be merged /// A List of DatItem objects representing the renamed roms public static ConcurrentList ResolveNames(ConcurrentList infiles) { // Create the output list ConcurrentList output = new(); // First we want to make sure the list is in alphabetical order Sort(ref infiles, true); // Now we want to loop through and check names DatItem lastItem = null; string lastrenamed = null; int lastid = 0; for (int i = 0; i < infiles.Count; i++) { DatItem datItem = infiles[i]; // If we have the first item, we automatically add it if (lastItem == null) { output.Add(datItem); lastItem = datItem; continue; } // Get the last item name, if applicable string lastItemName = lastItem.GetName() ?? lastItem.ItemType.ToString(); // Get the current item name, if applicable string datItemName = datItem.GetName() ?? datItem.ItemType.ToString(); // If the current item exactly matches the last item, then we don't add it if (datItem.GetDuplicateStatus(lastItem).HasFlag(DupeType.All)) { staticLogger.Verbose($"Exact duplicate found for '{datItemName}'"); continue; } // If the current name matches the previous name, rename the current item else if (datItemName == lastItemName) { staticLogger.Verbose($"Name duplicate found for '{datItemName}'"); if (datItem.ItemType == ItemType.Disk || datItem.ItemType == ItemType.File || datItem.ItemType == ItemType.Media || datItem.ItemType == ItemType.Rom) { datItemName += GetDuplicateSuffix(datItem); lastrenamed ??= datItemName; } // If we have a conflict with the last renamed item, do the right thing if (datItemName == lastrenamed) { lastrenamed = datItemName; datItemName += (lastid == 0 ? string.Empty : "_" + lastid); lastid++; } // If we have no conflict, then we want to reset the lastrenamed and id else { lastrenamed = null; lastid = 0; } // Set the item name back to the datItem datItem.SetName(datItemName); output.Add(datItem); } // Otherwise, we say that we have a valid named file else { output.Add(datItem); lastItem = datItem; lastrenamed = null; lastid = 0; } } // One last sort to make sure this is ordered Sort(ref output, true); return output; } /// /// Get duplicate suffix based on the item type /// private static string GetDuplicateSuffix(DatItem datItem) { if (datItem.ItemType == ItemType.Disk) return (datItem as Disk).GetDuplicateSuffix(); else if (datItem.ItemType == ItemType.File) return (datItem as Formats.File).GetDuplicateSuffix(); else if (datItem.ItemType == ItemType.Media) return (datItem as Media).GetDuplicateSuffix(); else if (datItem.ItemType == ItemType.Rom) return (datItem as Rom).GetDuplicateSuffix(); return "_1"; } /// /// Sort a list of File objects by SourceID, Game, and Name (in order) /// /// List of File objects representing the roms to be sorted /// True if files are not renamed, false otherwise /// True if it sorted correctly, false otherwise public static bool Sort(ref ConcurrentList roms, bool norename) { roms.Sort(delegate (DatItem x, DatItem y) { try { NaturalComparer nc = new(); // If machine names match, more refinement is needed if (x.Machine.Name == y.Machine.Name) { // If item types match, more refinement is needed if (x.ItemType == y.ItemType) { string xDirectoryName = Path.GetDirectoryName(Utilities.RemovePathUnsafeCharacters(x.GetName() ?? string.Empty)); string yDirectoryName = Path.GetDirectoryName(Utilities.RemovePathUnsafeCharacters(y.GetName() ?? string.Empty)); // If item directory names match, more refinement is needed if (xDirectoryName == yDirectoryName) { string xName = Path.GetFileName(Utilities.RemovePathUnsafeCharacters(x.GetName() ?? string.Empty)); string yName = Path.GetFileName(Utilities.RemovePathUnsafeCharacters(y.GetName() ?? string.Empty)); // If item names match, then compare on machine or source, depending on the flag if (xName == yName) return (norename ? nc.Compare(x.Machine.Name, y.Machine.Name) : x.Source.Index - y.Source.Index); // Otherwise, just sort based on item names return nc.Compare(xName, yName); } // Otherwise, just sort based on directory name return nc.Compare(xDirectoryName, yDirectoryName); } // Otherwise, just sort based on item type return x.ItemType - y.ItemType; } // Otherwise, just sort based on machine name return nc.Compare(x.Machine.Name, y.Machine.Name); } catch { // Absorb the error return 0; } }); return true; } #endregion #endregion // Static Methods } }