diff --git a/SabreTools.DatFiles/ItemDictionaryDB.cs b/SabreTools.DatFiles/ItemDictionaryDB.cs index dc507ae6..5a6116b1 100644 --- a/SabreTools.DatFiles/ItemDictionaryDB.cs +++ b/SabreTools.DatFiles/ItemDictionaryDB.cs @@ -614,7 +614,7 @@ namespace SabreTools.DatFiles /// Item to try to match /// True if the DAT is already sorted accordingly, false otherwise (default) /// List of matched DatItem objects - public ConcurrentList<(long, DatItem)> GetDuplicates(DatItem datItem, bool sorted = false) + public ConcurrentList<(long, DatItem)> GetDuplicates((long, DatItem) datItem, bool sorted = false) { ConcurrentList<(long, DatItem)> output = []; @@ -638,7 +638,7 @@ namespace SabreTools.DatFiles if (other.GetBoolFieldValue(DatItem.RemoveKey) == true) continue; - if (datItem.Equals(other)) + if (datItem.Item2.Equals(other)) { other.SetFieldValue(DatItem.RemoveKey, true); output.Add(other); @@ -660,7 +660,7 @@ namespace SabreTools.DatFiles /// Item to try to match /// True if the DAT is already sorted accordingly, false otherwise (default) /// True if it contains the rom, false otherwise - public bool HasDuplicates(DatItem datItem, bool sorted = false) + public bool HasDuplicates((long, DatItem) datItem, bool sorted = false) { // Check for an empty rom list first if (DatStatistics.TotalCount == 0) @@ -736,8 +736,12 @@ namespace SabreTools.DatFiles long lastIndex = output[i].Item1; DatItem lastrom = output[i].Item2; + // Get the sources associated with the items + var savedSource = _sources[_itemToSourceMapping[savedIndex]]; + var itemSource = _sources[_itemToSourceMapping[itemIndex]]; + // Get the duplicate status - dupetype = datItem.GetDuplicateStatus(lastrom); + dupetype = datItem.GetDuplicateStatus(itemSource, lastrom, savedSource); // If it's a duplicate, skip adding it to the output but add any missing information if (dupetype != 0x00) @@ -763,9 +767,9 @@ namespace SabreTools.DatFiles var itemMachine = _machines[_itemToMachineMapping[itemIndex]]; // If the current system has a lower ID than the previous, set the system accordingly - if (datItem.GetFieldValue(DatItem.SourceKey)?.Index < saveditem.GetFieldValue(DatItem.SourceKey)?.Index) + if (itemSource?.Index < savedSource?.Index) { - datItem.SetFieldValue(DatItem.SourceKey, datItem.GetFieldValue(DatItem.SourceKey)!.Clone() as Source); + _itemToSourceMapping[itemIndex] = _itemToSourceMapping[savedIndex]; _machines[_itemToMachineMapping[savedIndex]] = (itemMachine.Clone() as Machine)!; saveditem.SetName(datItem.GetName()); } @@ -1131,14 +1135,15 @@ namespace SabreTools.DatFiles /// Item to try to match /// True if the DAT is already sorted accordingly, false otherwise (default) /// Key to try to use - private string SortAndGetKey(DatItem datItem, bool sorted = false) + private string SortAndGetKey((long, DatItem) datItem, bool sorted = false) { // If we're not already sorted, take care of it if (!sorted) BucketBy(GetBestAvailable(), DedupeType.None); // Now that we have the sorted type, we get the proper key - return datItem.GetKey(_bucketedBy); + var source = GetSourceForItem(datItem.Item1); + return datItem.Item2.GetKey(_bucketedBy, source.Item2); } #endregion diff --git a/SabreTools.DatItems/DatItem.cs b/SabreTools.DatItems/DatItem.cs index d216c288..3660cba2 100644 --- a/SabreTools.DatItems/DatItem.cs +++ b/SabreTools.DatItems/DatItem.cs @@ -264,6 +264,52 @@ namespace SabreTools.DatItems return output; } + /// + /// Return the duplicate status of two items + /// + /// Source associated with this item + /// DatItem to check against + /// Source associated with the last item + /// The DupeType corresponding to the relationship between the two + public DupeType GetDuplicateStatus(Source? source, DatItem? lastItem, Source? lastSource) + { + DupeType output = 0x00; + + // If we don't have a duplicate at all, return none + if (!Equals(lastItem)) + return output; + + // If the duplicate is external already or should be, set it +#if NETFRAMEWORK + if ((lastItem.GetFieldValue(DatItem.DupeTypeKey) & DupeType.External) != 0 + || lastSource?.Index != source?.Index) +#else + if (lastItem.GetFieldValue(DatItem.DupeTypeKey).HasFlag(DupeType.External) + || lastSource?.Index != source?.Index) +#endif + { + var currentMachine = GetFieldValue(DatItem.MachineKey); + var lastMachine = lastItem?.GetFieldValue(DatItem.MachineKey); + if (lastMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) == currentMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) && lastItem?.GetName() == GetName()) + output = DupeType.External | DupeType.All; + else + output = DupeType.External | DupeType.Hash; + } + + // Otherwise, it's considered an internal dupe + else + { + var currentMachine = GetFieldValue(DatItem.MachineKey); + var lastMachine = lastItem?.GetFieldValue(DatItem.MachineKey); + if (lastMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) == currentMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) && lastItem?.GetName() == GetName()) + output = DupeType.Internal | DupeType.All; + else + output = DupeType.Internal | DupeType.Hash; + } + + return output; + } + #endregion #region Manipulation @@ -376,6 +422,71 @@ namespace SabreTools.DatItems return key; } + /// + /// Get the dictionary key that should be used for a given item and bucketing type + /// + /// ItemKey value representing what key to get + /// Source associated with the item for renaming + /// True if the key should be lowercased (default), false otherwise + /// True if games should only be compared on game and file name, false if system and source are counted + /// String representing the key to be used for the DatItem + public virtual string GetKey(ItemKey bucketedBy, Source? source, bool lower = true, bool norename = true) + { + // Set the output key as the default blank string + string key = string.Empty; + + // Now determine what the key should be based on the bucketedBy value + switch (bucketedBy) + { + case ItemKey.CRC: + key = Constants.CRCZero; + break; + + case ItemKey.Machine: + key = (norename ? string.Empty + : source?.Index.ToString().PadLeft(10, '0') + + "-") + + (string.IsNullOrEmpty(GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey)) + ? "Default" + : GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!); + if (lower) + key = key.ToLowerInvariant(); + + key ??= "null"; + + break; + + case ItemKey.MD5: + key = Constants.MD5Zero; + break; + + case ItemKey.SHA1: + key = Constants.SHA1Zero; + break; + + case ItemKey.SHA256: + key = Constants.SHA256Zero; + break; + + case ItemKey.SHA384: + key = Constants.SHA384Zero; + break; + + case ItemKey.SHA512: + key = Constants.SHA512Zero; + break; + + case ItemKey.SpamSum: + key = Constants.SpamSumZero; + break; + } + + // Double and triple check the key for corner cases + key ??= string.Empty; + + return key; + } + #endregion #endregion // Instance Methods diff --git a/SabreTools.DatTools/DatFileTool.cs b/SabreTools.DatTools/DatFileTool.cs index 96ceeea5..b949a026 100644 --- a/SabreTools.DatTools/DatFileTool.cs +++ b/SabreTools.DatTools/DatFileTool.cs @@ -313,7 +313,7 @@ namespace SabreTools.DatTools foreach ((long, DatItem) datItem in datItems) { - var dupes = datFile.ItemsDB.GetDuplicates(datItem.Item2, sorted: true); + var dupes = datFile.ItemsDB.GetDuplicates(datItem, sorted: true); if (datItem.Item2.Clone() is not DatItem newDatItem) continue; diff --git a/SabreTools.DatTools/Rebuilder.cs b/SabreTools.DatTools/Rebuilder.cs index d71ba29e..03f4c122 100644 --- a/SabreTools.DatTools/Rebuilder.cs +++ b/SabreTools.DatTools/Rebuilder.cs @@ -579,7 +579,7 @@ namespace SabreTools.DatTools /// True if the DAT should be used as a filter instead of a template, false otherwise /// Output list of duplicate items to rebuild to /// True if the item should be rebuilt, false otherwise - private static bool ShouldRebuildDB(DatFile datFile, DatItem datItem, Stream? stream, bool inverse, out ConcurrentList<(long, DatItem)> dupes) + private static bool ShouldRebuildDB(DatFile datFile, (long, DatItem) datItem, Stream? stream, bool inverse, out ConcurrentList<(long, DatItem)> dupes) { // Find if the file has duplicates in the DAT dupes = datFile.ItemsDB.GetDuplicates(datItem); diff --git a/SabreTools.DatTools/Verification.cs b/SabreTools.DatTools/Verification.cs index f8caff94..9c814a42 100644 --- a/SabreTools.DatTools/Verification.cs +++ b/SabreTools.DatTools/Verification.cs @@ -182,8 +182,8 @@ namespace SabreTools.DatTools continue; // Now we want to remove all duplicates from the DAT - datFile.ItemsDB.GetDuplicates(new Rom(fileinfo)) - .AddRange(datFile.ItemsDB.GetDuplicates(new Disk(fileinfo))); + datFile.ItemsDB.GetDuplicates((-1, new Rom(fileinfo))) + .AddRange(datFile.ItemsDB.GetDuplicates((-1, new Disk(fileinfo)))); } watch.Stop();