Handle most places where source was being set for IDDB

SabreTools.DatFiles/ItemDictionaryDB.cs

@@ -614,7 +614,7 @@ namespace SabreTools.DatFiles
         /// <param name="datItem">Item to try to match</param>
         /// <param name="sorted">True if the DAT is already sorted accordingly, false otherwise (default)</param>
         /// <returns>List of matched DatItem objects</returns>
-        public ConcurrentList<(long, DatItem)> GetDuplicates(DatItem datItem, bool sorted = false)
+        public ConcurrentList<(long, DatItem)> GetDuplicates((long, DatItem) datItem, bool sorted = false)
         {
             ConcurrentList<(long, DatItem)> output = [];
 
@@ -638,7 +638,7 @@ namespace SabreTools.DatFiles
                 if (other.GetBoolFieldValue(DatItem.RemoveKey) == true)
                     continue;
 
-                if (datItem.Equals(other))
+                if (datItem.Item2.Equals(other))
                 {
                     other.SetFieldValue<bool?>(DatItem.RemoveKey, true);
                     output.Add(other);
@@ -660,7 +660,7 @@ namespace SabreTools.DatFiles
         /// <param name="datItem">Item to try to match</param>
         /// <param name="sorted">True if the DAT is already sorted accordingly, false otherwise (default)</param>
         /// <returns>True if it contains the rom, false otherwise</returns>
-        public bool HasDuplicates(DatItem datItem, bool sorted = false)
+        public bool HasDuplicates((long, DatItem) datItem, bool sorted = false)
         {
             // Check for an empty rom list first
             if (DatStatistics.TotalCount == 0)
@@ -736,8 +736,12 @@ namespace SabreTools.DatFiles
                     long lastIndex = output[i].Item1;
                     DatItem lastrom = output[i].Item2;
 
+                    // Get the sources associated with the items
+                    var savedSource = _sources[_itemToSourceMapping[savedIndex]];
+                    var itemSource = _sources[_itemToSourceMapping[itemIndex]];
+
                     // Get the duplicate status
-                    dupetype = datItem.GetDuplicateStatus(lastrom);
+                    dupetype = datItem.GetDuplicateStatus(itemSource, lastrom, savedSource);
 
                     // If it's a duplicate, skip adding it to the output but add any missing information
                     if (dupetype != 0x00)
@@ -763,9 +767,9 @@ namespace SabreTools.DatFiles
                         var itemMachine = _machines[_itemToMachineMapping[itemIndex]];
 
                         // If the current system has a lower ID than the previous, set the system accordingly
-                        if (datItem.GetFieldValue<Source?>(DatItem.SourceKey)?.Index < saveditem.GetFieldValue<Source?>(DatItem.SourceKey)?.Index)
+                        if (itemSource?.Index < savedSource?.Index)
                         {
-                            datItem.SetFieldValue<Source?>(DatItem.SourceKey, datItem.GetFieldValue<Source?>(DatItem.SourceKey)!.Clone() as Source);
+                            _itemToSourceMapping[itemIndex] = _itemToSourceMapping[savedIndex];
                             _machines[_itemToMachineMapping[savedIndex]] = (itemMachine.Clone() as Machine)!;
                             saveditem.SetName(datItem.GetName());
                         }
@@ -1131,14 +1135,15 @@ namespace SabreTools.DatFiles
         /// <param name="datItem">Item to try to match</param>
         /// <param name="sorted">True if the DAT is already sorted accordingly, false otherwise (default)</param>
         /// <returns>Key to try to use</returns>
-        private string SortAndGetKey(DatItem datItem, bool sorted = false)
+        private string SortAndGetKey((long, DatItem) datItem, bool sorted = false)
         {
             // If we're not already sorted, take care of it
             if (!sorted)
                 BucketBy(GetBestAvailable(), DedupeType.None);
 
             // Now that we have the sorted type, we get the proper key
-            return datItem.GetKey(_bucketedBy);
+            var source = GetSourceForItem(datItem.Item1);
+            return datItem.Item2.GetKey(_bucketedBy, source.Item2);
         }
 
         #endregion

SabreTools.DatItems/DatItem.cs

@@ -264,6 +264,52 @@ namespace SabreTools.DatItems
             return output;
         }
 
+        /// <summary>
+        /// Return the duplicate status of two items
+        /// </summary>
+        /// <param name="source">Source associated with this item</param>
+        /// <param name="lastItem">DatItem to check against</param>
+        /// <param name="lastSource">Source associated with the last item</param>
+        /// <returns>The DupeType corresponding to the relationship between the two</returns>
+        public DupeType GetDuplicateStatus(Source? source, DatItem? lastItem, Source? lastSource)
+        {
+            DupeType output = 0x00;
+
+            // If we don't have a duplicate at all, return none
+            if (!Equals(lastItem))
+                return output;
+
+            // If the duplicate is external already or should be, set it
+#if NETFRAMEWORK
+            if ((lastItem.GetFieldValue<DupeType>(DatItem.DupeTypeKey) & DupeType.External) != 0
+                || lastSource?.Index != source?.Index)
+#else
+            if (lastItem.GetFieldValue<DupeType>(DatItem.DupeTypeKey).HasFlag(DupeType.External)
+                || lastSource?.Index != source?.Index)
+#endif
+            {
+                var currentMachine = GetFieldValue<Machine>(DatItem.MachineKey);
+                var lastMachine = lastItem?.GetFieldValue<Machine>(DatItem.MachineKey);
+                if (lastMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) == currentMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) && lastItem?.GetName() == GetName())
+                    output = DupeType.External | DupeType.All;
+                else
+                    output = DupeType.External | DupeType.Hash;
+            }
+
+            // Otherwise, it's considered an internal dupe
+            else
+            {
+                var currentMachine = GetFieldValue<Machine>(DatItem.MachineKey);
+                var lastMachine = lastItem?.GetFieldValue<Machine>(DatItem.MachineKey);
+                if (lastMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) == currentMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) && lastItem?.GetName() == GetName())
+                    output = DupeType.Internal | DupeType.All;
+                else
+                    output = DupeType.Internal | DupeType.Hash;
+            }
+
+            return output;
+        }
+
         #endregion
 
         #region Manipulation
@@ -376,6 +422,71 @@ namespace SabreTools.DatItems
             return key;
         }
 
+        /// <summary>
+        /// Get the dictionary key that should be used for a given item and bucketing type
+        /// </summary>
+        /// <param name="bucketedBy">ItemKey value representing what key to get</param>
+        /// <param name="source">Source associated with the item for renaming</param>
+        /// <param name="lower">True if the key should be lowercased (default), false otherwise</param>
+        /// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
+        /// <returns>String representing the key to be used for the DatItem</returns>
+        public virtual string GetKey(ItemKey bucketedBy, Source? source, bool lower = true, bool norename = true)
+        {
+            // Set the output key as the default blank string
+            string key = string.Empty;
+
+            // Now determine what the key should be based on the bucketedBy value
+            switch (bucketedBy)
+            {
+                case ItemKey.CRC:
+                    key = Constants.CRCZero;
+                    break;
+
+                case ItemKey.Machine:
+                    key = (norename ? string.Empty
+                        : source?.Index.ToString().PadLeft(10, '0')
+                            + "-")
+                    + (string.IsNullOrEmpty(GetFieldValue<Machine>(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey))
+                            ? "Default"
+                            : GetFieldValue<Machine>(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey)!);
+                    if (lower)
+                        key = key.ToLowerInvariant();
+
+                    key ??= "null";
+
+                    break;
+
+                case ItemKey.MD5:
+                    key = Constants.MD5Zero;
+                    break;
+
+                case ItemKey.SHA1:
+                    key = Constants.SHA1Zero;
+                    break;
+
+                case ItemKey.SHA256:
+                    key = Constants.SHA256Zero;
+                    break;
+
+                case ItemKey.SHA384:
+                    key = Constants.SHA384Zero;
+                    break;
+
+                case ItemKey.SHA512:
+                    key = Constants.SHA512Zero;
+                    break;
+
+                case ItemKey.SpamSum:
+                    key = Constants.SpamSumZero;
+                    break;
+            }
+
+            // Double and triple check the key for corner cases
+            key ??= string.Empty;
+
+            return key;
+        }
+
         #endregion
 
         #endregion // Instance Methods

SabreTools.DatTools/DatFileTool.cs

@@ -313,7 +313,7 @@ namespace SabreTools.DatTools
 
                     foreach ((long, DatItem) datItem in datItems)
                     {
-                        var dupes = datFile.ItemsDB.GetDuplicates(datItem.Item2, sorted: true);
+                        var dupes = datFile.ItemsDB.GetDuplicates(datItem, sorted: true);
                         if (datItem.Item2.Clone() is not DatItem newDatItem)
                             continue;
 

SabreTools.DatTools/Rebuilder.cs

@@ -579,7 +579,7 @@ namespace SabreTools.DatTools
         /// <param name="inverse">True if the DAT should be used as a filter instead of a template, false otherwise</param>
         /// <param name="dupes">Output list of duplicate items to rebuild to</param>
         /// <returns>True if the item should be rebuilt, false otherwise</returns>
-        private static bool ShouldRebuildDB(DatFile datFile, DatItem datItem, Stream? stream, bool inverse, out ConcurrentList<(long, DatItem)> dupes)
+        private static bool ShouldRebuildDB(DatFile datFile, (long, DatItem) datItem, Stream? stream, bool inverse, out ConcurrentList<(long, DatItem)> dupes)
         {
             // Find if the file has duplicates in the DAT
             dupes = datFile.ItemsDB.GetDuplicates(datItem);

SabreTools.DatTools/Verification.cs

@@ -182,8 +182,8 @@ namespace SabreTools.DatTools
                     continue;
 
                 // Now we want to remove all duplicates from the DAT
-                datFile.ItemsDB.GetDuplicates(new Rom(fileinfo))
+                datFile.ItemsDB.GetDuplicates((-1, new Rom(fileinfo)))
-                    .AddRange(datFile.ItemsDB.GetDuplicates(new Disk(fileinfo)));
+                    .AddRange(datFile.ItemsDB.GetDuplicates((-1, new Disk(fileinfo))));
             }
 
             watch.Stop();