diff --git a/SabreTools.DatFiles/DatFile.cs b/SabreTools.DatFiles/DatFile.cs index 44530a0c..accfca5a 100644 --- a/SabreTools.DatFiles/DatFile.cs +++ b/SabreTools.DatFiles/DatFile.cs @@ -793,7 +793,7 @@ namespace SabreTools.DatFiles List output = []; // First we want to make sure the list is in alphabetical order - DatItemTool.Sort(ref items, true); + DatFileTool.Sort(ref items, true); // Now we want to loop through and check names DatItem? lastItem = null; @@ -874,7 +874,7 @@ namespace SabreTools.DatFiles } // One last sort to make sure this is ordered - DatItemTool.Sort(ref output, true); + DatFileTool.Sort(ref output, true); return output; } @@ -890,7 +890,7 @@ namespace SabreTools.DatFiles List> output = []; // First we want to make sure the list is in alphabetical order - DatItemTool.SortDB(ref mappings, true); + DatFileTool.SortDB(ref mappings, true); // Now we want to loop through and check names DatItem? lastItem = null; @@ -968,7 +968,7 @@ namespace SabreTools.DatFiles } // One last sort to make sure this is ordered - DatItemTool.SortDB(ref output, true); + DatFileTool.SortDB(ref output, true); return output; } diff --git a/SabreTools.DatFiles/DatFileTool.cs b/SabreTools.DatFiles/DatFileTool.cs index a6343706..d8b410d5 100644 --- a/SabreTools.DatFiles/DatFileTool.cs +++ b/SabreTools.DatFiles/DatFileTool.cs @@ -5,9 +5,12 @@ using System.Linq; #if NET40_OR_GREATER || NETCOREAPP using System.Threading.Tasks; #endif +using SabreTools.Core.Tools; using SabreTools.DatItems; +using SabreTools.DatItems.Formats; using SabreTools.IO; using SabreTools.IO.Logging; +using SabreTools.Matching.Compare; namespace SabreTools.DatFiles { @@ -25,6 +28,221 @@ namespace SabreTools.DatFiles #endregion + #region Sorting and Merging + + /// + /// Merge an arbitrary set of DatItems based on the supplied information + /// + /// List of DatItem objects representing the items to be merged + /// A List of DatItem objects representing the merged items + public static List Merge(List? items) + { + // Check for null or blank inputs first + if (items == null || items.Count == 0) + return []; + + // Create output list + List output = []; + + // Then deduplicate them by checking to see if data matches previous saved roms + int nodumpCount = 0; + foreach (DatItem datItem in items) + { + // If we don't have a Disk, File, Media, or Rom, we skip checking for duplicates + if (datItem is not Disk && datItem is not DatItems.Formats.File && datItem is not Media && datItem is not Rom) + continue; + + // If it's a nodump, add and skip + if (datItem is Rom rom && rom.GetStringFieldValue(Models.Metadata.Rom.StatusKey).AsEnumValue() == ItemStatus.Nodump) + { + output.Add(datItem); + nodumpCount++; + continue; + } + else if (datItem is Disk disk && disk.GetStringFieldValue(Models.Metadata.Disk.StatusKey).AsEnumValue() == ItemStatus.Nodump) + { + output.Add(datItem); + nodumpCount++; + continue; + } + + // If it's the first non-nodump item in the list, don't touch it + if (output.Count == nodumpCount) + { + output.Add(datItem); + continue; + } + + // Find the index of the first duplicate, if one exists + int pos = output.FindIndex(lastItem => datItem.GetDuplicateStatus(lastItem) != 0x00); + if (pos < 0) + { + output.Add(datItem); + continue; + } + + // Get the duplicate item + DatItem savedItem = output[pos]; + DupeType dupetype = datItem.GetDuplicateStatus(savedItem); + + // Disks, File, Media, and Roms have more information to fill + if (datItem is Disk diskItem && savedItem is Disk savedDisk) + savedDisk.FillMissingInformation(diskItem); + else if (datItem is DatItems.Formats.File fileItem && savedItem is DatItems.Formats.File savedFile) + savedFile.FillMissingInformation(fileItem); + else if (datItem is Media mediaItem && savedItem is Media savedMedia) + savedMedia.FillMissingInformation(mediaItem); + else if (datItem is Rom romItem && savedItem is Rom savedRom) + savedRom.FillMissingInformation(romItem); + + // Set the duplicate type on the saved item + savedItem.SetFieldValue(DatItem.DupeTypeKey, dupetype); + + // Get the sources associated with the items + var savedSource = savedItem.GetFieldValue(DatItem.SourceKey); + var itemSource = datItem.GetFieldValue(DatItem.SourceKey); + + // Get the machines associated with the items + var savedMachine = savedItem.GetFieldValue(DatItem.MachineKey); + var itemMachine = datItem.GetFieldValue(DatItem.MachineKey); + + // If the current source has a lower ID than the saved, use the saved source + if (itemSource?.Index < savedSource?.Index) + { + datItem.SetFieldValue(DatItem.SourceKey, savedSource.Clone() as Source); + savedItem.CopyMachineInformation(datItem); + savedItem.SetName(datItem.GetName()); + } + + // If the saved machine is a child of the current machine, use the current machine instead + if (savedMachine?.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey) == itemMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) + || savedMachine?.GetStringFieldValue(Models.Metadata.Machine.RomOfKey) == itemMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey)) + { + savedItem.CopyMachineInformation(datItem); + savedItem.SetName(datItem.GetName()); + } + + // Replace the original item in the list + output.RemoveAt(pos); + output.Insert(pos, savedItem); + } + + // Then return the result + return output; + } + + /// + /// Sort a list of DatItem objects by SourceID, Game, and Name (in order) + /// + /// List of DatItem objects representing the items to be sorted + /// True if files are not renamed, false otherwise + /// True if it sorted correctly, false otherwise + public static bool Sort(ref List items, bool norename) + { + items.Sort(delegate (DatItem x, DatItem y) + { + try + { + var nc = new NaturalComparer(); + + // If machine names don't match + string? xMachineName = x.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); + string? yMachineName = y.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); + if (xMachineName != yMachineName) + return nc.Compare(xMachineName, yMachineName); + + // If types don't match + string? xType = x.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); + string? yType = y.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); + if (xType != yType) + return xType.AsEnumValue() - yType.AsEnumValue(); + + // If directory names don't match + string? xDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(x.GetName() ?? string.Empty)); + string? yDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(y.GetName() ?? string.Empty)); + if (xDirectoryName != yDirectoryName) + return nc.Compare(xDirectoryName, yDirectoryName); + + // If item names don't match + string? xName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(x.GetName() ?? string.Empty)); + string? yName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(y.GetName() ?? string.Empty)); + if (xName != yName) + return nc.Compare(xName, yName); + + // Otherwise, compare on machine or source, depending on the flag + int? xSourceIndex = x.GetFieldValue(DatItem.SourceKey)?.Index; + int? ySourceIndex = y.GetFieldValue(DatItem.SourceKey)?.Index; + return (norename ? nc.Compare(xMachineName, yMachineName) : (xSourceIndex - ySourceIndex) ?? 0); + } + catch + { + // Absorb the error + return 0; + } + }); + + return true; + } + + /// + /// Sort a list of DatItem objects by SourceID, Game, and Name (in order) + /// + /// List of item ID to DatItem mappings representing the items to be sorted + /// True if files are not renamed, false otherwise + /// True if it sorted correctly, false otherwise + public static bool SortDB(ref List> mappings, bool norename) + { + mappings.Sort(delegate (KeyValuePair x, KeyValuePair y) + { + try + { + var nc = new NaturalComparer(); + + // TODO: Fix this since DB uses an external map for machines + + // If machine names don't match + string? xMachineName = x.Value.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); + string? yMachineName = y.Value.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); + if (xMachineName != yMachineName) + return nc.Compare(xMachineName, yMachineName); + + // If types don't match + string? xType = x.Value.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); + string? yType = y.Value.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); + if (xType != yType) + return xType.AsEnumValue() - yType.AsEnumValue(); + + // If directory names don't match + string? xDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(x.Value.GetName() ?? string.Empty)); + string? yDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(y.Value.GetName() ?? string.Empty)); + if (xDirectoryName != yDirectoryName) + return nc.Compare(xDirectoryName, yDirectoryName); + + // If item names don't match + string? xName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(x.Value.GetName() ?? string.Empty)); + string? yName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(y.Value.GetName() ?? string.Empty)); + if (xName != yName) + return nc.Compare(xName, yName); + + // Otherwise, compare on machine or source, depending on the flag + int? xSourceIndex = x.Value.GetFieldValue(DatItem.SourceKey)?.Index; + int? ySourceIndex = y.Value.GetFieldValue(DatItem.SourceKey)?.Index; + return (norename ? nc.Compare(xMachineName, yMachineName) : (xSourceIndex - ySourceIndex) ?? 0); + } + catch + { + // Absorb the error + return 0; + } + }); + + return true; + } + + #endregion + + #region SuperDAT + /// /// Apply SuperDAT naming logic to a merged DatFile /// @@ -158,6 +376,10 @@ namespace SabreTools.DatFiles #endif } + #endregion + + #region Replacement + /// /// Replace item values from the base set represented by the current DAT /// @@ -374,6 +596,10 @@ namespace SabreTools.DatFiles watch.Stop(); } + #endregion + + #region Diffing + /// /// Output diffs against a base set represented by the current DAT /// @@ -576,7 +802,7 @@ namespace SabreTools.DatFiles foreach (var key in datFile.Items.Keys) #endif { - List items = DatItemTool.Merge(datFile.Items[key]); + List items = Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -801,7 +1027,7 @@ namespace SabreTools.DatFiles foreach (var key in datFile.Items.Keys) #endif { - List items = DatItemTool.Merge(datFile.Items[key]); + List items = Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -1011,7 +1237,7 @@ namespace SabreTools.DatFiles foreach (var key in datFile.Items.Keys) #endif { - List items = DatItemTool.Merge(datFile.Items[key]); + List items = Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -1163,6 +1389,10 @@ namespace SabreTools.DatFiles return outerDiffData; } + #endregion + + #region Population + /// /// Populate from multiple paths while returning the invividual headers /// @@ -1325,7 +1555,7 @@ namespace SabreTools.DatFiles foreach (var key in datFile.Items.Keys) #endif { - List items = DatItemTool.Merge(datFile.Items[key]); + List items = Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0) @@ -1406,5 +1636,7 @@ namespace SabreTools.DatFiles } #endif } + + #endregion } } diff --git a/SabreTools.DatFiles/ItemDictionary.cs b/SabreTools.DatFiles/ItemDictionary.cs index 8b5831f7..7fe708e9 100644 --- a/SabreTools.DatFiles/ItemDictionary.cs +++ b/SabreTools.DatFiles/ItemDictionary.cs @@ -759,11 +759,11 @@ namespace SabreTools.DatFiles #endif // Sort the list of items to be consistent - DatItemTool.Sort(ref sortedlist, false); + DatFileTool.Sort(ref sortedlist, false); // If we're merging the roms, do so if (dedupeType == DedupeType.Full || (dedupeType == DedupeType.Game && bucketBy == ItemKey.Machine)) - sortedlist = DatItemTool.Merge(sortedlist); + sortedlist = DatFileTool.Merge(sortedlist); // Add the list back to the dictionary Reset(key); @@ -794,7 +794,7 @@ namespace SabreTools.DatFiles // Sort the list of items to be consistent if (sortedlist != null) - DatItemTool.Sort(ref sortedlist, false); + DatFileTool.Sort(ref sortedlist, false); #if NET40_OR_GREATER || NETCOREAPP }); #else diff --git a/SabreTools.DatItems/DatItemTool.cs b/SabreTools.DatItems/DatItemTool.cs index 69946da6..75793726 100644 --- a/SabreTools.DatItems/DatItemTool.cs +++ b/SabreTools.DatItems/DatItemTool.cs @@ -1,5 +1,3 @@ -using System.Collections.Generic; -using System.IO; using SabreTools.Core.Tools; using SabreTools.DatItems.Formats; using SabreTools.FileTypes; @@ -7,7 +5,6 @@ using SabreTools.FileTypes.Aaru; using SabreTools.FileTypes.CHD; using SabreTools.IO.Extensions; using SabreTools.IO.Logging; -using SabreTools.Matching.Compare; namespace SabreTools.DatItems { @@ -266,218 +263,5 @@ namespace SabreTools.DatItems } #endregion - - #region Sorting and Merging - - /// - /// Merge an arbitrary set of DatItems based on the supplied information - /// - /// List of DatItem objects representing the items to be merged - /// A List of DatItem objects representing the merged items - public static List Merge(List? items) - { - // Check for null or blank inputs first - if (items == null || items.Count == 0) - return []; - - // Create output list - List output = []; - - // Then deduplicate them by checking to see if data matches previous saved roms - int nodumpCount = 0; - foreach (DatItem datItem in items) - { - // If we don't have a Disk, File, Media, or Rom, we skip checking for duplicates - if (datItem is not Disk && datItem is not Formats.File && datItem is not Media && datItem is not Rom) - continue; - - // If it's a nodump, add and skip - if (datItem is Rom rom && rom.GetStringFieldValue(Models.Metadata.Rom.StatusKey).AsEnumValue() == ItemStatus.Nodump) - { - output.Add(datItem); - nodumpCount++; - continue; - } - else if (datItem is Disk disk && disk.GetStringFieldValue(Models.Metadata.Disk.StatusKey).AsEnumValue() == ItemStatus.Nodump) - { - output.Add(datItem); - nodumpCount++; - continue; - } - - // If it's the first non-nodump item in the list, don't touch it - if (output.Count == nodumpCount) - { - output.Add(datItem); - continue; - } - - // Find the index of the first duplicate, if one exists - int pos = output.FindIndex(lastItem => datItem.GetDuplicateStatus(lastItem) != 0x00); - if (pos < 0) - { - output.Add(datItem); - continue; - } - - // Get the duplicate item - DatItem savedItem = output[pos]; - DupeType dupetype = datItem.GetDuplicateStatus(savedItem); - - // Disks, File, Media, and Roms have more information to fill - if (datItem is Disk diskItem && savedItem is Disk savedDisk) - savedDisk.FillMissingInformation(diskItem); - else if (datItem is Formats.File fileItem && savedItem is Formats.File savedFile) - savedFile.FillMissingInformation(fileItem); - else if (datItem is Media mediaItem && savedItem is Media savedMedia) - savedMedia.FillMissingInformation(mediaItem); - else if (datItem is Rom romItem && savedItem is Rom savedRom) - savedRom.FillMissingInformation(romItem); - - // Set the duplicate type on the saved item - savedItem.SetFieldValue(DatItem.DupeTypeKey, dupetype); - - // Get the sources associated with the items - var savedSource = savedItem.GetFieldValue(DatItem.SourceKey); - var itemSource = datItem.GetFieldValue(DatItem.SourceKey); - - // Get the machines associated with the items - var savedMachine = savedItem.GetFieldValue(DatItem.MachineKey); - var itemMachine = datItem.GetFieldValue(DatItem.MachineKey); - - // If the current source has a lower ID than the saved, use the saved source - if (itemSource?.Index < savedSource?.Index) - { - datItem.SetFieldValue(DatItem.SourceKey, savedSource.Clone() as Source); - savedItem.CopyMachineInformation(datItem); - savedItem.SetName(datItem.GetName()); - } - - // If the saved machine is a child of the current machine, use the current machine instead - if (savedMachine?.GetStringFieldValue(Models.Metadata.Machine.CloneOfKey) == itemMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey) - || savedMachine?.GetStringFieldValue(Models.Metadata.Machine.RomOfKey) == itemMachine?.GetStringFieldValue(Models.Metadata.Machine.NameKey)) - { - savedItem.CopyMachineInformation(datItem); - savedItem.SetName(datItem.GetName()); - } - - // Replace the original item in the list - output.RemoveAt(pos); - output.Insert(pos, savedItem); - } - - // Then return the result - return output; - } - - /// - /// Sort a list of DatItem objects by SourceID, Game, and Name (in order) - /// - /// List of DatItem objects representing the items to be sorted - /// True if files are not renamed, false otherwise - /// True if it sorted correctly, false otherwise - public static bool Sort(ref List items, bool norename) - { - items.Sort(delegate (DatItem x, DatItem y) - { - try - { - var nc = new NaturalComparer(); - - // If machine names don't match - string? xMachineName = x.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); - string? yMachineName = y.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); - if (xMachineName != yMachineName) - return nc.Compare(xMachineName, yMachineName); - - // If types don't match - string? xType = x.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); - string? yType = y.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); - if (xType != yType) - return xType.AsEnumValue() - yType.AsEnumValue(); - - // If directory names don't match - string? xDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(x.GetName() ?? string.Empty)); - string? yDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(y.GetName() ?? string.Empty)); - if (xDirectoryName != yDirectoryName) - return nc.Compare(xDirectoryName, yDirectoryName); - - // If item names don't match - string? xName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(x.GetName() ?? string.Empty)); - string? yName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(y.GetName() ?? string.Empty)); - if (xName != yName) - return nc.Compare(xName, yName); - - // Otherwise, compare on machine or source, depending on the flag - int? xSourceIndex = x.GetFieldValue(DatItem.SourceKey)?.Index; - int? ySourceIndex = y.GetFieldValue(DatItem.SourceKey)?.Index; - return (norename ? nc.Compare(xMachineName, yMachineName) : (xSourceIndex - ySourceIndex) ?? 0); - } - catch - { - // Absorb the error - return 0; - } - }); - - return true; - } - - /// - /// Sort a list of DatItem objects by SourceID, Game, and Name (in order) - /// - /// List of item ID to DatItem mappings representing the items to be sorted - /// True if files are not renamed, false otherwise - /// True if it sorted correctly, false otherwise - public static bool SortDB(ref List> mappings, bool norename) - { - mappings.Sort(delegate (KeyValuePair x, KeyValuePair y) - { - try - { - var nc = new NaturalComparer(); - - // TODO: Fix this since DB uses an external map for machines - - // If machine names don't match - string? xMachineName = x.Value.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); - string? yMachineName = y.Value.GetFieldValue(DatItem.MachineKey)?.GetStringFieldValue(Models.Metadata.Machine.NameKey); - if (xMachineName != yMachineName) - return nc.Compare(xMachineName, yMachineName); - - // If types don't match - string? xType = x.Value.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); - string? yType = y.Value.GetStringFieldValue(Models.Metadata.DatItem.TypeKey); - if (xType != yType) - return xType.AsEnumValue() - yType.AsEnumValue(); - - // If directory names don't match - string? xDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(x.Value.GetName() ?? string.Empty)); - string? yDirectoryName = Path.GetDirectoryName(TextHelper.RemovePathUnsafeCharacters(y.Value.GetName() ?? string.Empty)); - if (xDirectoryName != yDirectoryName) - return nc.Compare(xDirectoryName, yDirectoryName); - - // If item names don't match - string? xName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(x.Value.GetName() ?? string.Empty)); - string? yName = Path.GetFileName(TextHelper.RemovePathUnsafeCharacters(y.Value.GetName() ?? string.Empty)); - if (xName != yName) - return nc.Compare(xName, yName); - - // Otherwise, compare on machine or source, depending on the flag - int? xSourceIndex = x.Value.GetFieldValue(DatItem.SourceKey)?.Index; - int? ySourceIndex = y.Value.GetFieldValue(DatItem.SourceKey)?.Index; - return (norename ? nc.Compare(xMachineName, yMachineName) : (xSourceIndex - ySourceIndex) ?? 0); - } - catch - { - // Absorb the error - return 0; - } - }); - - return true; - } - - #endregion } } \ No newline at end of file diff --git a/SabreTools.DatItems/SabreTools.DatItems.csproj b/SabreTools.DatItems/SabreTools.DatItems.csproj index b3dcb51e..7c3487b9 100644 --- a/SabreTools.DatItems/SabreTools.DatItems.csproj +++ b/SabreTools.DatItems/SabreTools.DatItems.csproj @@ -30,7 +30,6 @@ - diff --git a/SabreTools.DatTools/Splitter.cs b/SabreTools.DatTools/Splitter.cs index d19d5672..134dabd7 100644 --- a/SabreTools.DatTools/Splitter.cs +++ b/SabreTools.DatTools/Splitter.cs @@ -904,7 +904,7 @@ namespace SabreTools.DatTools foreach (var key in datFile.Items.Keys) #endif { - List items = DatItemTool.Merge(datFile.Items[key]); + List items = DatFileTool.Merge(datFile.Items[key]); // If the rom list is empty or null, just skip it if (items == null || items.Count == 0)