Create DB version of Against; add passthroughs

This commit is contained in:
Matt Nadareski
2025-02-24 09:20:46 -05:00
parent 2cd17cd711
commit 1bed35c933
19 changed files with 198 additions and 68 deletions

View File

@@ -186,12 +186,12 @@ namespace SabreTools.DatFiles.Test
Assert.Equal(2, actualDatItems.Count);
var actualRom = Assert.Single(actualDatItems, i => i.Value is Rom);
var actualRomMachine = datFile.ItemsDB.GetMachineForItem(actualRom.Key);
var actualRomMachine = datFile.GetMachineForItemDB(actualRom.Key);
Assert.NotNull(actualRomMachine.Value);
Assert.Equal("machine/rom", actualRomMachine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey));
var actualDisk = Assert.Single(actualDatItems, i => i.Value is Disk);
var actualDiskMachine = datFile.ItemsDB.GetMachineForItem(actualDisk.Key);
var actualDiskMachine = datFile.GetMachineForItemDB(actualDisk.Key);
Assert.NotNull(actualDiskMachine.Value);
Assert.Equal("machine/disk", actualDiskMachine.Value.GetStringFieldValue(Models.Metadata.Machine.NameKey));
}

View File

@@ -671,7 +671,7 @@ namespace SabreTools.DatFiles.Test
Assert.Single(datFile.GetItemsForBucketDB("parent"));
long actual = Assert.Single(datFile.GetItemsForBucketDB("child")).Key;
Machine? actualMachine = datFile.ItemsDB.GetMachineForItem(actual).Value;
Machine? actualMachine = datFile.GetMachineForItemDB(actual).Value;
Assert.NotNull(actualMachine);
Assert.Equal("child", actualMachine.GetStringFieldValue(Models.Metadata.Machine.NameKey));
Assert.Equal("romof", actualMachine.GetStringFieldValue(Models.Metadata.Machine.RomOfKey));
@@ -777,7 +777,7 @@ namespace SabreTools.DatFiles.Test
Assert.Single(datFile.GetItemsForBucketDB("parent"));
long actual = Assert.Single(datFile.GetItemsForBucketDB("child")).Key;
Machine? actualMachine = datFile.ItemsDB.GetMachineForItem(actual).Value;
Machine? actualMachine = datFile.GetMachineForItemDB(actual).Value;
Assert.NotNull(actualMachine);
Assert.Equal("child", actualMachine.GetStringFieldValue(Models.Metadata.Machine.NameKey));
}

View File

@@ -598,7 +598,7 @@ namespace SabreTools.DatFiles
return;
// Get the current machine
var machine = ItemsDB.GetMachineForItem(datItem.Key);
var machine = GetMachineForItemDB(datItem.Key);
if (machine.Value == null)
return;

View File

@@ -378,7 +378,7 @@ namespace SabreTools.DatFiles
continue;
// Get the machine for the first item
var machine = ItemsDB.GetMachineForItem(items.First().Key);
var machine = GetMachineForItemDB(items.First().Key);
if (machine.Value == null)
continue;
@@ -396,14 +396,14 @@ namespace SabreTools.DatFiles
foreach (var item in items)
{
// Get the source for the current item
var source = ItemsDB.GetSourceForItem(item.Key);
var source = GetSourceForItemDB(item.Key);
// Get the parent items and current machine name
Dictionary<long, DatItem> parentItems = GetItemsForBucketDB(cloneOf);
if (parentItems.Count == 0)
continue;
string? machineName = ItemsDB.GetMachineForItem(item.Key).Value?
string? machineName = GetMachineForItemDB(item.Key).Value?
.GetStringFieldValue(Models.Metadata.Machine.NameKey);
// Special disk handling
@@ -558,10 +558,10 @@ namespace SabreTools.DatFiles
continue;
// Get the source for the first item
var source = ItemsDB.GetSourceForItem(items.First().Key);
var source = GetSourceForItemDB(items.First().Key);
// Get the machine for the first item in the list
var machine = ItemsDB.GetMachineForItem(items.First().Key);
var machine = GetMachineForItemDB(items.First().Key);
if (machine.Value == null)
continue;
@@ -587,7 +587,7 @@ namespace SabreTools.DatFiles
}
// Get the parent machine
var parentMachine = ItemsDB.GetMachineForItem(GetItemsForBucketDB(cloneOf).First().Key);
var parentMachine = GetMachineForItemDB(GetItemsForBucketDB(cloneOf).First().Key);
if (parentMachine.Value == null)
continue;
@@ -596,7 +596,7 @@ namespace SabreTools.DatFiles
string? romof = parentMachine.Value.GetStringFieldValue(Models.Metadata.Machine.RomOfKey);
foreach (var key in items.Keys)
{
var itemMachine = ItemsDB.GetMachineForItem(key);
var itemMachine = GetMachineForItemDB(key);
if (itemMachine.Value == null)
continue;
@@ -771,10 +771,10 @@ namespace SabreTools.DatFiles
continue;
// Get the source for the first item
var source = ItemsDB.GetSourceForItem(items.First().Key);
var source = GetSourceForItemDB(items.First().Key);
// Get the machine for the first item
var machine = ItemsDB.GetMachineForItem(items.First().Key);
var machine = GetMachineForItemDB(items.First().Key);
if (machine.Value == null)
continue;
@@ -822,7 +822,7 @@ namespace SabreTools.DatFiles
.Select(i => (i as DeviceRef)!.GetName()!));
// Set new machine information and add to the current machine
var copyFrom = ItemsDB.GetMachineForItem(items.First().Key);
var copyFrom = GetMachineForItemDB(items.First().Key);
if (copyFrom.Value == null)
continue;
@@ -878,7 +878,7 @@ namespace SabreTools.DatFiles
.Select(o => o.GetStringFieldValue(Models.Metadata.SlotOption.DevNameKey)!));
// Set new machine information and add to the current machine
var copyFrom = ItemsDB.GetMachineForItem(GetItemsForBucketDB(bucket).First().Key);
var copyFrom = GetMachineForItemDB(GetItemsForBucketDB(bucket).First().Key);
if (copyFrom.Value == null)
continue;
@@ -975,10 +975,10 @@ namespace SabreTools.DatFiles
continue;
// Get the source for the first item
var source = ItemsDB.GetSourceForItem(items.First().Key);
var source = GetSourceForItemDB(items.First().Key);
// Get the machine for the first item
var machine = ItemsDB.GetMachineForItem(items.First().Key);
var machine = GetMachineForItemDB(items.First().Key);
if (machine.Value == null)
continue;
@@ -1049,7 +1049,7 @@ namespace SabreTools.DatFiles
continue;
// Get the machine
var machine = ItemsDB.GetMachineForItem(items.First().Key);
var machine = GetMachineForItemDB(items.First().Key);
if (machine.Value == null)
continue;
@@ -1133,7 +1133,7 @@ namespace SabreTools.DatFiles
continue;
// Get the machine for the first item
var machine = ItemsDB.GetMachineForItem(items.First().Key);
var machine = GetMachineForItemDB(items.First().Key);
if (machine.Value == null)
continue;
@@ -1155,14 +1155,14 @@ namespace SabreTools.DatFiles
// Now we want to get the parent romof tag and put it in each of the remaining items
items = GetItemsForBucketDB(bucket);
machine = ItemsDB.GetMachineForItem(GetItemsForBucketDB(cloneOf).First().Key);
machine = GetMachineForItemDB(GetItemsForBucketDB(cloneOf).First().Key);
if (machine.Value == null)
continue;
string? romof = machine.Value.GetStringFieldValue(Models.Metadata.Machine.RomOfKey);
foreach (var item in items)
{
machine = ItemsDB.GetMachineForItem(item.Key);
machine = GetMachineForItemDB(item.Key);
if (machine.Value == null)
continue;
@@ -1231,7 +1231,7 @@ namespace SabreTools.DatFiles
continue;
// Get the machine for the item
var machine = ItemsDB.GetMachineForItem(items.First().Key);
var machine = GetMachineForItemDB(items.First().Key);
if (machine.Value == null)
continue;

View File

@@ -498,7 +498,7 @@ namespace SabreTools.DatFiles
continue;
// Create a machine to hold everything
var machine = ItemsDB.GetMachineForItem(items.First().Key).Value!.GetInternalClone();
var machine = GetMachineForItemDB(items.First().Key).Value!.GetInternalClone();
// Handle Trurip object, if it exists
if (machine.ContainsKey(Models.Metadata.Machine.TruripKey))

View File

@@ -271,6 +271,18 @@ namespace SabreTools.DatFiles
public IDictionary<long, Machine> GetMachinesDB()
=> ItemsDB.GetMachines();
/// <summary>
/// Get the index and machine associated with an item index
/// </summary>
public KeyValuePair<long, Machine?> GetMachineForItemDB(long itemIndex)
=> GetMachineForItemDB(itemIndex);
/// <summary>
/// Get the index and source associated with an item index
/// </summary>
public KeyValuePair<long, Source?> GetSourceForItemDB(long itemIndex)
=> GetSourceForItemDB(itemIndex);
/// <summary>
/// Remove a key from the file dictionary if it exists
/// </summary>

View File

@@ -135,7 +135,7 @@ namespace SabreTools.DatFiles.Formats
var datItem = new KeyValuePair<long, DatItem>(kvp.Key, ProcessNullifiedItem(kvp.Value));
// Get the machine for the item
var machine = ItemsDB.GetMachineForItem(datItem.Key);
var machine = GetMachineForItemDB(datItem.Key);
// Write out the item if we're using machine names or we're not ignoring
if (!Modifiers.UseRomName || !ShouldIgnore(datItem.Value, ignoreblanks))
@@ -179,7 +179,7 @@ namespace SabreTools.DatFiles.Formats
/// <param name="lastgame">The name of the last game to be output</param>
private void WriteDatItemDB(StreamWriter sw, KeyValuePair<long, DatItem> datItem, string? lastgame)
{
var machine = ItemsDB.GetMachineForItem(datItem.Key).Value;
var machine = GetMachineForItemDB(datItem.Key).Value;
WriteDatItemImpl(sw, datItem.Value, machine!, lastgame);
}

View File

@@ -489,7 +489,7 @@ namespace SabreTools.DatFiles.Formats
foreach (var kvp in items)
{
// Get the machine for the item
var machine = ItemsDB.GetMachineForItem(kvp.Key);
var machine = GetMachineForItemDB(kvp.Key);
// If we have a different game and we're not at the start of the list, output the end of last item
if (lastgame != null && !string.Equals(lastgame, machine.Value!.GetStringFieldValue(Models.Metadata.Machine.NameKey), StringComparison.OrdinalIgnoreCase))
@@ -621,7 +621,7 @@ namespace SabreTools.DatFiles.Formats
private void WriteDatItemDB(JsonTextWriter jtw, KeyValuePair<long, DatItem> datItem)
{
// Get the machine for the item
var machine = ItemsDB.GetMachineForItem(datItem.Key);
var machine = GetMachineForItemDB(datItem.Key);
// Pre-process the item name
ProcessItemName(datItem.Value, machine.Value, forceRemoveQuotes: true, forceRomName: false);

View File

@@ -318,7 +318,7 @@ namespace SabreTools.DatFiles.Formats
foreach (var kvp in items)
{
// Get the machine for the item
var machine = ItemsDB.GetMachineForItem(kvp.Key);
var machine = GetMachineForItemDB(kvp.Key);
// If we have a different game and we're not at the start of the list, output the end of last item
if (lastgame != null && !string.Equals(lastgame, machine.Value!.GetStringFieldValue(Models.Metadata.Machine.NameKey), StringComparison.OrdinalIgnoreCase))
@@ -445,7 +445,7 @@ namespace SabreTools.DatFiles.Formats
private void WriteDatItemDB(XmlTextWriter xtw, KeyValuePair<long, DatItem> datItem)
{
// Get the machine for the item
var machine = ItemsDB.GetMachineForItem(datItem.Key);
var machine = GetMachineForItemDB(datItem.Key);
// Pre-process the item name
ProcessItemName(datItem.Value, machine.Value, forceRemoveQuotes: true, forceRomName: false);

View File

@@ -1156,16 +1156,6 @@ namespace SabreTools.DatFiles
return GetBucketKey(datItem.Key, _bucketedBy, lower: true, norename: true);
}
/// <summary>
/// Class used during deduplication
/// </summary>
private struct ItemMappings(DatItem item, long machineId, long sourceId)
{
public DatItem Item = item;
public long MachineId = machineId;
public long SourceId = sourceId;
}
#endregion
#region Statistics

View File

@@ -0,0 +1,12 @@
namespace SabreTools.DatFiles
{
/// <summary>
/// Class used during deduplication
/// </summary>
public struct ItemMappings(DatItems.DatItem item, long machineId, long sourceId)
{
public DatItems.DatItem Item = item;
public long MachineId = machineId;
public long SourceId = sourceId;
}
}

View File

@@ -202,7 +202,7 @@ namespace SabreTools.DatTools
continue;
// Get the machine associated with the item, if possible
var machine = datFile.ItemsDB.GetMachineForItem(item.Key);
var machine = datFile.GetMachineForItemDB(item.Key);
if (machine.Value == null)
continue;

View File

@@ -38,6 +38,19 @@ namespace SabreTools.DatTools
intDat.Deduplicate();
}
AgainstImpl(datFile, intDat, useGames);
AgainstDBImpl(datFile, intDat, useGames);
watch.Stop();
}
/// <summary>
/// Output diffs against a base set represented by the current DAT
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="intDat">DatFile to replace the values in</param>
/// <param name="useGames">True to diff using games, false to use hashes</param>
private static void AgainstImpl(DatFile datFile, DatFile intDat, bool useGames)
{
// Then we compare against the base DAT
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(intDat.Items.SortedKeys, Core.Globals.ParallelOptions, key =>
@@ -96,8 +109,8 @@ namespace SabreTools.DatTools
// Standard Against uses hashes
else
{
List<DatItem>? datItems = intDat.GetItemsForBucket(key);
if (datItems == null)
List<DatItem> datItems = intDat.GetItemsForBucket(key);
if (datItems.Count == 0)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
@@ -120,8 +133,111 @@ namespace SabreTools.DatTools
#else
}
#endif
}
watch.Stop();
/// <summary>
/// Output diffs against a base set represented by the current DAT
/// </summary>
/// <param name="datFile">Current DatFile object to use for updating</param>
/// <param name="intDat">DatFile to replace the values in</param>
/// <param name="useGames">True to diff using games, false to use hashes</param>
private static void AgainstDBImpl(DatFile datFile, DatFile intDat, bool useGames)
{
// Then we compare against the base DAT
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(intDat.ItemsDB.SortedKeys, Core.Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
Parallel.ForEach(intDat.ItemsDB.SortedKeys, key =>
#else
foreach (var key in intDat.ItemsDB.SortedKeys)
#endif
{
// Game Against uses game names
if (useGames)
{
// If the key is null, keep it
var intList = intDat.GetItemsForBucketDB(key);
if (intList.Count == 0)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
// If the base DAT doesn't contain the key, keep it
List<DatItem> list = [.. datFile.GetItemsForBucketDB(key).Values];
if (list.Count == 0)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
// If the number of items is different, then keep it
if (list.Count != intList.Count)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
//
// Otherwise, compare by name and hash the remaining files
bool exactMatch = true;
foreach (KeyValuePair<long, DatItem> item in intList)
{
// TODO: Make this granular to name as well
if (!list.Contains(item.Value))
{
exactMatch = false;
break;
}
}
// If we have an exact match, remove the game
if (exactMatch)
intDat.RemoveBucket(key);
}
// Standard Against uses hashes
else
{
Dictionary<long, DatItem> datItems = intDat.GetItemsForBucketDB(key);
if (datItems == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
List<KeyValuePair<long, DatItem>> keepDatItems = [];
foreach (KeyValuePair<long, DatItem> datItem in datItems)
{
if (!datFile.HasDuplicates(datItem, true))
keepDatItems.Add(datItem);
}
// Get all existing mappings
List<ItemMappings> currentMappings = keepDatItems.ConvertAll(item =>
{
return new ItemMappings(
item.Value,
intDat.GetMachineForItemDB(item.Key).Key,
intDat.GetSourceForItemDB(item.Key).Key
);
});
// Now add the new list to the key
intDat.RemoveBucketDB(key);
currentMappings.ForEach(map =>
intDat.AddItemDB(map.Item, map.MachineId, map.SourceId, statsOnly: false));
}
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
}
#endregion
@@ -327,8 +443,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
// If the current item isn't an external duplicate
#if NET20 || NET35
@@ -547,8 +663,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
// Get the source associated with the item
var source = datFile.ItemsDB.GetSource(sourceIndex);
@@ -725,8 +841,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
// If the current item isn't a duplicate
#if NET20 || NET35
@@ -857,8 +973,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
// Get the source associated with the item
var source = datFile.ItemsDB.GetSource(sourceIndex);

View File

@@ -178,7 +178,7 @@ namespace SabreTools.DatTools
// Loop through and set the fields accordingly
foreach (var datItem in datItems)
{
var machine = datFile.ItemsDB.GetMachineForItem(datItem.Key);
var machine = datFile.GetMachineForItemDB(datItem.Key);
if (machine.Value != null)
setter.SetFields(machine.Value);

View File

@@ -189,11 +189,11 @@ namespace SabreTools.DatTools
foreach (var item in items)
{
var source = datFile.ItemsDB.GetSourceForItem(item.Key);
var source = datFile.GetSourceForItemDB(item.Key);
if (source.Value == null)
continue;
var machine = datFile.ItemsDB.GetMachineForItem(item.Key);
var machine = datFile.GetMachineForItemDB(item.Key);
if (machine.Value == null)
continue;

View File

@@ -287,8 +287,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = addFrom.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = addFrom.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = addFrom.GetMachineForItemDB(item.Key).Key;
long sourceIndex = addFrom.GetSourceForItemDB(item.Key).Key;
addTo.AddItemDB(item.Value, machineRemapping[machineIndex], sourceRemapping[sourceIndex], statsOnly: false);

View File

@@ -220,8 +220,8 @@ namespace SabreTools.DatTools
foreach (var datItem in datItems)
{
var datMachine = datFile.ItemsDB.GetMachineForItem(datFile.GetItemsForBucketDB(key)!.First().Key);
var intMachine = intDat.ItemsDB.GetMachineForItem(datItem.Key);
var datMachine = datFile.GetMachineForItemDB(datFile.GetItemsForBucketDB(key)!.First().Key);
var intMachine = intDat.GetMachineForItemDB(datItem.Key);
if (datMachine.Value != null && intMachine.Value != null)
Replacer.ReplaceFields(intMachine.Value, datMachine.Value, machineFieldNames, onlySame);
}

View File

@@ -176,8 +176,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
if (newExtA.Contains((item.Value.GetName() ?? string.Empty).GetNormalizedExtension()))
{
@@ -406,8 +406,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
// Only process Disk, Media, and Rom
switch (item.Value)
@@ -726,8 +726,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
// If the file is not a Rom, it automatically goes in the "lesser" dat
if (item.Value is not Rom rom)
@@ -978,8 +978,8 @@ namespace SabreTools.DatTools
#endif
{
// Get the machine and source index for this item
long machineIndex = datFile.ItemsDB.GetMachineForItem(item.Key).Key;
long sourceIndex = datFile.ItemsDB.GetSourceForItem(item.Key).Key;
long machineIndex = datFile.GetMachineForItemDB(item.Key).Key;
long sourceIndex = datFile.GetSourceForItemDB(item.Key).Key;
if (item.Value.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>() == itemType)
indexDat.AddItemDB(item.Value, machineRemapping[machineIndex], sourceRemapping[sourceIndex], statsOnly: false);

View File

@@ -280,7 +280,7 @@ namespace SabreTools.DatTools
foreach (var item in items)
{
// Get the source associated with the item
var source = datFile.ItemsDB.GetSourceForItem(item.Key);
var source = datFile.GetSourceForItemDB(item.Key);
// Unmatched items will have a source ID of int.MaxValue, remove all others
if (source.Value?.Index != int.MaxValue)