From c2ca5a0a6dc4a47c5e0424470bc63d6417e7f938 Mon Sep 17 00:00:00 2001 From: Matt Nadareski Date: Tue, 19 Mar 2024 23:35:29 -0400 Subject: [PATCH] Add more IDDB accessors; make more IDDB variants of methods --- SabreTools.DatFiles/DatFile.Removal.cs | 2 +- SabreTools.DatFiles/ItemDictionaryDB.cs | 148 ++++++----- SabreTools.DatTools/Splitter.cs | 331 ++++++++++++++++++++++++ SabreTools.DatTools/Verification.cs | 2 +- SabreTools.Filtering/Cleaner.cs | 2 +- SabreTools.Filtering/ExtraIni.cs | 2 +- SabreTools/Features/Split.cs | 3 + 7 files changed, 418 insertions(+), 72 deletions(-) diff --git a/SabreTools.DatFiles/DatFile.Removal.cs b/SabreTools.DatFiles/DatFile.Removal.cs index 4bcc6bae..6cecc4f3 100644 --- a/SabreTools.DatFiles/DatFile.Removal.cs +++ b/SabreTools.DatFiles/DatFile.Removal.cs @@ -74,7 +74,7 @@ namespace SabreTools.DatFiles foreach (var key in ItemsDB.SortedKeys) #endif { - var items = ItemsDB.GetDatItemsForBucket(key); + var items = ItemsDB.GetItemsForBucket(key); if (items == null) #if NET40_OR_GREATER || NETCOREAPP return; diff --git a/SabreTools.DatFiles/ItemDictionaryDB.cs b/SabreTools.DatFiles/ItemDictionaryDB.cs index 3c97083e..18b830e1 100644 --- a/SabreTools.DatFiles/ItemDictionaryDB.cs +++ b/SabreTools.DatFiles/ItemDictionaryDB.cs @@ -305,6 +305,51 @@ namespace SabreTools.DatFiles return _items[index]; } + /// + /// Get all items from the current dictionary + /// + public DatItem[] GetItems() + => _items.Select(kvp => kvp.Value).ToArray(); + + /// + /// Get the indices and items associated with a bucket name + /// + public (long, DatItem)[]? GetItemsForBucket(string bucketName, bool filter = false) + { + if (!_buckets.ContainsKey(bucketName)) + return null; + + var itemIds = _buckets[bucketName]; + + var datItems = new List<(long, DatItem)>(); + foreach (long itemId in itemIds) + { + if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true)) + datItems.Add((itemId, _items[itemId])); + } + + return [.. datItems]; + } + + /// + /// Get the indices and items associated with a machine index + /// + public (long, DatItem)[]? GetItemsForMachine(long machineIndex, bool filter = false) + { + var itemIds = _itemToMachineMapping + .Where(mapping => mapping.Value == machineIndex) + .Select(mapping => mapping.Key); + + var datItems = new List<(long, DatItem)>(); + foreach (long itemId in itemIds) + { + if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true)) + datItems.Add((itemId, _items[itemId])); + } + + return [.. datItems]; + } + /// /// Get a machine based on the index /// @@ -345,43 +390,10 @@ namespace SabreTools.DatFiles } /// - /// Get the indices and items associated with a bucket name + /// Get all machines from the current dictionary /// - public (long, DatItem)[]? GetDatItemsForBucket(string bucketName, bool filter = false) - { - if (!_buckets.ContainsKey(bucketName)) - return null; - - var itemIds = _buckets[bucketName]; - - var datItems = new List<(long, DatItem)>(); - foreach (long itemId in itemIds) - { - if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true)) - datItems.Add((itemId, _items[itemId])); - } - - return [.. datItems]; - } - - /// - /// Get the indices and items associated with a machine index - /// - public (long, DatItem)[]? GetDatItemsForMachine(long machineIndex, bool filter = false) - { - var itemIds = _itemToMachineMapping - .Where(mapping => mapping.Value == machineIndex) - .Select(mapping => mapping.Key); - - var datItems = new List<(long, DatItem)>(); - foreach (long itemId in itemIds) - { - if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true)) - datItems.Add((itemId, _items[itemId])); - } - - return [.. datItems]; - } + public Machine[] GetMachines() + => _machines.Select(kvp => kvp.Value).ToArray(); /// /// Remove an item, returning if it could be removed @@ -518,7 +530,7 @@ namespace SabreTools.DatFiles string key = SortAndGetKey(datItem, sorted); // If the key doesn't exist, return the empty list - var roms = GetDatItemsForBucket(key); + var roms = GetItemsForBucket(key); if (roms == null || roms.Length == 0) return output; @@ -562,7 +574,7 @@ namespace SabreTools.DatFiles string key = SortAndGetKey(datItem, sorted); // If the key doesn't exist - var roms = GetDatItemsForBucket(key); + var roms = GetItemsForBucket(key); if (roms == null || roms.Length == 0) return false; @@ -1115,7 +1127,7 @@ namespace SabreTools.DatFiles Dictionary> parents = []; foreach (string key in SortedKeys) { - var items = GetDatItemsForBucket(key); + var items = GetItemsForBucket(key); if (items == null || items.Length == 0) continue; @@ -1193,7 +1205,7 @@ namespace SabreTools.DatFiles foreach (var key in SortedKeys) #endif { - var items = GetDatItemsForBucket(key); + var items = GetItemsForBucket(key); if (items == null) #if NET40_OR_GREATER || NETCOREAPP return; @@ -1229,7 +1241,7 @@ namespace SabreTools.DatFiles foreach (var key in SortedKeys) #endif { - var items = GetDatItemsForBucket(key); + var items = GetItemsForBucket(key); if (items == null) #if NET40_OR_GREATER || NETCOREAPP return; @@ -1283,7 +1295,7 @@ namespace SabreTools.DatFiles foreach (var key in SortedKeys) #endif { - var items = GetDatItemsForBucket(key); + var items = GetItemsForBucket(key); if (items == null) #if NET40_OR_GREATER || NETCOREAPP return; @@ -1323,7 +1335,7 @@ namespace SabreTools.DatFiles /// Name of the bucket to filter on private void ExecuteFilterOnBucket(FilterRunner filterRunner, string bucketName) { - (long, DatItem)[]? items = GetDatItemsForBucket(bucketName); + (long, DatItem)[]? items = GetItemsForBucket(bucketName); if (items == null) return; @@ -1381,7 +1393,7 @@ namespace SabreTools.DatFiles foreach (var key in SortedKeys) #endif { - var items = GetDatItemsForBucket(key); + var items = GetItemsForBucket(key); if (items == null) #if NET40_OR_GREATER || NETCOREAPP return; @@ -1438,7 +1450,7 @@ namespace SabreTools.DatFiles List games = [.. SortedKeys]; foreach (string game in games) { - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); // If the game has no items in it, we want to continue if (items == null || items.Length == 0) @@ -1463,7 +1475,7 @@ namespace SabreTools.DatFiles continue; // If the parent doesn't have any items, we want to continue - var parentItems = GetDatItemsForBucket(parent.Item2!); + var parentItems = GetItemsForBucket(parent.Item2!); if (parentItems == null || parentItems.Length == 0) continue; @@ -1489,7 +1501,7 @@ namespace SabreTools.DatFiles foreach (string game in games) { // Get the items for this game - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); // If the machine doesn't have items, we continue if (items == null || items.Length == 0) @@ -1534,7 +1546,7 @@ namespace SabreTools.DatFiles continue; // If the machine doesn't exist then we continue - var devItems = GetDatItemsForBucket(deviceReference); + var devItems = GetItemsForBucket(deviceReference); if (devItems == null || devItems.Length == 0) continue; @@ -1544,14 +1556,14 @@ namespace SabreTools.DatFiles .Select(i => (i.Item2 as DeviceRef)!.GetName()!)); // Set new machine information and add to the current machine - var copyFrom = GetMachineForItem(GetDatItemsForBucket(game)![0].Item1); + var copyFrom = GetMachineForItem(GetItemsForBucket(game)![0].Item1); if (copyFrom.Item2 == null) continue; foreach ((long, DatItem) item in devItems) { // If the parent machine doesn't already contain this item, add it - if (!GetDatItemsForBucket(game)! + if (!GetItemsForBucket(game)! .Any(i => i.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey) == item.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey) && i.Item2.GetName() == item.Item2.GetName())) { @@ -1589,7 +1601,7 @@ namespace SabreTools.DatFiles continue; // If the machine doesn't exist then we continue - var slotItems = GetDatItemsForBucket(slotOption); + var slotItems = GetItemsForBucket(slotOption); if (slotItems == null || slotItems.Length == 0) continue; @@ -1601,14 +1613,14 @@ namespace SabreTools.DatFiles .Select(o => o.GetStringFieldValue(Models.Metadata.SlotOption.DevNameKey)!)); // Set new machine information and add to the current machine - var copyFrom = GetMachineForItem(GetDatItemsForBucket(game)![0].Item1); + var copyFrom = GetMachineForItem(GetItemsForBucket(game)![0].Item1); if (copyFrom.Item2 == null) continue; foreach ((long, DatItem) item in slotItems) { // If the parent machine doesn't already contain this item, add it - if (!GetDatItemsForBucket(game)! + if (!GetItemsForBucket(game)! .Any(i => i.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey) == item.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey) && i.Item2.GetName() == item.Item2.GetName())) { @@ -1651,7 +1663,7 @@ namespace SabreTools.DatFiles foreach (string game in games) { // If the game has no items in it, we want to continue - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); if (items == null || items.Length == 0) continue; @@ -1674,7 +1686,7 @@ namespace SabreTools.DatFiles continue; // If the parent doesn't have any items, we want to continue - var parentItems = GetDatItemsForBucket(parent.Item2!); + var parentItems = GetItemsForBucket(parent.Item2!); if (parentItems == null || parentItems.Length == 0) continue; @@ -1690,12 +1702,12 @@ namespace SabreTools.DatFiles } // Get the parent machine - var parentMachine = GetMachineForItem(GetDatItemsForBucket(parent.Item2!)![0].Item1); + var parentMachine = GetMachineForItem(GetItemsForBucket(parent.Item2!)![0].Item1); if (parentMachine.Item2 == null) continue; // Now we want to get the parent romof tag and put it in each of the items - items = GetDatItemsForBucket(game); + items = GetItemsForBucket(game); string? romof = parentMachine.Item2.GetStringFieldValue(Models.Metadata.Machine.RomOfKey); foreach ((long, DatItem) item in items!) { @@ -1719,7 +1731,7 @@ namespace SabreTools.DatFiles foreach (string game in games) { // If the game has no items in it, we want to continue - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); if (items == null || items.Length == 0) continue; @@ -1741,11 +1753,11 @@ namespace SabreTools.DatFiles if (string.IsNullOrEmpty(parent.Item2)) continue; - items = GetDatItemsForBucket(game); + items = GetItemsForBucket(game); foreach ((long, DatItem) item in items!) { // Get the parent items and current machine name - var parentItems = GetDatItemsForBucket(parent.Item2!); + var parentItems = GetItemsForBucket(parent.Item2!); string? machineName = GetMachineForItem(item.Item1).Item2?.GetStringFieldValue(Models.Metadata.Machine.NameKey); // Special disk handling @@ -1844,7 +1856,7 @@ namespace SabreTools.DatFiles List games = [.. SortedKeys]; foreach (string game in games) { - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); if (items == null || items.Length == 0) continue; @@ -1874,7 +1886,7 @@ namespace SabreTools.DatFiles foreach (string game in games) { // If the game has no items in it, we want to continue - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); if (items == null || items.Length == 0) continue; @@ -1897,7 +1909,7 @@ namespace SabreTools.DatFiles continue; // If the parent doesn't have any items, we want to continue - var parentItems = GetDatItemsForBucket(parent!); + var parentItems = GetItemsForBucket(parent!); if (parentItems == null || parentItems.Length == 0) continue; @@ -1921,7 +1933,7 @@ namespace SabreTools.DatFiles List games = [.. SortedKeys]; foreach (string game in games) { - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); if (items == null) continue; @@ -1944,7 +1956,7 @@ namespace SabreTools.DatFiles continue; // If the parent doesn't have any items, we want to continue - var parentItems = GetDatItemsForBucket(parent!); + var parentItems = GetItemsForBucket(parent!); if (parentItems == null || parentItems.Length == 0) continue; @@ -1959,8 +1971,8 @@ namespace SabreTools.DatFiles } // Now we want to get the parent romof tag and put it in each of the remaining items - items = GetDatItemsForBucket(game); - machine = GetMachineForItem(GetDatItemsForBucket(parent!)![0].Item1); + items = GetItemsForBucket(game); + machine = GetMachineForItem(GetItemsForBucket(parent!)![0].Item1); if (machine.Item2 == null) continue; @@ -1984,7 +1996,7 @@ namespace SabreTools.DatFiles List games = [.. SortedKeys]; foreach (string game in games) { - var items = GetDatItemsForBucket(game); + var items = GetItemsForBucket(game); if (items == null) continue; diff --git a/SabreTools.DatTools/Splitter.cs b/SabreTools.DatTools/Splitter.cs index faac7e88..9f81a820 100644 --- a/SabreTools.DatTools/Splitter.cs +++ b/SabreTools.DatTools/Splitter.cs @@ -109,6 +109,84 @@ namespace SabreTools.DatTools return (extADat, extBDat); } + /// + /// Split a DAT by input extensions + /// + /// Current DatFile object to split + /// List of extensions to split on (first DAT) + /// List of extensions to split on (second DAT) + /// Extension Set A and Extension Set B DatFiles + public static (DatFile? extADat, DatFile? extBDat) SplitByExtensionDB(DatFile datFile, List extA, List extB) + { + // If roms is empty, return false + if (datFile.ItemsDB.DatStatistics.TotalCount == 0) + return (null, null); + + InternalStopwatch watch = new($"Splitting DAT by extension"); + + // Make sure all of the extensions don't have a dot at the beginning + var newExtA = extA.Select(s => s.TrimStart('.').ToLowerInvariant()).ToArray(); + string newExtAString = string.Join(",", newExtA); + + var newExtB = extB.Select(s => s.TrimStart('.').ToLowerInvariant()).ToArray(); + string newExtBString = string.Join(",", newExtB); + + // Set all of the appropriate outputs for each of the subsets + DatFile extADat = DatFile.Create(datFile.Header.CloneStandard()); + extADat.Header.SetFieldValue(DatHeader.FileNameKey, extADat.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" ({newExtAString})"); + extADat.Header.SetFieldValue(Models.Metadata.Header.NameKey, extADat.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" ({newExtAString})"); + extADat.Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, extADat.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" ({newExtAString})"); + + DatFile extBDat = DatFile.Create(datFile.Header.CloneStandard()); + extBDat.Header.SetFieldValue(DatHeader.FileNameKey, extBDat.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" ({newExtBString})"); + extBDat.Header.SetFieldValue(Models.Metadata.Header.NameKey, extBDat.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" ({newExtBString})"); + extBDat.Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, extBDat.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" ({newExtBString})"); + + // Now separate the roms accordingly +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(datFile.ItemsDB.SortedKeys, key => +#else + foreach (var key in datFile.ItemsDB.SortedKeys) +#endif + { + var items = datFile.ItemsDB.GetItemsForBucket(key); + if (items == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + // TODO: Determine how we figure out the machine index + foreach ((long, DatItem) item in items) + { + if (newExtA.Contains((item.Item2.GetName() ?? string.Empty).GetNormalizedExtension())) + { + extADat.ItemsDB.AddItem(item.Item2, -1, false); + } + else if (newExtB.Contains((item.Item2.GetName() ?? string.Empty).GetNormalizedExtension())) + { + extBDat.ItemsDB.AddItem(item.Item2, -1, false); + } + else + { + extADat.ItemsDB.AddItem(item.Item2, -1, false); + extBDat.ItemsDB.AddItem(item.Item2, -1, false); + } + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + + // Then return both DatFiles + watch.Stop(); + return (extADat, extBDat); + } + /// /// Split a DAT by best available hashes /// @@ -241,6 +319,140 @@ namespace SabreTools.DatTools return fieldDats; } + /// + /// Split a DAT by best available hashes + /// + /// Current DatFile object to split + /// Dictionary of Field to DatFile mappings + public static Dictionary SplitByHashDB(DatFile datFile) + { + // Create each of the respective output DATs + var watch = new InternalStopwatch($"Splitting DAT by best available hashes"); + + // Create the set of field-to-dat mappings + Dictionary fieldDats = []; + + // TODO: Can this be made into a loop? + fieldDats[Models.Metadata.Rom.StatusKey] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats[Models.Metadata.Rom.StatusKey].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.StatusKey].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (Nodump)"); + fieldDats[Models.Metadata.Rom.StatusKey].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.StatusKey].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (Nodump)"); + fieldDats[Models.Metadata.Rom.StatusKey].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.StatusKey].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (Nodump)"); + + fieldDats[Models.Metadata.Rom.SHA512Key] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats[Models.Metadata.Rom.SHA512Key].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA512Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-512)"); + fieldDats[Models.Metadata.Rom.SHA512Key].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA512Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-512)"); + fieldDats[Models.Metadata.Rom.SHA512Key].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA512Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-512)"); + + fieldDats[Models.Metadata.Rom.SHA384Key] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats[Models.Metadata.Rom.SHA384Key].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA384Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-384)"); + fieldDats[Models.Metadata.Rom.SHA384Key].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA384Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-384)"); + fieldDats[Models.Metadata.Rom.SHA384Key].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA384Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-384)"); + + fieldDats[Models.Metadata.Rom.SHA256Key] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats[Models.Metadata.Rom.SHA256Key].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA256Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-256)"); + fieldDats[Models.Metadata.Rom.SHA256Key].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA256Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-256)"); + fieldDats[Models.Metadata.Rom.SHA256Key].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA256Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-256)"); + + fieldDats[Models.Metadata.Rom.SHA1Key] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats[Models.Metadata.Rom.SHA1Key].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA1Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-1)"); + fieldDats[Models.Metadata.Rom.SHA1Key].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA1Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-1)"); + fieldDats[Models.Metadata.Rom.SHA1Key].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA1Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-1)"); + + fieldDats[Models.Metadata.Rom.MD5Key] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats[Models.Metadata.Rom.MD5Key].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.MD5Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (MD5)"); + fieldDats[Models.Metadata.Rom.MD5Key].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.MD5Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (MD5)"); + fieldDats[Models.Metadata.Rom.MD5Key].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.MD5Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (MD5)"); + + fieldDats[Models.Metadata.Rom.CRCKey] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats[Models.Metadata.Rom.CRCKey].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.CRCKey].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (CRC)"); + fieldDats[Models.Metadata.Rom.CRCKey].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.CRCKey].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (CRC)"); + fieldDats[Models.Metadata.Rom.CRCKey].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.CRCKey].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (CRC)"); + + fieldDats["null"] = DatFile.Create(datFile.Header.CloneStandard()); + fieldDats["null"].Header.SetFieldValue(DatHeader.FileNameKey, fieldDats["null"].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (Other)"); + fieldDats["null"].Header.SetFieldValue(Models.Metadata.Header.NameKey, fieldDats["null"].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (Other)"); + fieldDats["null"].Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, fieldDats["null"].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (Other)"); + + // Now populate each of the DAT objects in turn +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(datFile.ItemsDB.SortedKeys, key => +#else + foreach (var key in datFile.ItemsDB.SortedKeys) +#endif + { + var items = datFile.ItemsDB.GetItemsForBucket(key); + if (items == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + // TODO: Determine how we figure out the machine index + foreach ((long, DatItem) item in items) + { + // If the file is not a Disk, Media, or Rom, continue + switch (item.Item2) + { + case Disk disk: + if (disk.GetStringFieldValue(Models.Metadata.Disk.StatusKey).AsEnumValue() == ItemStatus.Nodump) + fieldDats[Models.Metadata.Disk.StatusKey].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(disk.GetStringFieldValue(Models.Metadata.Disk.SHA1Key))) + fieldDats[Models.Metadata.Disk.SHA1Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(disk.GetStringFieldValue(Models.Metadata.Disk.MD5Key))) + fieldDats[Models.Metadata.Disk.MD5Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(disk.GetStringFieldValue(Models.Metadata.Disk.MD5Key))) + fieldDats[Models.Metadata.Disk.MD5Key].ItemsDB.AddItem(item.Item2, -1, false); + else + fieldDats["null"].ItemsDB.AddItem(item.Item2, -1, false); + break; + + case Media media: + if (!string.IsNullOrEmpty(media.GetStringFieldValue(Models.Metadata.Media.SHA256Key))) + fieldDats[Models.Metadata.Media.SHA256Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(media.GetStringFieldValue(Models.Metadata.Media.SHA1Key))) + fieldDats[Models.Metadata.Media.SHA1Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(media.GetStringFieldValue(Models.Metadata.Media.MD5Key))) + fieldDats[Models.Metadata.Media.MD5Key].ItemsDB.AddItem(item.Item2, -1, false); + else + fieldDats["null"].ItemsDB.AddItem(item.Item2, -1, false); + break; + + case Rom rom: + if (rom.GetStringFieldValue(Models.Metadata.Rom.StatusKey).AsEnumValue() == ItemStatus.Nodump) + fieldDats[Models.Metadata.Rom.StatusKey].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA512Key))) + fieldDats[Models.Metadata.Rom.SHA512Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA384Key))) + fieldDats[Models.Metadata.Rom.SHA384Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA256Key))) + fieldDats[Models.Metadata.Rom.SHA256Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key))) + fieldDats[Models.Metadata.Rom.SHA1Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.MD5Key))) + fieldDats[Models.Metadata.Rom.MD5Key].ItemsDB.AddItem(item.Item2, -1, false); + else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey))) + fieldDats[Models.Metadata.Rom.CRCKey].ItemsDB.AddItem(item.Item2, -1, false); + else + fieldDats["null"].ItemsDB.AddItem(item.Item2, -1, false); + break; + + default: + continue; + } + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + + watch.Stop(); + return fieldDats; + } + /// /// Split a SuperDAT by lowest available directory level /// @@ -435,6 +647,74 @@ namespace SabreTools.DatTools return (lessThan, greaterThan); } + /// + /// Split a DAT by size of Rom + /// + /// Current DatFile object to split + /// Long value representing the split point + /// Less Than and Greater Than DatFiles + public static (DatFile lessThan, DatFile greaterThan) SplitBySizeDB(DatFile datFile, long radix) + { + // Create each of the respective output DATs + var watch = new InternalStopwatch($"Splitting DAT by size"); + + DatFile lessThan = DatFile.Create(datFile.Header.CloneStandard()); + lessThan.Header.SetFieldValue(DatHeader.FileNameKey, lessThan.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" (less than {radix})"); + lessThan.Header.SetFieldValue(Models.Metadata.Header.NameKey, lessThan.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" (less than {radix})"); + lessThan.Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, lessThan.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" (less than {radix})"); + + DatFile greaterThan = DatFile.Create(datFile.Header.CloneStandard()); + greaterThan.Header.SetFieldValue(DatHeader.FileNameKey, greaterThan.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" (equal-greater than {radix})"); + greaterThan.Header.SetFieldValue(Models.Metadata.Header.NameKey, greaterThan.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" (equal-greater than {radix})"); + greaterThan.Header.SetFieldValue(Models.Metadata.Header.DescriptionKey, greaterThan.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" (equal-greater than {radix})"); + + // Now populate each of the DAT objects in turn +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(datFile.ItemsDB.SortedKeys, key => +#else + foreach (var key in datFile.ItemsDB.SortedKeys) +#endif + { + var items = datFile.ItemsDB.GetItemsForBucket(key); + if (items == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + // TODO: Determine how we figure out the machine index + foreach ((long, DatItem) item in items) + { + // If the file is not a Rom, it automatically goes in the "lesser" dat + if (item.Item2 is not Rom rom) + lessThan.ItemsDB.AddItem(item.Item2, -1, false); + + // If the file is a Rom and has no size, put it in the "lesser" dat + else if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) == null) + lessThan.ItemsDB.AddItem(item.Item2, -1, false); + + // If the file is a Rom and less than the radix, put it in the "lesser" dat + else if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) < radix) + lessThan.ItemsDB.AddItem(item.Item2, -1, false); + + // If the file is a Rom and greater than or equal to the radix, put it in the "greater" dat + else if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) >= radix) + greaterThan.ItemsDB.AddItem(item.Item2, -1, false); + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } +#endif + + // Then return both DatFiles + watch.Stop(); + return (lessThan, greaterThan); + } + /// /// Split a DAT by size of Rom /// @@ -565,6 +845,7 @@ namespace SabreTools.DatTools #endif { FillWithItemType(datFile, typeDats[itemType], itemType); + FillWithItemTypeDB(datFile, typeDats[itemType], itemType); #if NET40_OR_GREATER || NETCOREAPP }); #else @@ -612,6 +893,56 @@ namespace SabreTools.DatTools }); #else } +#endif + } + + /// + /// Fill a DatFile with all items with a particular ItemType + /// + /// Current DatFile object to split + /// DatFile to add found items to + /// ItemType to retrieve items for + /// DatFile containing all items with the ItemType/returns> + private static void FillWithItemTypeDB(DatFile datFile, DatFile indexDat, ItemType itemType) + { + // Loop through and add the items for this index to the output +#if NET452_OR_GREATER || NETCOREAPP + Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key => +#elif NET40_OR_GREATER + Parallel.ForEach(datFile.ItemsDB.SortedKeys, key => +#else + foreach (var key in datFile.ItemsDB.SortedKeys) +#endif + { + // Get the current items + var datItems = datFile.ItemsDB.GetItemsForBucket(key); + if (datItems == null) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + ConcurrentList items = DatItem.Merge(datItems.Select(i => i.Item2).ToConcurrentList()); + + // If the rom list is empty or null, just skip it + if (items == null || items.Count == 0) +#if NET40_OR_GREATER || NETCOREAPP + return; +#else + continue; +#endif + + foreach (DatItem item in items) + { + // TODO: Determine how we figure out the machine index + if (item.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue() == itemType) + indexDat.ItemsDB.AddItem(item, -1, false); + } +#if NET40_OR_GREATER || NETCOREAPP + }); +#else + } #endif } } diff --git a/SabreTools.DatTools/Verification.cs b/SabreTools.DatTools/Verification.cs index 6a84a9b4..ba3a9d65 100644 --- a/SabreTools.DatTools/Verification.cs +++ b/SabreTools.DatTools/Verification.cs @@ -268,7 +268,7 @@ namespace SabreTools.DatTools var keys = datFile.ItemsDB.SortedKeys.ToList(); foreach (string key in keys) { - var items = datFile.ItemsDB.GetDatItemsForBucket(key); + var items = datFile.ItemsDB.GetItemsForBucket(key); if (items == null) continue; diff --git a/SabreTools.Filtering/Cleaner.cs b/SabreTools.Filtering/Cleaner.cs index 574f51d3..a48b596a 100644 --- a/SabreTools.Filtering/Cleaner.cs +++ b/SabreTools.Filtering/Cleaner.cs @@ -209,7 +209,7 @@ namespace SabreTools.Filtering foreach (string key in keys) { // For every item in the current key - var items = datFile.ItemsDB.GetDatItemsForBucket(key); + var items = datFile.ItemsDB.GetItemsForBucket(key); if (items == null) continue; diff --git a/SabreTools.Filtering/ExtraIni.cs b/SabreTools.Filtering/ExtraIni.cs index 2558eea1..7fc961a3 100644 --- a/SabreTools.Filtering/ExtraIni.cs +++ b/SabreTools.Filtering/ExtraIni.cs @@ -171,7 +171,7 @@ namespace SabreTools.Filtering foreach (string game in games) { // Get the list of DatItems for the machine - var datItems = datFile.ItemsDB.GetDatItemsForBucket(game); + var datItems = datFile.ItemsDB.GetItemsForBucket(game); if (datItems == null) continue; diff --git a/SabreTools/Features/Split.cs b/SabreTools/Features/Split.cs index d5e12fcd..16586119 100644 --- a/SabreTools/Features/Split.cs +++ b/SabreTools/Features/Split.cs @@ -76,6 +76,7 @@ namespace SabreTools.Features if (splittingMode.HasFlag(SplittingMode.Extension)) { (DatFile? extADat, DatFile? extBDat) = DatTools.Splitter.SplitByExtension(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue)); + //(DatFile? extADat, DatFile? extBDat) = DatTools.Splitter.SplitByExtensionDB(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue)); if (extADat != null && extBDat != null) { var watch = new InternalStopwatch("Outputting extension-split DATs"); @@ -92,6 +93,7 @@ namespace SabreTools.Features if (splittingMode.HasFlag(SplittingMode.Hash)) { Dictionary typeDats = DatTools.Splitter.SplitByHash(internalDat); + //Dictionary typeDats = DatTools.Splitter.SplitByHashDB(internalDat); var watch = new InternalStopwatch("Outputting hash-split DATs"); @@ -129,6 +131,7 @@ namespace SabreTools.Features if (splittingMode.HasFlag(SplittingMode.Size)) { (DatFile lessThan, DatFile greaterThan) = DatTools.Splitter.SplitBySize(internalDat, GetInt64(features, RadixInt64Value)); + //(DatFile lessThan, DatFile greaterThan) = DatTools.Splitter.SplitBySizeDB(internalDat, GetInt64(features, RadixInt64Value)); var watch = new InternalStopwatch("Outputting size-split DATs");