claunia/SabreTools
1
0
Fork 0
mirror of https://github.com/claunia/SabreTools.git synced 2025-12-16 19:14:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add more IDDB accessors; make more IDDB variants of methods

Browse Source
This commit is contained in:
Matt Nadareski
2024-03-19 23:35:29 -04:00
parent efc184627a
commit c2ca5a0a6d
7 changed files with 418 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
SabreTools.DatFiles/DatFile.Removal.cs
View File

@@ -74,7 +74,7 @@ namespace SabreTools.DatFiles
foreach (var key in ItemsDB.SortedKeys)
#endif
{
var items = ItemsDB.GetDatItemsForBucket(key);
var items = ItemsDB.GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;

148
SabreTools.DatFiles/ItemDictionaryDB.cs
View File

@@ -305,6 +305,51 @@ namespace SabreTools.DatFiles
return _items[index];
}
/// <summary>
/// Get all items from the current dictionary
/// </summary>
public DatItem[] GetItems()
=> _items.Select(kvp => kvp.Value).ToArray();
/// <summary>
/// Get the indices and items associated with a bucket name
/// </summary>
public (long, DatItem)[]? GetItemsForBucket(string bucketName, bool filter = false)
{
if (!_buckets.ContainsKey(bucketName))
return null;
var itemIds = _buckets[bucketName];
var datItems = new List<(long, DatItem)>();
foreach (long itemId in itemIds)
{
if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true))
datItems.Add((itemId, _items[itemId]));
}
return [.. datItems];
}
/// <summary>
/// Get the indices and items associated with a machine index
/// </summary>
public (long, DatItem)[]? GetItemsForMachine(long machineIndex, bool filter = false)
{
var itemIds = _itemToMachineMapping
.Where(mapping => mapping.Value == machineIndex)
.Select(mapping => mapping.Key);
var datItems = new List<(long, DatItem)>();
foreach (long itemId in itemIds)
{
if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true))
datItems.Add((itemId, _items[itemId]));
}
return [.. datItems];
}
/// <summary>
/// Get a machine based on the index
/// </summary>
@@ -345,43 +390,10 @@ namespace SabreTools.DatFiles
}
/// <summary>
/// Get the indices and items associated with a bucket name
/// Get all machines from the current dictionary
/// </summary>
public (long, DatItem)[]? GetDatItemsForBucket(string bucketName, bool filter = false)
{
if (!_buckets.ContainsKey(bucketName))
return null;
var itemIds = _buckets[bucketName];
var datItems = new List<(long, DatItem)>();
foreach (long itemId in itemIds)
{
if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true))
datItems.Add((itemId, _items[itemId]));
}
return [.. datItems];
}
/// <summary>
/// Get the indices and items associated with a machine index
/// </summary>
public (long, DatItem)[]? GetDatItemsForMachine(long machineIndex, bool filter = false)
{
var itemIds = _itemToMachineMapping
.Where(mapping => mapping.Value == machineIndex)
.Select(mapping => mapping.Key);
var datItems = new List<(long, DatItem)>();
foreach (long itemId in itemIds)
{
if (_items.ContainsKey(itemId) && (!filter || _items[itemId].GetBoolFieldValue(DatItem.RemoveKey) != true))
datItems.Add((itemId, _items[itemId]));
}
return [.. datItems];
}
public Machine[] GetMachines()
=> _machines.Select(kvp => kvp.Value).ToArray();
/// <summary>
/// Remove an item, returning if it could be removed
@@ -518,7 +530,7 @@ namespace SabreTools.DatFiles
string key = SortAndGetKey(datItem, sorted);
// If the key doesn't exist, return the empty list
var roms = GetDatItemsForBucket(key);
var roms = GetItemsForBucket(key);
if (roms == null || roms.Length == 0)
return output;
@@ -562,7 +574,7 @@ namespace SabreTools.DatFiles
string key = SortAndGetKey(datItem, sorted);
// If the key doesn't exist
var roms = GetDatItemsForBucket(key);
var roms = GetItemsForBucket(key);
if (roms == null || roms.Length == 0)
return false;
@@ -1115,7 +1127,7 @@ namespace SabreTools.DatFiles
Dictionary<string, List<string>> parents = [];
foreach (string key in SortedKeys)
{
var items = GetDatItemsForBucket(key);
var items = GetItemsForBucket(key);
if (items == null || items.Length == 0)
continue;
@@ -1193,7 +1205,7 @@ namespace SabreTools.DatFiles
foreach (var key in SortedKeys)
#endif
{
var items = GetDatItemsForBucket(key);
var items = GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
@@ -1229,7 +1241,7 @@ namespace SabreTools.DatFiles
foreach (var key in SortedKeys)
#endif
{
var items = GetDatItemsForBucket(key);
var items = GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
@@ -1283,7 +1295,7 @@ namespace SabreTools.DatFiles
foreach (var key in SortedKeys)
#endif
{
var items = GetDatItemsForBucket(key);
var items = GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
@@ -1323,7 +1335,7 @@ namespace SabreTools.DatFiles
/// <param name="bucketName">Name of the bucket to filter on</param>
private void ExecuteFilterOnBucket(FilterRunner filterRunner, string bucketName)
{
(long, DatItem)[]? items = GetDatItemsForBucket(bucketName);
(long, DatItem)[]? items = GetItemsForBucket(bucketName);
if (items == null)
return;
@@ -1381,7 +1393,7 @@ namespace SabreTools.DatFiles
foreach (var key in SortedKeys)
#endif
{
var items = GetDatItemsForBucket(key);
var items = GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
@@ -1438,7 +1450,7 @@ namespace SabreTools.DatFiles
List<string> games = [.. SortedKeys];
foreach (string game in games)
{
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
// If the game has no items in it, we want to continue
if (items == null || items.Length == 0)
@@ -1463,7 +1475,7 @@ namespace SabreTools.DatFiles
continue;
// If the parent doesn't have any items, we want to continue
var parentItems = GetDatItemsForBucket(parent.Item2!);
var parentItems = GetItemsForBucket(parent.Item2!);
if (parentItems == null || parentItems.Length == 0)
continue;
@@ -1489,7 +1501,7 @@ namespace SabreTools.DatFiles
foreach (string game in games)
{
// Get the items for this game
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
// If the machine doesn't have items, we continue
if (items == null || items.Length == 0)
@@ -1534,7 +1546,7 @@ namespace SabreTools.DatFiles
continue;
// If the machine doesn't exist then we continue
var devItems = GetDatItemsForBucket(deviceReference);
var devItems = GetItemsForBucket(deviceReference);
if (devItems == null || devItems.Length == 0)
continue;
@@ -1544,14 +1556,14 @@ namespace SabreTools.DatFiles
.Select(i => (i.Item2 as DeviceRef)!.GetName()!));
// Set new machine information and add to the current machine
var copyFrom = GetMachineForItem(GetDatItemsForBucket(game)![0].Item1);
var copyFrom = GetMachineForItem(GetItemsForBucket(game)![0].Item1);
if (copyFrom.Item2 == null)
continue;
foreach ((long, DatItem) item in devItems)
{
// If the parent machine doesn't already contain this item, add it
if (!GetDatItemsForBucket(game)!
if (!GetItemsForBucket(game)!
.Any(i => i.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey) == item.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey)
&& i.Item2.GetName() == item.Item2.GetName()))
{
@@ -1589,7 +1601,7 @@ namespace SabreTools.DatFiles
continue;
// If the machine doesn't exist then we continue
var slotItems = GetDatItemsForBucket(slotOption);
var slotItems = GetItemsForBucket(slotOption);
if (slotItems == null || slotItems.Length == 0)
continue;
@@ -1601,14 +1613,14 @@ namespace SabreTools.DatFiles
.Select(o => o.GetStringFieldValue(Models.Metadata.SlotOption.DevNameKey)!));
// Set new machine information and add to the current machine
var copyFrom = GetMachineForItem(GetDatItemsForBucket(game)![0].Item1);
var copyFrom = GetMachineForItem(GetItemsForBucket(game)![0].Item1);
if (copyFrom.Item2 == null)
continue;
foreach ((long, DatItem) item in slotItems)
{
// If the parent machine doesn't already contain this item, add it
if (!GetDatItemsForBucket(game)!
if (!GetItemsForBucket(game)!
.Any(i => i.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey) == item.Item2.GetStringFieldValue(Models.Metadata.DatItem.TypeKey)
&& i.Item2.GetName() == item.Item2.GetName()))
{
@@ -1651,7 +1663,7 @@ namespace SabreTools.DatFiles
foreach (string game in games)
{
// If the game has no items in it, we want to continue
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
if (items == null || items.Length == 0)
continue;
@@ -1674,7 +1686,7 @@ namespace SabreTools.DatFiles
continue;
// If the parent doesn't have any items, we want to continue
var parentItems = GetDatItemsForBucket(parent.Item2!);
var parentItems = GetItemsForBucket(parent.Item2!);
if (parentItems == null || parentItems.Length == 0)
continue;
@@ -1690,12 +1702,12 @@ namespace SabreTools.DatFiles
}
// Get the parent machine
var parentMachine = GetMachineForItem(GetDatItemsForBucket(parent.Item2!)![0].Item1);
var parentMachine = GetMachineForItem(GetItemsForBucket(parent.Item2!)![0].Item1);
if (parentMachine.Item2 == null)
continue;
// Now we want to get the parent romof tag and put it in each of the items
items = GetDatItemsForBucket(game);
items = GetItemsForBucket(game);
string? romof = parentMachine.Item2.GetStringFieldValue(Models.Metadata.Machine.RomOfKey);
foreach ((long, DatItem) item in items!)
{
@@ -1719,7 +1731,7 @@ namespace SabreTools.DatFiles
foreach (string game in games)
{
// If the game has no items in it, we want to continue
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
if (items == null || items.Length == 0)
continue;
@@ -1741,11 +1753,11 @@ namespace SabreTools.DatFiles
if (string.IsNullOrEmpty(parent.Item2))
continue;
items = GetDatItemsForBucket(game);
items = GetItemsForBucket(game);
foreach ((long, DatItem) item in items!)
{
// Get the parent items and current machine name
var parentItems = GetDatItemsForBucket(parent.Item2!);
var parentItems = GetItemsForBucket(parent.Item2!);
string? machineName = GetMachineForItem(item.Item1).Item2?.GetStringFieldValue(Models.Metadata.Machine.NameKey);
// Special disk handling
@@ -1844,7 +1856,7 @@ namespace SabreTools.DatFiles
List<string> games = [.. SortedKeys];
foreach (string game in games)
{
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
if (items == null || items.Length == 0)
continue;
@@ -1874,7 +1886,7 @@ namespace SabreTools.DatFiles
foreach (string game in games)
{
// If the game has no items in it, we want to continue
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
if (items == null || items.Length == 0)
continue;
@@ -1897,7 +1909,7 @@ namespace SabreTools.DatFiles
continue;
// If the parent doesn't have any items, we want to continue
var parentItems = GetDatItemsForBucket(parent!);
var parentItems = GetItemsForBucket(parent!);
if (parentItems == null || parentItems.Length == 0)
continue;
@@ -1921,7 +1933,7 @@ namespace SabreTools.DatFiles
List<string> games = [.. SortedKeys];
foreach (string game in games)
{
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
if (items == null)
continue;
@@ -1944,7 +1956,7 @@ namespace SabreTools.DatFiles
continue;
// If the parent doesn't have any items, we want to continue
var parentItems = GetDatItemsForBucket(parent!);
var parentItems = GetItemsForBucket(parent!);
if (parentItems == null || parentItems.Length == 0)
continue;
@@ -1959,8 +1971,8 @@ namespace SabreTools.DatFiles
}
// Now we want to get the parent romof tag and put it in each of the remaining items
items = GetDatItemsForBucket(game);
machine = GetMachineForItem(GetDatItemsForBucket(parent!)![0].Item1);
items = GetItemsForBucket(game);
machine = GetMachineForItem(GetItemsForBucket(parent!)![0].Item1);
if (machine.Item2 == null)
continue;
@@ -1984,7 +1996,7 @@ namespace SabreTools.DatFiles
List<string> games = [.. SortedKeys];
foreach (string game in games)
{
var items = GetDatItemsForBucket(game);
var items = GetItemsForBucket(game);
if (items == null)
continue;

331
SabreTools.DatTools/Splitter.cs
View File

@@ -109,6 +109,84 @@ namespace SabreTools.DatTools
return (extADat, extBDat);
}
/// <summary>
/// Split a DAT by input extensions
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="extA">List of extensions to split on (first DAT)</param>
/// <param name="extB">List of extensions to split on (second DAT)</param>
/// <returns>Extension Set A and Extension Set B DatFiles</returns>
public static (DatFile? extADat, DatFile? extBDat) SplitByExtensionDB(DatFile datFile, List<string> extA, List<string> extB)
{
// If roms is empty, return false
if (datFile.ItemsDB.DatStatistics.TotalCount == 0)
return (null, null);
InternalStopwatch watch = new($"Splitting DAT by extension");
// Make sure all of the extensions don't have a dot at the beginning
var newExtA = extA.Select(s => s.TrimStart('.').ToLowerInvariant()).ToArray();
string newExtAString = string.Join(",", newExtA);
var newExtB = extB.Select(s => s.TrimStart('.').ToLowerInvariant()).ToArray();
string newExtBString = string.Join(",", newExtB);
// Set all of the appropriate outputs for each of the subsets
DatFile extADat = DatFile.Create(datFile.Header.CloneStandard());
extADat.Header.SetFieldValue<string?>(DatHeader.FileNameKey, extADat.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" ({newExtAString})");
extADat.Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, extADat.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" ({newExtAString})");
extADat.Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, extADat.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" ({newExtAString})");
DatFile extBDat = DatFile.Create(datFile.Header.CloneStandard());
extBDat.Header.SetFieldValue<string?>(DatHeader.FileNameKey, extBDat.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" ({newExtBString})");
extBDat.Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, extBDat.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" ({newExtBString})");
extBDat.Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, extBDat.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" ({newExtBString})");
// Now separate the roms accordingly
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
Parallel.ForEach(datFile.ItemsDB.SortedKeys, key =>
#else
foreach (var key in datFile.ItemsDB.SortedKeys)
#endif
{
var items = datFile.ItemsDB.GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
// TODO: Determine how we figure out the machine index
foreach ((long, DatItem) item in items)
{
if (newExtA.Contains((item.Item2.GetName() ?? string.Empty).GetNormalizedExtension()))
{
extADat.ItemsDB.AddItem(item.Item2, -1, false);
}
else if (newExtB.Contains((item.Item2.GetName() ?? string.Empty).GetNormalizedExtension()))
{
extBDat.ItemsDB.AddItem(item.Item2, -1, false);
}
else
{
extADat.ItemsDB.AddItem(item.Item2, -1, false);
extBDat.ItemsDB.AddItem(item.Item2, -1, false);
}
}
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
// Then return both DatFiles
watch.Stop();
return (extADat, extBDat);
}
/// <summary>
/// Split a DAT by best available hashes
/// </summary>
@@ -241,6 +319,140 @@ namespace SabreTools.DatTools
return fieldDats;
}
/// <summary>
/// Split a DAT by best available hashes
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <returns>Dictionary of Field to DatFile mappings</returns>
public static Dictionary<string, DatFile> SplitByHashDB(DatFile datFile)
{
// Create each of the respective output DATs
var watch = new InternalStopwatch($"Splitting DAT by best available hashes");
// Create the set of field-to-dat mappings
Dictionary<string, DatFile> fieldDats = [];
// TODO: Can this be made into a loop?
fieldDats[Models.Metadata.Rom.StatusKey] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Models.Metadata.Rom.StatusKey].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.StatusKey].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (Nodump)");
fieldDats[Models.Metadata.Rom.StatusKey].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.StatusKey].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (Nodump)");
fieldDats[Models.Metadata.Rom.StatusKey].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.StatusKey].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (Nodump)");
fieldDats[Models.Metadata.Rom.SHA512Key] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Models.Metadata.Rom.SHA512Key].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA512Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-512)");
fieldDats[Models.Metadata.Rom.SHA512Key].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA512Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-512)");
fieldDats[Models.Metadata.Rom.SHA512Key].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA512Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-512)");
fieldDats[Models.Metadata.Rom.SHA384Key] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Models.Metadata.Rom.SHA384Key].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA384Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-384)");
fieldDats[Models.Metadata.Rom.SHA384Key].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA384Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-384)");
fieldDats[Models.Metadata.Rom.SHA384Key].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA384Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-384)");
fieldDats[Models.Metadata.Rom.SHA256Key] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Models.Metadata.Rom.SHA256Key].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA256Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-256)");
fieldDats[Models.Metadata.Rom.SHA256Key].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA256Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-256)");
fieldDats[Models.Metadata.Rom.SHA256Key].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA256Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-256)");
fieldDats[Models.Metadata.Rom.SHA1Key] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Models.Metadata.Rom.SHA1Key].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.SHA1Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (SHA-1)");
fieldDats[Models.Metadata.Rom.SHA1Key].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.SHA1Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (SHA-1)");
fieldDats[Models.Metadata.Rom.SHA1Key].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.SHA1Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (SHA-1)");
fieldDats[Models.Metadata.Rom.MD5Key] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Models.Metadata.Rom.MD5Key].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.MD5Key].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (MD5)");
fieldDats[Models.Metadata.Rom.MD5Key].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.MD5Key].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (MD5)");
fieldDats[Models.Metadata.Rom.MD5Key].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.MD5Key].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (MD5)");
fieldDats[Models.Metadata.Rom.CRCKey] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats[Models.Metadata.Rom.CRCKey].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats[Models.Metadata.Rom.CRCKey].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (CRC)");
fieldDats[Models.Metadata.Rom.CRCKey].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats[Models.Metadata.Rom.CRCKey].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (CRC)");
fieldDats[Models.Metadata.Rom.CRCKey].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats[Models.Metadata.Rom.CRCKey].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (CRC)");
fieldDats["null"] = DatFile.Create(datFile.Header.CloneStandard());
fieldDats["null"].Header.SetFieldValue<string?>(DatHeader.FileNameKey, fieldDats["null"].Header.GetStringFieldValue(DatHeader.FileNameKey) + " (Other)");
fieldDats["null"].Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, fieldDats["null"].Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + " (Other)");
fieldDats["null"].Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, fieldDats["null"].Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + " (Other)");
// Now populate each of the DAT objects in turn
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
Parallel.ForEach(datFile.ItemsDB.SortedKeys, key =>
#else
foreach (var key in datFile.ItemsDB.SortedKeys)
#endif
{
var items = datFile.ItemsDB.GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
// TODO: Determine how we figure out the machine index
foreach ((long, DatItem) item in items)
{
// If the file is not a Disk, Media, or Rom, continue
switch (item.Item2)
{
case Disk disk:
if (disk.GetStringFieldValue(Models.Metadata.Disk.StatusKey).AsEnumValue<ItemStatus>() == ItemStatus.Nodump)
fieldDats[Models.Metadata.Disk.StatusKey].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(disk.GetStringFieldValue(Models.Metadata.Disk.SHA1Key)))
fieldDats[Models.Metadata.Disk.SHA1Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(disk.GetStringFieldValue(Models.Metadata.Disk.MD5Key)))
fieldDats[Models.Metadata.Disk.MD5Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(disk.GetStringFieldValue(Models.Metadata.Disk.MD5Key)))
fieldDats[Models.Metadata.Disk.MD5Key].ItemsDB.AddItem(item.Item2, -1, false);
else
fieldDats["null"].ItemsDB.AddItem(item.Item2, -1, false);
break;
case Media media:
if (!string.IsNullOrEmpty(media.GetStringFieldValue(Models.Metadata.Media.SHA256Key)))
fieldDats[Models.Metadata.Media.SHA256Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(media.GetStringFieldValue(Models.Metadata.Media.SHA1Key)))
fieldDats[Models.Metadata.Media.SHA1Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(media.GetStringFieldValue(Models.Metadata.Media.MD5Key)))
fieldDats[Models.Metadata.Media.MD5Key].ItemsDB.AddItem(item.Item2, -1, false);
else
fieldDats["null"].ItemsDB.AddItem(item.Item2, -1, false);
break;
case Rom rom:
if (rom.GetStringFieldValue(Models.Metadata.Rom.StatusKey).AsEnumValue<ItemStatus>() == ItemStatus.Nodump)
fieldDats[Models.Metadata.Rom.StatusKey].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA512Key)))
fieldDats[Models.Metadata.Rom.SHA512Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA384Key)))
fieldDats[Models.Metadata.Rom.SHA384Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA256Key)))
fieldDats[Models.Metadata.Rom.SHA256Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.SHA1Key)))
fieldDats[Models.Metadata.Rom.SHA1Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.MD5Key)))
fieldDats[Models.Metadata.Rom.MD5Key].ItemsDB.AddItem(item.Item2, -1, false);
else if (!string.IsNullOrEmpty(rom.GetStringFieldValue(Models.Metadata.Rom.CRCKey)))
fieldDats[Models.Metadata.Rom.CRCKey].ItemsDB.AddItem(item.Item2, -1, false);
else
fieldDats["null"].ItemsDB.AddItem(item.Item2, -1, false);
break;
default:
continue;
}
}
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
watch.Stop();
return fieldDats;
}
/// <summary>
/// Split a SuperDAT by lowest available directory level
/// </summary>
@@ -435,6 +647,74 @@ namespace SabreTools.DatTools
return (lessThan, greaterThan);
}
/// <summary>
/// Split a DAT by size of Rom
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="radix">Long value representing the split point</param>
/// <returns>Less Than and Greater Than DatFiles</returns>
public static (DatFile lessThan, DatFile greaterThan) SplitBySizeDB(DatFile datFile, long radix)
{
// Create each of the respective output DATs
var watch = new InternalStopwatch($"Splitting DAT by size");
DatFile lessThan = DatFile.Create(datFile.Header.CloneStandard());
lessThan.Header.SetFieldValue<string?>(DatHeader.FileNameKey, lessThan.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" (less than {radix})");
lessThan.Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, lessThan.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" (less than {radix})");
lessThan.Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, lessThan.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" (less than {radix})");
DatFile greaterThan = DatFile.Create(datFile.Header.CloneStandard());
greaterThan.Header.SetFieldValue<string?>(DatHeader.FileNameKey, greaterThan.Header.GetStringFieldValue(DatHeader.FileNameKey) + $" (equal-greater than {radix})");
greaterThan.Header.SetFieldValue<string?>(Models.Metadata.Header.NameKey, greaterThan.Header.GetStringFieldValue(Models.Metadata.Header.NameKey) + $" (equal-greater than {radix})");
greaterThan.Header.SetFieldValue<string?>(Models.Metadata.Header.DescriptionKey, greaterThan.Header.GetStringFieldValue(Models.Metadata.Header.DescriptionKey) + $" (equal-greater than {radix})");
// Now populate each of the DAT objects in turn
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
Parallel.ForEach(datFile.ItemsDB.SortedKeys, key =>
#else
foreach (var key in datFile.ItemsDB.SortedKeys)
#endif
{
var items = datFile.ItemsDB.GetItemsForBucket(key);
if (items == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
// TODO: Determine how we figure out the machine index
foreach ((long, DatItem) item in items)
{
// If the file is not a Rom, it automatically goes in the "lesser" dat
if (item.Item2 is not Rom rom)
lessThan.ItemsDB.AddItem(item.Item2, -1, false);
// If the file is a Rom and has no size, put it in the "lesser" dat
else if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) == null)
lessThan.ItemsDB.AddItem(item.Item2, -1, false);
// If the file is a Rom and less than the radix, put it in the "lesser" dat
else if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) < radix)
lessThan.ItemsDB.AddItem(item.Item2, -1, false);
// If the file is a Rom and greater than or equal to the radix, put it in the "greater" dat
else if (rom.GetInt64FieldValue(Models.Metadata.Rom.SizeKey) >= radix)
greaterThan.ItemsDB.AddItem(item.Item2, -1, false);
}
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
// Then return both DatFiles
watch.Stop();
return (lessThan, greaterThan);
}
/// <summary>
/// Split a DAT by size of Rom
/// </summary>
@@ -565,6 +845,7 @@ namespace SabreTools.DatTools
#endif
{
FillWithItemType(datFile, typeDats[itemType], itemType);
FillWithItemTypeDB(datFile, typeDats[itemType], itemType);
#if NET40_OR_GREATER || NETCOREAPP
});
#else
@@ -612,6 +893,56 @@ namespace SabreTools.DatTools
});
#else
}
#endif
}
/// <summary>
/// Fill a DatFile with all items with a particular ItemType
/// </summary>
/// <param name="datFile">Current DatFile object to split</param>
/// <param name="indexDat">DatFile to add found items to</param>
/// <param name="itemType">ItemType to retrieve items for</param>
/// <returns>DatFile containing all items with the ItemType/returns>
private static void FillWithItemTypeDB(DatFile datFile, DatFile indexDat, ItemType itemType)
{
// Loop through and add the items for this index to the output
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(datFile.ItemsDB.SortedKeys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
Parallel.ForEach(datFile.ItemsDB.SortedKeys, key =>
#else
foreach (var key in datFile.ItemsDB.SortedKeys)
#endif
{
// Get the current items
var datItems = datFile.ItemsDB.GetItemsForBucket(key);
if (datItems == null)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
ConcurrentList<DatItem> items = DatItem.Merge(datItems.Select(i => i.Item2).ToConcurrentList());
// If the rom list is empty or null, just skip it
if (items == null || items.Count == 0)
#if NET40_OR_GREATER || NETCOREAPP
return;
#else
continue;
#endif
foreach (DatItem item in items)
{
// TODO: Determine how we figure out the machine index
if (item.GetStringFieldValue(Models.Metadata.DatItem.TypeKey).AsEnumValue<ItemType>() == itemType)
indexDat.ItemsDB.AddItem(item, -1, false);
}
#if NET40_OR_GREATER || NETCOREAPP
});
#else
}
#endif
}
}

2
SabreTools.DatTools/Verification.cs
View File

@@ -268,7 +268,7 @@ namespace SabreTools.DatTools
var keys = datFile.ItemsDB.SortedKeys.ToList();
foreach (string key in keys)
{
var items = datFile.ItemsDB.GetDatItemsForBucket(key);
var items = datFile.ItemsDB.GetItemsForBucket(key);
if (items == null)
continue;

2
SabreTools.Filtering/Cleaner.cs
View File

@@ -209,7 +209,7 @@ namespace SabreTools.Filtering
foreach (string key in keys)
{
// For every item in the current key
var items = datFile.ItemsDB.GetDatItemsForBucket(key);
var items = datFile.ItemsDB.GetItemsForBucket(key);
if (items == null)
continue;

2
SabreTools.Filtering/ExtraIni.cs
View File

@@ -171,7 +171,7 @@ namespace SabreTools.Filtering
foreach (string game in games)
{
// Get the list of DatItems for the machine
var datItems = datFile.ItemsDB.GetDatItemsForBucket(game);
var datItems = datFile.ItemsDB.GetItemsForBucket(game);
if (datItems == null)
continue;

3
SabreTools/Features/Split.cs
View File

@@ -76,6 +76,7 @@ namespace SabreTools.Features
if (splittingMode.HasFlag(SplittingMode.Extension))
{
(DatFile? extADat, DatFile? extBDat) = DatTools.Splitter.SplitByExtension(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue));
//(DatFile? extADat, DatFile? extBDat) = DatTools.Splitter.SplitByExtensionDB(internalDat, GetList(features, ExtAListValue), GetList(features, ExtBListValue));
if (extADat != null && extBDat != null)
{
var watch = new InternalStopwatch("Outputting extension-split DATs");
@@ -92,6 +93,7 @@ namespace SabreTools.Features
if (splittingMode.HasFlag(SplittingMode.Hash))
{
Dictionary<string, DatFile> typeDats = DatTools.Splitter.SplitByHash(internalDat);
//Dictionary<string, DatFile> typeDats = DatTools.Splitter.SplitByHashDB(internalDat);
var watch = new InternalStopwatch("Outputting hash-split DATs");
@@ -129,6 +131,7 @@ namespace SabreTools.Features
if (splittingMode.HasFlag(SplittingMode.Size))
{
(DatFile lessThan, DatFile greaterThan) = DatTools.Splitter.SplitBySize(internalDat, GetInt64(features, RadixInt64Value));
//(DatFile lessThan, DatFile greaterThan) = DatTools.Splitter.SplitBySizeDB(internalDat, GetInt64(features, RadixInt64Value));
var watch = new InternalStopwatch("Outputting size-split DATs");