mirror of
https://github.com/claunia/SabreTools.git
synced 2025-12-16 19:14:27 +00:00
Retool ItemDictionaryDB for future plans
This commit is contained in:
@@ -1,20 +1,19 @@
|
|||||||
#if NET462_OR_GREATER || NETCOREAPP
|
using System.Collections;
|
||||||
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
using System;
|
using System.Collections.Concurrent;
|
||||||
using System.Collections;
|
#endif
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using System.IO;
|
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
#endif
|
||||||
using System.Xml.Serialization;
|
using System.Xml.Serialization;
|
||||||
using Microsoft.Data.Sqlite;
|
|
||||||
using Newtonsoft.Json;
|
using Newtonsoft.Json;
|
||||||
using SabreTools.Core;
|
using SabreTools.Core;
|
||||||
using SabreTools.Core.Tools;
|
using SabreTools.Core.Tools;
|
||||||
using SabreTools.DatItems;
|
using SabreTools.DatItems;
|
||||||
using SabreTools.DatItems.Formats;
|
using SabreTools.DatItems.Formats;
|
||||||
using SabreTools.Hashing;
|
using SabreTools.Hashing;
|
||||||
using SabreTools.IO;
|
|
||||||
using SabreTools.Logging;
|
using SabreTools.Logging;
|
||||||
using SabreTools.Matching;
|
using SabreTools.Matching;
|
||||||
|
|
||||||
@@ -39,14 +38,22 @@ namespace SabreTools.DatFiles
|
|||||||
private DedupeType mergedBy;
|
private DedupeType mergedBy;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Internal database connection for the class
|
/// Internal dictionary for all items
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private readonly SqliteConnection? dbc = null;
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
|
private readonly ConcurrentDictionary<string, ConcurrentList<DatItem>?> items;
|
||||||
|
#else
|
||||||
|
private readonly Dictionary<string, ConcurrentList<DatItem>?> items;
|
||||||
|
#endif
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Internal item dictionary name
|
/// Internal dictionary for all machines
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private string? itemDictionaryFileName = null;
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
|
private readonly ConcurrentDictionary<string, Machine> machines;
|
||||||
|
#else
|
||||||
|
private readonly Dictionary<string, Machine> machines;
|
||||||
|
#endif
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Lock for statistics calculation
|
/// Lock for statistics calculation
|
||||||
@@ -62,72 +69,16 @@ namespace SabreTools.DatFiles
|
|||||||
|
|
||||||
#region Publically available fields
|
#region Publically available fields
|
||||||
|
|
||||||
#region Database
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Item dictionary file name
|
|
||||||
/// </summary>
|
|
||||||
public string? ItemDictionaryFileName
|
|
||||||
{
|
|
||||||
get
|
|
||||||
{
|
|
||||||
if (string.IsNullOrEmpty(itemDictionaryFileName))
|
|
||||||
itemDictionaryFileName = Path.Combine(PathTool.GetRuntimeDirectory(), $"itemDictionary{Guid.NewGuid()}.sqlite");
|
|
||||||
|
|
||||||
return itemDictionaryFileName;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Item dictionary connection string
|
|
||||||
/// </summary>
|
|
||||||
public string ItemDictionaryConnectionString => $"Data Source={ItemDictionaryFileName};";
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region Keys
|
#region Keys
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Get the keys from the file database
|
/// Get the keys from the file dictionary
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>List of the keys</returns>
|
/// <returns>List of the keys</returns>
|
||||||
[JsonIgnore, XmlIgnore]
|
[JsonIgnore, XmlIgnore]
|
||||||
public ICollection<string> Keys
|
public ICollection<string> Keys
|
||||||
{
|
{
|
||||||
get
|
get { return items.Keys; }
|
||||||
{
|
|
||||||
// If we have no database connection, we can't do anything
|
|
||||||
if (dbc == null)
|
|
||||||
return Array.Empty<string>();
|
|
||||||
|
|
||||||
// Open the database connection
|
|
||||||
dbc.Open();
|
|
||||||
|
|
||||||
string query = $"SELECT key FROM keys";
|
|
||||||
SqliteCommand slc = new(query, dbc);
|
|
||||||
SqliteDataReader sldr = slc.ExecuteReader();
|
|
||||||
|
|
||||||
List<string> keys = GetKeys();
|
|
||||||
if (sldr.HasRows)
|
|
||||||
{
|
|
||||||
while (sldr.Read())
|
|
||||||
{
|
|
||||||
keys.Add(sldr.GetString(0));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dispose of database objects
|
|
||||||
slc.Dispose();
|
|
||||||
sldr.Dispose();
|
|
||||||
dbc.Close();
|
|
||||||
|
|
||||||
return keys;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<string> GetKeys()
|
|
||||||
{
|
|
||||||
return new List<string>();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -135,14 +86,11 @@ namespace SabreTools.DatFiles
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <returns>List of the keys in sorted order</returns>
|
/// <returns>List of the keys in sorted order</returns>
|
||||||
[JsonIgnore, XmlIgnore]
|
[JsonIgnore, XmlIgnore]
|
||||||
public List<string>? SortedKeys
|
public List<string> SortedKeys
|
||||||
{
|
{
|
||||||
get
|
get
|
||||||
{
|
{
|
||||||
if (Keys == null)
|
var keys = items.Keys.ToList();
|
||||||
return null;
|
|
||||||
|
|
||||||
var keys = Keys.ToList();
|
|
||||||
keys.Sort(new NaturalComparer());
|
keys.Sort(new NaturalComparer());
|
||||||
return keys;
|
return keys;
|
||||||
}
|
}
|
||||||
@@ -178,7 +126,7 @@ namespace SabreTools.DatFiles
|
|||||||
public long TotalSize { get; private set; } = 0;
|
public long TotalSize { get; private set; } = 0;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Number of hashes for each hash type
|
/// Number of items for each hash type
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[JsonIgnore, XmlIgnore]
|
[JsonIgnore, XmlIgnore]
|
||||||
public Dictionary<HashType, long> HashCounts { get; private set; } = [];
|
public Dictionary<HashType, long> HashCounts { get; private set; } = [];
|
||||||
@@ -215,64 +163,17 @@ namespace SabreTools.DatFiles
|
|||||||
// Ensure the key exists
|
// Ensure the key exists
|
||||||
EnsureKey(key);
|
EnsureKey(key);
|
||||||
|
|
||||||
// If we have no database connection, we can't do anything
|
|
||||||
if (dbc == null)
|
|
||||||
return null;
|
|
||||||
|
|
||||||
// Open the database connection
|
|
||||||
dbc.Open();
|
|
||||||
|
|
||||||
string query = $"SELECT item FROM groups WHERE key='{key}'";
|
|
||||||
SqliteCommand slc = new(query, dbc);
|
|
||||||
SqliteDataReader sldr = slc.ExecuteReader();
|
|
||||||
|
|
||||||
ConcurrentList<DatItem> items = new();
|
|
||||||
if (sldr.HasRows)
|
|
||||||
{
|
|
||||||
while (sldr.Read())
|
|
||||||
{
|
|
||||||
string itemString = sldr.GetString(0);
|
|
||||||
DatItem? datItem = JsonConvert.DeserializeObject<DatItem>(itemString);
|
|
||||||
if (datItem != null)
|
|
||||||
items.Add(datItem);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Dispose of database objects
|
|
||||||
slc.Dispose();
|
|
||||||
sldr.Dispose();
|
|
||||||
dbc.Close();
|
|
||||||
|
|
||||||
// Now return the value
|
// Now return the value
|
||||||
return items;
|
return items[key];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
set
|
set
|
||||||
{
|
{
|
||||||
Remove(key);
|
Remove(key);
|
||||||
if (value == null)
|
if (value == null)
|
||||||
{
|
items[key] = null;
|
||||||
// If we have no database connection, we can't do anything
|
|
||||||
if (dbc == null)
|
|
||||||
return;
|
|
||||||
|
|
||||||
// Open the database connection
|
|
||||||
dbc.Open();
|
|
||||||
|
|
||||||
// Now remove the value
|
|
||||||
string itemString = JsonConvert.SerializeObject(value);
|
|
||||||
string query = $"DELETE FROM groups WHERE key='{key}'";
|
|
||||||
SqliteCommand slc = new(query, dbc);
|
|
||||||
slc.ExecuteNonQuery();
|
|
||||||
|
|
||||||
// Dispose of database objects
|
|
||||||
slc.Dispose();
|
|
||||||
dbc.Close();
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
|
||||||
AddRange(key, value);
|
AddRange(key, value);
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -293,22 +194,8 @@ namespace SabreTools.DatFiles
|
|||||||
if (value == null)
|
if (value == null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// If we have no database connection, we can't do anything
|
|
||||||
if (dbc == null)
|
|
||||||
return;
|
|
||||||
|
|
||||||
// Open the database connection
|
|
||||||
dbc.Open();
|
|
||||||
|
|
||||||
// Now add the value
|
// Now add the value
|
||||||
string itemString = JsonConvert.SerializeObject(value);
|
items[key]!.Add(value);
|
||||||
string query = $"INSERT INTO groups (key, item) VALUES ('{key}', '{itemString}')";
|
|
||||||
SqliteCommand slc = new(query, dbc);
|
|
||||||
slc.ExecuteNonQuery();
|
|
||||||
|
|
||||||
// Dispose of database objects
|
|
||||||
slc.Dispose();
|
|
||||||
dbc.Close();
|
|
||||||
|
|
||||||
// Now update the statistics
|
// Now update the statistics
|
||||||
AddItemStatistics(value);
|
AddItemStatistics(value);
|
||||||
@@ -404,10 +291,7 @@ namespace SabreTools.DatFiles
|
|||||||
EnsureKey(key);
|
EnsureKey(key);
|
||||||
|
|
||||||
// Now add the value
|
// Now add the value
|
||||||
foreach (DatItem item in value)
|
items[key]!.AddRange(value);
|
||||||
{
|
|
||||||
Add(key, item);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Now update the statistics
|
// Now update the statistics
|
||||||
foreach (DatItem item in value)
|
foreach (DatItem item in value)
|
||||||
@@ -421,7 +305,7 @@ namespace SabreTools.DatFiles
|
|||||||
/// Add statistics from another DatStats object
|
/// Add statistics from another DatStats object
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="stats">DatStats object to add from</param>
|
/// <param name="stats">DatStats object to add from</param>
|
||||||
public void AddStatistics(ItemDictionaryDB stats)
|
public void AddStatistics(ItemDictionary stats)
|
||||||
{
|
{
|
||||||
TotalCount += stats.Count;
|
TotalCount += stats.Count;
|
||||||
|
|
||||||
@@ -464,7 +348,7 @@ namespace SabreTools.DatFiles
|
|||||||
// Explicit lock for some weird corner cases
|
// Explicit lock for some weird corner cases
|
||||||
lock (key)
|
lock (key)
|
||||||
{
|
{
|
||||||
return Keys?.Contains(key) == true;
|
return items.ContainsKey(key);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -483,11 +367,11 @@ namespace SabreTools.DatFiles
|
|||||||
// Explicit lock for some weird corner cases
|
// Explicit lock for some weird corner cases
|
||||||
lock (key)
|
lock (key)
|
||||||
{
|
{
|
||||||
if (Keys?.Contains(key) != true)
|
if (items.ContainsKey(key) && items[key] != null)
|
||||||
return false;
|
return items[key]!.Contains(value);
|
||||||
|
|
||||||
return this[key]?.Contains(value) == true;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -497,23 +381,12 @@ namespace SabreTools.DatFiles
|
|||||||
public void EnsureKey(string key)
|
public void EnsureKey(string key)
|
||||||
{
|
{
|
||||||
// If the key is missing from the dictionary, add it
|
// If the key is missing from the dictionary, add it
|
||||||
if (Keys?.Contains(key) == true)
|
if (!items.ContainsKey(key))
|
||||||
return;
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
|
items.TryAdd(key, []);
|
||||||
// If we have no database connection, we can't do anything
|
#else
|
||||||
if (dbc == null)
|
items[key] = [];
|
||||||
return;
|
#endif
|
||||||
|
|
||||||
// Open the database connection
|
|
||||||
dbc.Open();
|
|
||||||
|
|
||||||
string query = $"INSERT INTO keys (key) VALUES ('{key}')";
|
|
||||||
SqliteCommand slc = new(query, dbc);
|
|
||||||
slc.ExecuteNonQuery();
|
|
||||||
|
|
||||||
// Dispose of database objects
|
|
||||||
slc.Dispose();
|
|
||||||
dbc.Close();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -525,14 +398,14 @@ namespace SabreTools.DatFiles
|
|||||||
lock (key)
|
lock (key)
|
||||||
{
|
{
|
||||||
// Get the list, if possible
|
// Get the list, if possible
|
||||||
ConcurrentList<DatItem>? fi = this[key];
|
ConcurrentList<DatItem>? fi = items[key];
|
||||||
if (fi == null)
|
if (fi == null)
|
||||||
return new ConcurrentList<DatItem>();
|
return [];
|
||||||
|
|
||||||
// Filter the list
|
// Filter the list
|
||||||
return fi.Where(i => i != null)
|
return fi.Where(i => i != null)
|
||||||
.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)
|
.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)
|
||||||
.Where(i => i.GetFieldValue<Machine>(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey) != null)
|
.Where(i => i.GetFieldValue<Machine>(DatItem.MachineKey) != null)
|
||||||
.ToConcurrentList();
|
.ToConcurrentList();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -547,18 +420,21 @@ namespace SabreTools.DatFiles
|
|||||||
lock (key)
|
lock (key)
|
||||||
{
|
{
|
||||||
// If the key doesn't exist, return
|
// If the key doesn't exist, return
|
||||||
if (!ContainsKey(key) || this[key] == null)
|
if (!ContainsKey(key) || items[key] == null)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
// Remove the statistics first
|
// Remove the statistics first
|
||||||
foreach (DatItem item in this[key]!)
|
foreach (DatItem item in items[key]!)
|
||||||
{
|
{
|
||||||
RemoveItemStatistics(item);
|
RemoveItemStatistics(item);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove the key from the dictionary
|
// Remove the key from the dictionary
|
||||||
this[key] = null;
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
return true;
|
return items.TryRemove(key, out _);
|
||||||
|
#else
|
||||||
|
return items.Remove(key);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -573,29 +449,13 @@ namespace SabreTools.DatFiles
|
|||||||
lock (key)
|
lock (key)
|
||||||
{
|
{
|
||||||
// If the key and value doesn't exist, return
|
// If the key and value doesn't exist, return
|
||||||
if (!Contains(key, value))
|
if (!Contains(key, value) || items[key] == null)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
// Remove the statistics first
|
// Remove the statistics first
|
||||||
RemoveItemStatistics(value);
|
RemoveItemStatistics(value);
|
||||||
|
|
||||||
// If we have no database connection, we can't do anything
|
return items[key]!.Remove(value);
|
||||||
if (dbc == null)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
// Open the database connection
|
|
||||||
dbc.Open();
|
|
||||||
|
|
||||||
// Now remove the value
|
|
||||||
string itemString = JsonConvert.SerializeObject(value);
|
|
||||||
string query = $"DELETE FROM groups WHERE key='{key}' AND item='{itemString}'";
|
|
||||||
SqliteCommand slc = new(query, dbc);
|
|
||||||
slc.ExecuteNonQuery();
|
|
||||||
|
|
||||||
// Dispose of database objects
|
|
||||||
slc.Dispose();
|
|
||||||
dbc.Close();
|
|
||||||
return true;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -606,17 +466,17 @@ namespace SabreTools.DatFiles
|
|||||||
public bool Reset(string key)
|
public bool Reset(string key)
|
||||||
{
|
{
|
||||||
// If the key doesn't exist, return
|
// If the key doesn't exist, return
|
||||||
if (!ContainsKey(key) || this[key] == null)
|
if (!ContainsKey(key) || items[key] == null)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
// Remove the statistics first
|
// Remove the statistics first
|
||||||
foreach (DatItem item in this[key]!)
|
foreach (DatItem item in items[key]!)
|
||||||
{
|
{
|
||||||
RemoveItemStatistics(item);
|
RemoveItemStatistics(item);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove the key from the dictionary
|
// Remove the key from the dictionary
|
||||||
this[key] = null;
|
items[key] = [];
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -633,7 +493,7 @@ namespace SabreTools.DatFiles
|
|||||||
/// Remove from the statistics given a DatItem
|
/// Remove from the statistics given a DatItem
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="item">Item to remove info for</param>
|
/// <param name="item">Item to remove info for</param>
|
||||||
public void RemoveItemStatistics(DatItem? item)
|
public void RemoveItemStatistics(DatItem item)
|
||||||
{
|
{
|
||||||
// If we have a null item, we can't do anything
|
// If we have a null item, we can't do anything
|
||||||
if (item == null)
|
if (item == null)
|
||||||
@@ -861,54 +721,18 @@ namespace SabreTools.DatFiles
|
|||||||
{
|
{
|
||||||
bucketedBy = ItemKey.NULL;
|
bucketedBy = ItemKey.NULL;
|
||||||
mergedBy = DedupeType.None;
|
mergedBy = DedupeType.None;
|
||||||
dbc = new SqliteConnection(ItemDictionaryConnectionString);
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
|
items = new ConcurrentDictionary<string, ConcurrentList<DatItem>?>();
|
||||||
|
machines = new ConcurrentDictionary<string, Machine>();
|
||||||
|
#else
|
||||||
|
items = new Dictionary<string, ConcurrentList<DatItem>?>();
|
||||||
|
machines = new Dictionary<string, Machine>();
|
||||||
|
#endif
|
||||||
logger = new Logger(this);
|
logger = new Logger(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
|
|
||||||
#region Database
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Ensure that the database exists and has the proper schema
|
|
||||||
/// </summary>
|
|
||||||
protected void EnsureDatabase()
|
|
||||||
{
|
|
||||||
// Make sure the file exists
|
|
||||||
if (ItemDictionaryFileName != null && !System.IO.File.Exists(ItemDictionaryFileName))
|
|
||||||
System.IO.File.Create(ItemDictionaryFileName);
|
|
||||||
|
|
||||||
// If we have no database connection, we can't do anything
|
|
||||||
if (dbc == null)
|
|
||||||
return;
|
|
||||||
|
|
||||||
// Open the database connection
|
|
||||||
dbc.Open();
|
|
||||||
|
|
||||||
// Make sure the database has the correct schema
|
|
||||||
string query = @"
|
|
||||||
CREATE TABLE IF NOT EXISTS keys (
|
|
||||||
'key' TEXT NOT NULL
|
|
||||||
PRIMARY KEY (key)
|
|
||||||
)";
|
|
||||||
SqliteCommand slc = new(query, dbc);
|
|
||||||
slc.ExecuteNonQuery();
|
|
||||||
|
|
||||||
query = @"
|
|
||||||
CREATE TABLE IF NOT EXISTS groups (
|
|
||||||
'key' TEXT NOT NULL
|
|
||||||
'item` TEXT NOT NULL
|
|
||||||
)";
|
|
||||||
|
|
||||||
slc = new SqliteCommand(query, dbc);
|
|
||||||
slc.ExecuteNonQuery();
|
|
||||||
|
|
||||||
slc.Dispose();
|
|
||||||
dbc.Close();
|
|
||||||
}
|
|
||||||
|
|
||||||
#endregion
|
|
||||||
|
|
||||||
#region Custom Functionality
|
#region Custom Functionality
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
@@ -920,8 +744,12 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
|
/// <param name="norename">True if games should only be compared on game and file name, false if system and source are counted</param>
|
||||||
public void BucketBy(ItemKey bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true)
|
public void BucketBy(ItemKey bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true)
|
||||||
{
|
{
|
||||||
// If we have a situation where there's no database or no keys at all, we skip
|
// If we have a situation where there's no dictionary or no keys at all, we skip
|
||||||
if (dbc == null || Keys.Count == 0)
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
|
if (items == null || items.IsEmpty)
|
||||||
|
#else
|
||||||
|
if (items == null || items.Count == 0)
|
||||||
|
#endif
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// If the sorted type isn't the same, we want to sort the dictionary accordingly
|
// If the sorted type isn't the same, we want to sort the dictionary accordingly
|
||||||
@@ -936,7 +764,8 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
mergedBy = DedupeType.None;
|
mergedBy = DedupeType.None;
|
||||||
|
|
||||||
// First do the initial sort of all of the roms inplace
|
// First do the initial sort of all of the roms inplace
|
||||||
List<string> oldkeys = Keys.ToList();
|
List<string> oldkeys = [.. Keys];
|
||||||
|
|
||||||
#if NET452_OR_GREATER || NETCOREAPP
|
#if NET452_OR_GREATER || NETCOREAPP
|
||||||
Parallel.For(0, oldkeys.Count, Globals.ParallelOptions, k =>
|
Parallel.For(0, oldkeys.Count, Globals.ParallelOptions, k =>
|
||||||
#elif NET40_OR_GREATER
|
#elif NET40_OR_GREATER
|
||||||
@@ -947,11 +776,7 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
{
|
{
|
||||||
string key = oldkeys[k];
|
string key = oldkeys[k];
|
||||||
if (this[key] == null)
|
if (this[key] == null)
|
||||||
#if NET40_OR_GREATER || NETCOREAPP
|
Remove(key);
|
||||||
return;
|
|
||||||
#else
|
|
||||||
continue;
|
|
||||||
#endif
|
|
||||||
|
|
||||||
// Now add each of the roms to their respective keys
|
// Now add each of the roms to their respective keys
|
||||||
for (int i = 0; i < this[key]!.Count; i++)
|
for (int i = 0; i < this[key]!.Count; i++)
|
||||||
@@ -990,7 +815,7 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
// Set the sorted type
|
// Set the sorted type
|
||||||
mergedBy = dedupeType;
|
mergedBy = dedupeType;
|
||||||
|
|
||||||
List<string> keys = Keys.ToList();
|
List<string> keys = [.. Keys];
|
||||||
#if NET452_OR_GREATER || NETCOREAPP
|
#if NET452_OR_GREATER || NETCOREAPP
|
||||||
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
|
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
|
||||||
#elif NET40_OR_GREATER
|
#elif NET40_OR_GREATER
|
||||||
@@ -1002,7 +827,11 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
// Get the possibly unsorted list
|
// Get the possibly unsorted list
|
||||||
ConcurrentList<DatItem>? sortedlist = this[key]?.ToConcurrentList();
|
ConcurrentList<DatItem>? sortedlist = this[key]?.ToConcurrentList();
|
||||||
if (sortedlist == null)
|
if (sortedlist == null)
|
||||||
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
return;
|
return;
|
||||||
|
#else
|
||||||
|
continue;
|
||||||
|
#endif
|
||||||
|
|
||||||
// Sort the list of items to be consistent
|
// Sort the list of items to be consistent
|
||||||
DatItem.Sort(ref sortedlist, false);
|
DatItem.Sort(ref sortedlist, false);
|
||||||
@@ -1023,7 +852,7 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
// If the merge type is the same, we want to sort the dictionary to be consistent
|
// If the merge type is the same, we want to sort the dictionary to be consistent
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
List<string> keys = Keys.ToList();
|
List<string> keys = [.. Keys];
|
||||||
#if NET452_OR_GREATER || NETCOREAPP
|
#if NET452_OR_GREATER || NETCOREAPP
|
||||||
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
|
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
|
||||||
#elif NET40_OR_GREATER
|
#elif NET40_OR_GREATER
|
||||||
@@ -1051,20 +880,28 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public void ClearEmpty()
|
public void ClearEmpty()
|
||||||
{
|
{
|
||||||
var keys = Keys.Where(k => k != null).ToList();
|
var keys = items.Keys.Where(k => k != null).ToList();
|
||||||
foreach (string key in keys)
|
foreach (string key in keys)
|
||||||
{
|
{
|
||||||
// If the key doesn't exist, skip
|
// If the key doesn't exist, skip
|
||||||
if (!Keys.Contains(key))
|
if (!items.ContainsKey(key))
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
// If the value is null, remove
|
// If the value is null, remove
|
||||||
else if (this[key] == null)
|
else if (items[key] == null)
|
||||||
this[key] = null;
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
|
items.TryRemove(key, out _);
|
||||||
|
#else
|
||||||
|
items.Remove(key);
|
||||||
|
#endif
|
||||||
|
|
||||||
// If there are no non-blank items, remove
|
// If there are no non-blank items, remove
|
||||||
else if (!this[key]!.Any(i => i != null && i is not Blank))
|
else if (!items[key]!.Any(i => i != null && i is not Blank))
|
||||||
this[key] = null;
|
#if NET40_OR_GREATER || NETCOREAPP
|
||||||
|
items.TryRemove(key, out _);
|
||||||
|
#else
|
||||||
|
items.Remove(key);
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1073,13 +910,10 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public void ClearMarked()
|
public void ClearMarked()
|
||||||
{
|
{
|
||||||
if (Keys == null)
|
var keys = items.Keys.ToList();
|
||||||
return;
|
|
||||||
|
|
||||||
var keys = Keys.ToList();
|
|
||||||
foreach (string key in keys)
|
foreach (string key in keys)
|
||||||
{
|
{
|
||||||
ConcurrentList<DatItem>? oldItemList = this[key];
|
ConcurrentList<DatItem>? oldItemList = items[key];
|
||||||
ConcurrentList<DatItem>? newItemList = oldItemList?.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)?.ToConcurrentList();
|
ConcurrentList<DatItem>? newItemList = oldItemList?.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)?.ToConcurrentList();
|
||||||
|
|
||||||
Remove(key);
|
Remove(key);
|
||||||
@@ -1095,7 +929,7 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
/// <returns>List of matched DatItem objects</returns>
|
/// <returns>List of matched DatItem objects</returns>
|
||||||
public ConcurrentList<DatItem> GetDuplicates(DatItem datItem, bool sorted = false)
|
public ConcurrentList<DatItem> GetDuplicates(DatItem datItem, bool sorted = false)
|
||||||
{
|
{
|
||||||
ConcurrentList<DatItem> output = new();
|
ConcurrentList<DatItem> output = [];
|
||||||
|
|
||||||
// Check for an empty rom list first
|
// Check for an empty rom list first
|
||||||
if (TotalCount == 0)
|
if (TotalCount == 0)
|
||||||
@@ -1105,12 +939,15 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
string key = SortAndGetKey(datItem, sorted);
|
string key = SortAndGetKey(datItem, sorted);
|
||||||
|
|
||||||
// If the key doesn't exist, return the empty list
|
// If the key doesn't exist, return the empty list
|
||||||
if (!ContainsKey(key) || this[key] == null)
|
if (!ContainsKey(key))
|
||||||
return output;
|
return output;
|
||||||
|
|
||||||
// Try to find duplicates
|
// Try to find duplicates
|
||||||
ConcurrentList<DatItem> roms = this[key]!;
|
ConcurrentList<DatItem>? roms = this[key];
|
||||||
ConcurrentList<DatItem> left = new();
|
if (roms == null)
|
||||||
|
return output;
|
||||||
|
|
||||||
|
ConcurrentList<DatItem> left = [];
|
||||||
for (int i = 0; i < roms.Count; i++)
|
for (int i = 0; i < roms.Count; i++)
|
||||||
{
|
{
|
||||||
DatItem other = roms[i];
|
DatItem other = roms[i];
|
||||||
@@ -1169,13 +1006,13 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
ResetStatistics();
|
ResetStatistics();
|
||||||
|
|
||||||
// If we have a blank Dat in any way, return
|
// If we have a blank Dat in any way, return
|
||||||
if (dbc == null || Keys == null)
|
if (items == null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
// Loop through and add
|
// Loop through and add
|
||||||
foreach (string key in Keys)
|
foreach (string key in items.Keys)
|
||||||
{
|
{
|
||||||
ConcurrentList<DatItem>? datItems = this[key];
|
ConcurrentList<DatItem>? datItems = items[key];
|
||||||
if (datItems == null)
|
if (datItems == null)
|
||||||
continue;
|
continue;
|
||||||
|
|
||||||
@@ -1256,30 +1093,52 @@ CREATE TABLE IF NOT EXISTS groups (
|
|||||||
|
|
||||||
#region IDictionary Implementations
|
#region IDictionary Implementations
|
||||||
|
|
||||||
public ICollection<ConcurrentList<DatItem>?> Values => throw new NotImplementedException();
|
public ICollection<ConcurrentList<DatItem>?> Values => ((IDictionary<string, ConcurrentList<DatItem>?>)items).Values;
|
||||||
|
|
||||||
public int Count => throw new NotImplementedException();
|
public int Count => ((ICollection<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).Count;
|
||||||
|
|
||||||
public bool IsReadOnly => throw new NotImplementedException();
|
public bool IsReadOnly => ((ICollection<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).IsReadOnly;
|
||||||
|
|
||||||
public bool TryGetValue(string key, out ConcurrentList<DatItem>? value) => throw new NotImplementedException();
|
public bool TryGetValue(string key, out ConcurrentList<DatItem>? value)
|
||||||
|
{
|
||||||
|
return ((IDictionary<string, ConcurrentList<DatItem>?>)items).TryGetValue(key, out value);
|
||||||
|
}
|
||||||
|
|
||||||
public void Add(KeyValuePair<string, ConcurrentList<DatItem>?> item) => throw new NotImplementedException();
|
public void Add(KeyValuePair<string, ConcurrentList<DatItem>?> item)
|
||||||
|
{
|
||||||
|
((ICollection<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).Add(item);
|
||||||
|
}
|
||||||
|
|
||||||
public void Clear() => throw new NotImplementedException();
|
public void Clear()
|
||||||
|
{
|
||||||
|
((ICollection<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).Clear();
|
||||||
|
}
|
||||||
|
|
||||||
public bool Contains(KeyValuePair<string, ConcurrentList<DatItem>?> item) => throw new NotImplementedException();
|
public bool Contains(KeyValuePair<string, ConcurrentList<DatItem>?> item)
|
||||||
|
{
|
||||||
|
return ((ICollection<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).Contains(item);
|
||||||
|
}
|
||||||
|
|
||||||
public void CopyTo(KeyValuePair<string, ConcurrentList<DatItem>?>[] array, int arrayIndex) => throw new NotImplementedException();
|
public void CopyTo(KeyValuePair<string, ConcurrentList<DatItem>?>[] array, int arrayIndex)
|
||||||
|
{
|
||||||
|
((ICollection<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).CopyTo(array, arrayIndex);
|
||||||
|
}
|
||||||
|
|
||||||
public bool Remove(KeyValuePair<string, ConcurrentList<DatItem>?> item) => throw new NotImplementedException();
|
public bool Remove(KeyValuePair<string, ConcurrentList<DatItem>?> item)
|
||||||
|
{
|
||||||
|
return ((ICollection<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).Remove(item);
|
||||||
|
}
|
||||||
|
|
||||||
public IEnumerator<KeyValuePair<string, ConcurrentList<DatItem>?>> GetEnumerator() => throw new NotImplementedException();
|
public IEnumerator<KeyValuePair<string, ConcurrentList<DatItem>?>> GetEnumerator()
|
||||||
|
{
|
||||||
|
return ((IEnumerable<KeyValuePair<string, ConcurrentList<DatItem>?>>)items).GetEnumerator();
|
||||||
|
}
|
||||||
|
|
||||||
IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException();
|
IEnumerator IEnumerable.GetEnumerator()
|
||||||
|
{
|
||||||
|
return ((IEnumerable)items).GetEnumerator();
|
||||||
|
}
|
||||||
|
|
||||||
#endregion
|
#endregion
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#endif
|
|
||||||
|
|||||||
@@ -26,11 +26,6 @@
|
|||||||
<ProjectReference Include="..\SabreTools.Logging\SabreTools.Logging.csproj" />
|
<ProjectReference Include="..\SabreTools.Logging\SabreTools.Logging.csproj" />
|
||||||
</ItemGroup>
|
</ItemGroup>
|
||||||
|
|
||||||
<!-- Support for old .NET versions -->
|
|
||||||
<ItemGroup Condition="!$(TargetFramework.StartsWith(`net2`)) AND !$(TargetFramework.StartsWith(`net3`)) AND !$(TargetFramework.StartsWith(`net40`)) AND !$(TargetFramework.StartsWith(`net452`))">
|
|
||||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="8.0.3" />
|
|
||||||
</ItemGroup>
|
|
||||||
|
|
||||||
<ItemGroup>
|
<ItemGroup>
|
||||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||||
<PackageReference Include="SabreTools.Hashing" Version="1.1.4" />
|
<PackageReference Include="SabreTools.Hashing" Version="1.1.4" />
|
||||||
|
|||||||
Reference in New Issue
Block a user