diff --git a/SabreTools.DatFiles/ItemDictionaryDB.cs b/SabreTools.DatFiles/ItemDictionaryDB.cs
index 24f291af..6d9d1433 100644
--- a/SabreTools.DatFiles/ItemDictionaryDB.cs
+++ b/SabreTools.DatFiles/ItemDictionaryDB.cs
@@ -1,20 +1,19 @@
-#if NET462_OR_GREATER || NETCOREAPP
-
-using System;
-using System.Collections;
+using System.Collections;
+#if NET40_OR_GREATER || NETCOREAPP
+using System.Collections.Concurrent;
+#endif
using System.Collections.Generic;
-using System.IO;
using System.Linq;
+#if NET40_OR_GREATER || NETCOREAPP
using System.Threading.Tasks;
+#endif
using System.Xml.Serialization;
-using Microsoft.Data.Sqlite;
using Newtonsoft.Json;
using SabreTools.Core;
using SabreTools.Core.Tools;
using SabreTools.DatItems;
using SabreTools.DatItems.Formats;
using SabreTools.Hashing;
-using SabreTools.IO;
using SabreTools.Logging;
using SabreTools.Matching;
@@ -39,14 +38,22 @@ namespace SabreTools.DatFiles
private DedupeType mergedBy;
///
- /// Internal database connection for the class
+ /// Internal dictionary for all items
///
- private readonly SqliteConnection? dbc = null;
+#if NET40_OR_GREATER || NETCOREAPP
+ private readonly ConcurrentDictionary?> items;
+#else
+ private readonly Dictionary?> items;
+#endif
///
- /// Internal item dictionary name
+ /// Internal dictionary for all machines
///
- private string? itemDictionaryFileName = null;
+#if NET40_OR_GREATER || NETCOREAPP
+ private readonly ConcurrentDictionary machines;
+#else
+ private readonly Dictionary machines;
+#endif
///
/// Lock for statistics calculation
@@ -62,72 +69,16 @@ namespace SabreTools.DatFiles
#region Publically available fields
- #region Database
-
- ///
- /// Item dictionary file name
- ///
- public string? ItemDictionaryFileName
- {
- get
- {
- if (string.IsNullOrEmpty(itemDictionaryFileName))
- itemDictionaryFileName = Path.Combine(PathTool.GetRuntimeDirectory(), $"itemDictionary{Guid.NewGuid()}.sqlite");
-
- return itemDictionaryFileName;
- }
- }
-
- ///
- /// Item dictionary connection string
- ///
- public string ItemDictionaryConnectionString => $"Data Source={ItemDictionaryFileName};";
-
- #endregion
-
#region Keys
///
- /// Get the keys from the file database
+ /// Get the keys from the file dictionary
///
/// List of the keys
[JsonIgnore, XmlIgnore]
public ICollection Keys
{
- get
- {
- // If we have no database connection, we can't do anything
- if (dbc == null)
- return Array.Empty();
-
- // Open the database connection
- dbc.Open();
-
- string query = $"SELECT key FROM keys";
- SqliteCommand slc = new(query, dbc);
- SqliteDataReader sldr = slc.ExecuteReader();
-
- List keys = GetKeys();
- if (sldr.HasRows)
- {
- while (sldr.Read())
- {
- keys.Add(sldr.GetString(0));
- }
- }
-
- // Dispose of database objects
- slc.Dispose();
- sldr.Dispose();
- dbc.Close();
-
- return keys;
- }
- }
-
- private static List GetKeys()
- {
- return new List();
+ get { return items.Keys; }
}
///
@@ -135,14 +86,11 @@ namespace SabreTools.DatFiles
///
/// List of the keys in sorted order
[JsonIgnore, XmlIgnore]
- public List? SortedKeys
+ public List SortedKeys
{
get
{
- if (Keys == null)
- return null;
-
- var keys = Keys.ToList();
+ var keys = items.Keys.ToList();
keys.Sort(new NaturalComparer());
return keys;
}
@@ -178,7 +126,7 @@ namespace SabreTools.DatFiles
public long TotalSize { get; private set; } = 0;
///
- /// Number of hashes for each hash type
+ /// Number of items for each hash type
///
[JsonIgnore, XmlIgnore]
public Dictionary HashCounts { get; private set; } = [];
@@ -215,64 +163,17 @@ namespace SabreTools.DatFiles
// Ensure the key exists
EnsureKey(key);
- // If we have no database connection, we can't do anything
- if (dbc == null)
- return null;
-
- // Open the database connection
- dbc.Open();
-
- string query = $"SELECT item FROM groups WHERE key='{key}'";
- SqliteCommand slc = new(query, dbc);
- SqliteDataReader sldr = slc.ExecuteReader();
-
- ConcurrentList items = new();
- if (sldr.HasRows)
- {
- while (sldr.Read())
- {
- string itemString = sldr.GetString(0);
- DatItem? datItem = JsonConvert.DeserializeObject(itemString);
- if (datItem != null)
- items.Add(datItem);
- }
- }
-
- // Dispose of database objects
- slc.Dispose();
- sldr.Dispose();
- dbc.Close();
-
// Now return the value
- return items;
+ return items[key];
}
}
set
{
Remove(key);
if (value == null)
- {
- // If we have no database connection, we can't do anything
- if (dbc == null)
- return;
-
- // Open the database connection
- dbc.Open();
-
- // Now remove the value
- string itemString = JsonConvert.SerializeObject(value);
- string query = $"DELETE FROM groups WHERE key='{key}'";
- SqliteCommand slc = new(query, dbc);
- slc.ExecuteNonQuery();
-
- // Dispose of database objects
- slc.Dispose();
- dbc.Close();
- }
+ items[key] = null;
else
- {
AddRange(key, value);
- }
}
}
@@ -293,22 +194,8 @@ namespace SabreTools.DatFiles
if (value == null)
return;
- // If we have no database connection, we can't do anything
- if (dbc == null)
- return;
-
- // Open the database connection
- dbc.Open();
-
// Now add the value
- string itemString = JsonConvert.SerializeObject(value);
- string query = $"INSERT INTO groups (key, item) VALUES ('{key}', '{itemString}')";
- SqliteCommand slc = new(query, dbc);
- slc.ExecuteNonQuery();
-
- // Dispose of database objects
- slc.Dispose();
- dbc.Close();
+ items[key]!.Add(value);
// Now update the statistics
AddItemStatistics(value);
@@ -404,10 +291,7 @@ namespace SabreTools.DatFiles
EnsureKey(key);
// Now add the value
- foreach (DatItem item in value)
- {
- Add(key, item);
- }
+ items[key]!.AddRange(value);
// Now update the statistics
foreach (DatItem item in value)
@@ -421,7 +305,7 @@ namespace SabreTools.DatFiles
/// Add statistics from another DatStats object
///
/// DatStats object to add from
- public void AddStatistics(ItemDictionaryDB stats)
+ public void AddStatistics(ItemDictionary stats)
{
TotalCount += stats.Count;
@@ -464,7 +348,7 @@ namespace SabreTools.DatFiles
// Explicit lock for some weird corner cases
lock (key)
{
- return Keys?.Contains(key) == true;
+ return items.ContainsKey(key);
}
}
@@ -483,11 +367,11 @@ namespace SabreTools.DatFiles
// Explicit lock for some weird corner cases
lock (key)
{
- if (Keys?.Contains(key) != true)
- return false;
-
- return this[key]?.Contains(value) == true;
+ if (items.ContainsKey(key) && items[key] != null)
+ return items[key]!.Contains(value);
}
+
+ return false;
}
///
@@ -497,23 +381,12 @@ namespace SabreTools.DatFiles
public void EnsureKey(string key)
{
// If the key is missing from the dictionary, add it
- if (Keys?.Contains(key) == true)
- return;
-
- // If we have no database connection, we can't do anything
- if (dbc == null)
- return;
-
- // Open the database connection
- dbc.Open();
-
- string query = $"INSERT INTO keys (key) VALUES ('{key}')";
- SqliteCommand slc = new(query, dbc);
- slc.ExecuteNonQuery();
-
- // Dispose of database objects
- slc.Dispose();
- dbc.Close();
+ if (!items.ContainsKey(key))
+#if NET40_OR_GREATER || NETCOREAPP
+ items.TryAdd(key, []);
+#else
+ items[key] = [];
+#endif
}
///
@@ -525,14 +398,14 @@ namespace SabreTools.DatFiles
lock (key)
{
// Get the list, if possible
- ConcurrentList? fi = this[key];
+ ConcurrentList? fi = items[key];
if (fi == null)
- return new ConcurrentList();
+ return [];
// Filter the list
return fi.Where(i => i != null)
.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)
- .Where(i => i.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey) != null)
+ .Where(i => i.GetFieldValue(DatItem.MachineKey) != null)
.ToConcurrentList();
}
}
@@ -547,18 +420,21 @@ namespace SabreTools.DatFiles
lock (key)
{
// If the key doesn't exist, return
- if (!ContainsKey(key) || this[key] == null)
+ if (!ContainsKey(key) || items[key] == null)
return false;
// Remove the statistics first
- foreach (DatItem item in this[key]!)
+ foreach (DatItem item in items[key]!)
{
RemoveItemStatistics(item);
}
// Remove the key from the dictionary
- this[key] = null;
- return true;
+#if NET40_OR_GREATER || NETCOREAPP
+ return items.TryRemove(key, out _);
+#else
+ return items.Remove(key);
+#endif
}
}
@@ -573,29 +449,13 @@ namespace SabreTools.DatFiles
lock (key)
{
// If the key and value doesn't exist, return
- if (!Contains(key, value))
+ if (!Contains(key, value) || items[key] == null)
return false;
// Remove the statistics first
RemoveItemStatistics(value);
- // If we have no database connection, we can't do anything
- if (dbc == null)
- return false;
-
- // Open the database connection
- dbc.Open();
-
- // Now remove the value
- string itemString = JsonConvert.SerializeObject(value);
- string query = $"DELETE FROM groups WHERE key='{key}' AND item='{itemString}'";
- SqliteCommand slc = new(query, dbc);
- slc.ExecuteNonQuery();
-
- // Dispose of database objects
- slc.Dispose();
- dbc.Close();
- return true;
+ return items[key]!.Remove(value);
}
}
@@ -606,17 +466,17 @@ namespace SabreTools.DatFiles
public bool Reset(string key)
{
// If the key doesn't exist, return
- if (!ContainsKey(key) || this[key] == null)
+ if (!ContainsKey(key) || items[key] == null)
return false;
// Remove the statistics first
- foreach (DatItem item in this[key]!)
+ foreach (DatItem item in items[key]!)
{
RemoveItemStatistics(item);
}
// Remove the key from the dictionary
- this[key] = null;
+ items[key] = [];
return true;
}
@@ -633,7 +493,7 @@ namespace SabreTools.DatFiles
/// Remove from the statistics given a DatItem
///
/// Item to remove info for
- public void RemoveItemStatistics(DatItem? item)
+ public void RemoveItemStatistics(DatItem item)
{
// If we have a null item, we can't do anything
if (item == null)
@@ -861,54 +721,18 @@ namespace SabreTools.DatFiles
{
bucketedBy = ItemKey.NULL;
mergedBy = DedupeType.None;
- dbc = new SqliteConnection(ItemDictionaryConnectionString);
+#if NET40_OR_GREATER || NETCOREAPP
+ items = new ConcurrentDictionary?>();
+ machines = new ConcurrentDictionary();
+#else
+ items = new Dictionary?>();
+ machines = new Dictionary();
+#endif
logger = new Logger(this);
}
#endregion
- #region Database
-
- ///
- /// Ensure that the database exists and has the proper schema
- ///
- protected void EnsureDatabase()
- {
- // Make sure the file exists
- if (ItemDictionaryFileName != null && !System.IO.File.Exists(ItemDictionaryFileName))
- System.IO.File.Create(ItemDictionaryFileName);
-
- // If we have no database connection, we can't do anything
- if (dbc == null)
- return;
-
- // Open the database connection
- dbc.Open();
-
- // Make sure the database has the correct schema
- string query = @"
-CREATE TABLE IF NOT EXISTS keys (
- 'key' TEXT NOT NULL
- PRIMARY KEY (key)
-)";
- SqliteCommand slc = new(query, dbc);
- slc.ExecuteNonQuery();
-
- query = @"
-CREATE TABLE IF NOT EXISTS groups (
- 'key' TEXT NOT NULL
- 'item` TEXT NOT NULL
-)";
-
- slc = new SqliteCommand(query, dbc);
- slc.ExecuteNonQuery();
-
- slc.Dispose();
- dbc.Close();
- }
-
- #endregion
-
#region Custom Functionality
///
@@ -920,8 +744,12 @@ CREATE TABLE IF NOT EXISTS groups (
/// True if games should only be compared on game and file name, false if system and source are counted
public void BucketBy(ItemKey bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true)
{
- // If we have a situation where there's no database or no keys at all, we skip
- if (dbc == null || Keys.Count == 0)
+ // If we have a situation where there's no dictionary or no keys at all, we skip
+#if NET40_OR_GREATER || NETCOREAPP
+ if (items == null || items.IsEmpty)
+#else
+ if (items == null || items.Count == 0)
+#endif
return;
// If the sorted type isn't the same, we want to sort the dictionary accordingly
@@ -936,7 +764,8 @@ CREATE TABLE IF NOT EXISTS groups (
mergedBy = DedupeType.None;
// First do the initial sort of all of the roms inplace
- List oldkeys = Keys.ToList();
+ List oldkeys = [.. Keys];
+
#if NET452_OR_GREATER || NETCOREAPP
Parallel.For(0, oldkeys.Count, Globals.ParallelOptions, k =>
#elif NET40_OR_GREATER
@@ -947,11 +776,7 @@ CREATE TABLE IF NOT EXISTS groups (
{
string key = oldkeys[k];
if (this[key] == null)
-#if NET40_OR_GREATER || NETCOREAPP
- return;
-#else
- continue;
-#endif
+ Remove(key);
// Now add each of the roms to their respective keys
for (int i = 0; i < this[key]!.Count; i++)
@@ -990,7 +815,7 @@ CREATE TABLE IF NOT EXISTS groups (
// Set the sorted type
mergedBy = dedupeType;
- List keys = Keys.ToList();
+ List keys = [.. Keys];
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
@@ -1002,7 +827,11 @@ CREATE TABLE IF NOT EXISTS groups (
// Get the possibly unsorted list
ConcurrentList? sortedlist = this[key]?.ToConcurrentList();
if (sortedlist == null)
+#if NET40_OR_GREATER || NETCOREAPP
return;
+#else
+ continue;
+#endif
// Sort the list of items to be consistent
DatItem.Sort(ref sortedlist, false);
@@ -1023,7 +852,7 @@ CREATE TABLE IF NOT EXISTS groups (
// If the merge type is the same, we want to sort the dictionary to be consistent
else
{
- List keys = Keys.ToList();
+ List keys = [.. Keys];
#if NET452_OR_GREATER || NETCOREAPP
Parallel.ForEach(keys, Globals.ParallelOptions, key =>
#elif NET40_OR_GREATER
@@ -1051,20 +880,28 @@ CREATE TABLE IF NOT EXISTS groups (
///
public void ClearEmpty()
{
- var keys = Keys.Where(k => k != null).ToList();
+ var keys = items.Keys.Where(k => k != null).ToList();
foreach (string key in keys)
{
// If the key doesn't exist, skip
- if (!Keys.Contains(key))
+ if (!items.ContainsKey(key))
continue;
// If the value is null, remove
- else if (this[key] == null)
- this[key] = null;
+ else if (items[key] == null)
+#if NET40_OR_GREATER || NETCOREAPP
+ items.TryRemove(key, out _);
+#else
+ items.Remove(key);
+#endif
// If there are no non-blank items, remove
- else if (!this[key]!.Any(i => i != null && i is not Blank))
- this[key] = null;
+ else if (!items[key]!.Any(i => i != null && i is not Blank))
+#if NET40_OR_GREATER || NETCOREAPP
+ items.TryRemove(key, out _);
+#else
+ items.Remove(key);
+#endif
}
}
@@ -1073,13 +910,10 @@ CREATE TABLE IF NOT EXISTS groups (
///
public void ClearMarked()
{
- if (Keys == null)
- return;
-
- var keys = Keys.ToList();
+ var keys = items.Keys.ToList();
foreach (string key in keys)
{
- ConcurrentList? oldItemList = this[key];
+ ConcurrentList? oldItemList = items[key];
ConcurrentList? newItemList = oldItemList?.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)?.ToConcurrentList();
Remove(key);
@@ -1095,7 +929,7 @@ CREATE TABLE IF NOT EXISTS groups (
/// List of matched DatItem objects
public ConcurrentList GetDuplicates(DatItem datItem, bool sorted = false)
{
- ConcurrentList output = new();
+ ConcurrentList output = [];
// Check for an empty rom list first
if (TotalCount == 0)
@@ -1105,12 +939,15 @@ CREATE TABLE IF NOT EXISTS groups (
string key = SortAndGetKey(datItem, sorted);
// If the key doesn't exist, return the empty list
- if (!ContainsKey(key) || this[key] == null)
+ if (!ContainsKey(key))
return output;
// Try to find duplicates
- ConcurrentList roms = this[key]!;
- ConcurrentList left = new();
+ ConcurrentList? roms = this[key];
+ if (roms == null)
+ return output;
+
+ ConcurrentList left = [];
for (int i = 0; i < roms.Count; i++)
{
DatItem other = roms[i];
@@ -1169,13 +1006,13 @@ CREATE TABLE IF NOT EXISTS groups (
ResetStatistics();
// If we have a blank Dat in any way, return
- if (dbc == null || Keys == null)
+ if (items == null)
return;
// Loop through and add
- foreach (string key in Keys)
+ foreach (string key in items.Keys)
{
- ConcurrentList? datItems = this[key];
+ ConcurrentList? datItems = items[key];
if (datItems == null)
continue;
@@ -1256,30 +1093,52 @@ CREATE TABLE IF NOT EXISTS groups (
#region IDictionary Implementations
- public ICollection?> Values => throw new NotImplementedException();
+ public ICollection?> Values => ((IDictionary?>)items).Values;
- public int Count => throw new NotImplementedException();
+ public int Count => ((ICollection?>>)items).Count;
- public bool IsReadOnly => throw new NotImplementedException();
+ public bool IsReadOnly => ((ICollection?>>)items).IsReadOnly;
- public bool TryGetValue(string key, out ConcurrentList? value) => throw new NotImplementedException();
+ public bool TryGetValue(string key, out ConcurrentList? value)
+ {
+ return ((IDictionary?>)items).TryGetValue(key, out value);
+ }
- public void Add(KeyValuePair?> item) => throw new NotImplementedException();
+ public void Add(KeyValuePair?> item)
+ {
+ ((ICollection?>>)items).Add(item);
+ }
- public void Clear() => throw new NotImplementedException();
+ public void Clear()
+ {
+ ((ICollection?>>)items).Clear();
+ }
- public bool Contains(KeyValuePair?> item) => throw new NotImplementedException();
+ public bool Contains(KeyValuePair?> item)
+ {
+ return ((ICollection?>>)items).Contains(item);
+ }
- public void CopyTo(KeyValuePair?>[] array, int arrayIndex) => throw new NotImplementedException();
+ public void CopyTo(KeyValuePair?>[] array, int arrayIndex)
+ {
+ ((ICollection?>>)items).CopyTo(array, arrayIndex);
+ }
- public bool Remove(KeyValuePair?> item) => throw new NotImplementedException();
+ public bool Remove(KeyValuePair?> item)
+ {
+ return ((ICollection?>>)items).Remove(item);
+ }
- public IEnumerator?>> GetEnumerator() => throw new NotImplementedException();
+ public IEnumerator?>> GetEnumerator()
+ {
+ return ((IEnumerable?>>)items).GetEnumerator();
+ }
- IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException();
+ IEnumerator IEnumerable.GetEnumerator()
+ {
+ return ((IEnumerable)items).GetEnumerator();
+ }
#endregion
}
}
-
-#endif
diff --git a/SabreTools.DatFiles/SabreTools.DatFiles.csproj b/SabreTools.DatFiles/SabreTools.DatFiles.csproj
index 13803880..493131ba 100644
--- a/SabreTools.DatFiles/SabreTools.DatFiles.csproj
+++ b/SabreTools.DatFiles/SabreTools.DatFiles.csproj
@@ -26,11 +26,6 @@
-
-
-
-
-