diff --git a/SabreTools.DatFiles/ItemDictionaryDB.cs b/SabreTools.DatFiles/ItemDictionaryDB.cs index 24f291af..6d9d1433 100644 --- a/SabreTools.DatFiles/ItemDictionaryDB.cs +++ b/SabreTools.DatFiles/ItemDictionaryDB.cs @@ -1,20 +1,19 @@ -#if NET462_OR_GREATER || NETCOREAPP - -using System; -using System.Collections; +using System.Collections; +#if NET40_OR_GREATER || NETCOREAPP +using System.Collections.Concurrent; +#endif using System.Collections.Generic; -using System.IO; using System.Linq; +#if NET40_OR_GREATER || NETCOREAPP using System.Threading.Tasks; +#endif using System.Xml.Serialization; -using Microsoft.Data.Sqlite; using Newtonsoft.Json; using SabreTools.Core; using SabreTools.Core.Tools; using SabreTools.DatItems; using SabreTools.DatItems.Formats; using SabreTools.Hashing; -using SabreTools.IO; using SabreTools.Logging; using SabreTools.Matching; @@ -39,14 +38,22 @@ namespace SabreTools.DatFiles private DedupeType mergedBy; /// - /// Internal database connection for the class + /// Internal dictionary for all items /// - private readonly SqliteConnection? dbc = null; +#if NET40_OR_GREATER || NETCOREAPP + private readonly ConcurrentDictionary?> items; +#else + private readonly Dictionary?> items; +#endif /// - /// Internal item dictionary name + /// Internal dictionary for all machines /// - private string? itemDictionaryFileName = null; +#if NET40_OR_GREATER || NETCOREAPP + private readonly ConcurrentDictionary machines; +#else + private readonly Dictionary machines; +#endif /// /// Lock for statistics calculation @@ -62,72 +69,16 @@ namespace SabreTools.DatFiles #region Publically available fields - #region Database - - /// - /// Item dictionary file name - /// - public string? ItemDictionaryFileName - { - get - { - if (string.IsNullOrEmpty(itemDictionaryFileName)) - itemDictionaryFileName = Path.Combine(PathTool.GetRuntimeDirectory(), $"itemDictionary{Guid.NewGuid()}.sqlite"); - - return itemDictionaryFileName; - } - } - - /// - /// Item dictionary connection string - /// - public string ItemDictionaryConnectionString => $"Data Source={ItemDictionaryFileName};"; - - #endregion - #region Keys /// - /// Get the keys from the file database + /// Get the keys from the file dictionary /// /// List of the keys [JsonIgnore, XmlIgnore] public ICollection Keys { - get - { - // If we have no database connection, we can't do anything - if (dbc == null) - return Array.Empty(); - - // Open the database connection - dbc.Open(); - - string query = $"SELECT key FROM keys"; - SqliteCommand slc = new(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); - - List keys = GetKeys(); - if (sldr.HasRows) - { - while (sldr.Read()) - { - keys.Add(sldr.GetString(0)); - } - } - - // Dispose of database objects - slc.Dispose(); - sldr.Dispose(); - dbc.Close(); - - return keys; - } - } - - private static List GetKeys() - { - return new List(); + get { return items.Keys; } } /// @@ -135,14 +86,11 @@ namespace SabreTools.DatFiles /// /// List of the keys in sorted order [JsonIgnore, XmlIgnore] - public List? SortedKeys + public List SortedKeys { get { - if (Keys == null) - return null; - - var keys = Keys.ToList(); + var keys = items.Keys.ToList(); keys.Sort(new NaturalComparer()); return keys; } @@ -178,7 +126,7 @@ namespace SabreTools.DatFiles public long TotalSize { get; private set; } = 0; /// - /// Number of hashes for each hash type + /// Number of items for each hash type /// [JsonIgnore, XmlIgnore] public Dictionary HashCounts { get; private set; } = []; @@ -215,64 +163,17 @@ namespace SabreTools.DatFiles // Ensure the key exists EnsureKey(key); - // If we have no database connection, we can't do anything - if (dbc == null) - return null; - - // Open the database connection - dbc.Open(); - - string query = $"SELECT item FROM groups WHERE key='{key}'"; - SqliteCommand slc = new(query, dbc); - SqliteDataReader sldr = slc.ExecuteReader(); - - ConcurrentList items = new(); - if (sldr.HasRows) - { - while (sldr.Read()) - { - string itemString = sldr.GetString(0); - DatItem? datItem = JsonConvert.DeserializeObject(itemString); - if (datItem != null) - items.Add(datItem); - } - } - - // Dispose of database objects - slc.Dispose(); - sldr.Dispose(); - dbc.Close(); - // Now return the value - return items; + return items[key]; } } set { Remove(key); if (value == null) - { - // If we have no database connection, we can't do anything - if (dbc == null) - return; - - // Open the database connection - dbc.Open(); - - // Now remove the value - string itemString = JsonConvert.SerializeObject(value); - string query = $"DELETE FROM groups WHERE key='{key}'"; - SqliteCommand slc = new(query, dbc); - slc.ExecuteNonQuery(); - - // Dispose of database objects - slc.Dispose(); - dbc.Close(); - } + items[key] = null; else - { AddRange(key, value); - } } } @@ -293,22 +194,8 @@ namespace SabreTools.DatFiles if (value == null) return; - // If we have no database connection, we can't do anything - if (dbc == null) - return; - - // Open the database connection - dbc.Open(); - // Now add the value - string itemString = JsonConvert.SerializeObject(value); - string query = $"INSERT INTO groups (key, item) VALUES ('{key}', '{itemString}')"; - SqliteCommand slc = new(query, dbc); - slc.ExecuteNonQuery(); - - // Dispose of database objects - slc.Dispose(); - dbc.Close(); + items[key]!.Add(value); // Now update the statistics AddItemStatistics(value); @@ -404,10 +291,7 @@ namespace SabreTools.DatFiles EnsureKey(key); // Now add the value - foreach (DatItem item in value) - { - Add(key, item); - } + items[key]!.AddRange(value); // Now update the statistics foreach (DatItem item in value) @@ -421,7 +305,7 @@ namespace SabreTools.DatFiles /// Add statistics from another DatStats object /// /// DatStats object to add from - public void AddStatistics(ItemDictionaryDB stats) + public void AddStatistics(ItemDictionary stats) { TotalCount += stats.Count; @@ -464,7 +348,7 @@ namespace SabreTools.DatFiles // Explicit lock for some weird corner cases lock (key) { - return Keys?.Contains(key) == true; + return items.ContainsKey(key); } } @@ -483,11 +367,11 @@ namespace SabreTools.DatFiles // Explicit lock for some weird corner cases lock (key) { - if (Keys?.Contains(key) != true) - return false; - - return this[key]?.Contains(value) == true; + if (items.ContainsKey(key) && items[key] != null) + return items[key]!.Contains(value); } + + return false; } /// @@ -497,23 +381,12 @@ namespace SabreTools.DatFiles public void EnsureKey(string key) { // If the key is missing from the dictionary, add it - if (Keys?.Contains(key) == true) - return; - - // If we have no database connection, we can't do anything - if (dbc == null) - return; - - // Open the database connection - dbc.Open(); - - string query = $"INSERT INTO keys (key) VALUES ('{key}')"; - SqliteCommand slc = new(query, dbc); - slc.ExecuteNonQuery(); - - // Dispose of database objects - slc.Dispose(); - dbc.Close(); + if (!items.ContainsKey(key)) +#if NET40_OR_GREATER || NETCOREAPP + items.TryAdd(key, []); +#else + items[key] = []; +#endif } /// @@ -525,14 +398,14 @@ namespace SabreTools.DatFiles lock (key) { // Get the list, if possible - ConcurrentList? fi = this[key]; + ConcurrentList? fi = items[key]; if (fi == null) - return new ConcurrentList(); + return []; // Filter the list return fi.Where(i => i != null) .Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true) - .Where(i => i.GetFieldValue(DatItem.MachineKey)!.GetStringFieldValue(Models.Metadata.Machine.NameKey) != null) + .Where(i => i.GetFieldValue(DatItem.MachineKey) != null) .ToConcurrentList(); } } @@ -547,18 +420,21 @@ namespace SabreTools.DatFiles lock (key) { // If the key doesn't exist, return - if (!ContainsKey(key) || this[key] == null) + if (!ContainsKey(key) || items[key] == null) return false; // Remove the statistics first - foreach (DatItem item in this[key]!) + foreach (DatItem item in items[key]!) { RemoveItemStatistics(item); } // Remove the key from the dictionary - this[key] = null; - return true; +#if NET40_OR_GREATER || NETCOREAPP + return items.TryRemove(key, out _); +#else + return items.Remove(key); +#endif } } @@ -573,29 +449,13 @@ namespace SabreTools.DatFiles lock (key) { // If the key and value doesn't exist, return - if (!Contains(key, value)) + if (!Contains(key, value) || items[key] == null) return false; // Remove the statistics first RemoveItemStatistics(value); - // If we have no database connection, we can't do anything - if (dbc == null) - return false; - - // Open the database connection - dbc.Open(); - - // Now remove the value - string itemString = JsonConvert.SerializeObject(value); - string query = $"DELETE FROM groups WHERE key='{key}' AND item='{itemString}'"; - SqliteCommand slc = new(query, dbc); - slc.ExecuteNonQuery(); - - // Dispose of database objects - slc.Dispose(); - dbc.Close(); - return true; + return items[key]!.Remove(value); } } @@ -606,17 +466,17 @@ namespace SabreTools.DatFiles public bool Reset(string key) { // If the key doesn't exist, return - if (!ContainsKey(key) || this[key] == null) + if (!ContainsKey(key) || items[key] == null) return false; // Remove the statistics first - foreach (DatItem item in this[key]!) + foreach (DatItem item in items[key]!) { RemoveItemStatistics(item); } // Remove the key from the dictionary - this[key] = null; + items[key] = []; return true; } @@ -633,7 +493,7 @@ namespace SabreTools.DatFiles /// Remove from the statistics given a DatItem /// /// Item to remove info for - public void RemoveItemStatistics(DatItem? item) + public void RemoveItemStatistics(DatItem item) { // If we have a null item, we can't do anything if (item == null) @@ -861,54 +721,18 @@ namespace SabreTools.DatFiles { bucketedBy = ItemKey.NULL; mergedBy = DedupeType.None; - dbc = new SqliteConnection(ItemDictionaryConnectionString); +#if NET40_OR_GREATER || NETCOREAPP + items = new ConcurrentDictionary?>(); + machines = new ConcurrentDictionary(); +#else + items = new Dictionary?>(); + machines = new Dictionary(); +#endif logger = new Logger(this); } #endregion - #region Database - - /// - /// Ensure that the database exists and has the proper schema - /// - protected void EnsureDatabase() - { - // Make sure the file exists - if (ItemDictionaryFileName != null && !System.IO.File.Exists(ItemDictionaryFileName)) - System.IO.File.Create(ItemDictionaryFileName); - - // If we have no database connection, we can't do anything - if (dbc == null) - return; - - // Open the database connection - dbc.Open(); - - // Make sure the database has the correct schema - string query = @" -CREATE TABLE IF NOT EXISTS keys ( - 'key' TEXT NOT NULL - PRIMARY KEY (key) -)"; - SqliteCommand slc = new(query, dbc); - slc.ExecuteNonQuery(); - - query = @" -CREATE TABLE IF NOT EXISTS groups ( - 'key' TEXT NOT NULL - 'item` TEXT NOT NULL -)"; - - slc = new SqliteCommand(query, dbc); - slc.ExecuteNonQuery(); - - slc.Dispose(); - dbc.Close(); - } - - #endregion - #region Custom Functionality /// @@ -920,8 +744,12 @@ CREATE TABLE IF NOT EXISTS groups ( /// True if games should only be compared on game and file name, false if system and source are counted public void BucketBy(ItemKey bucketBy, DedupeType dedupeType, bool lower = true, bool norename = true) { - // If we have a situation where there's no database or no keys at all, we skip - if (dbc == null || Keys.Count == 0) + // If we have a situation where there's no dictionary or no keys at all, we skip +#if NET40_OR_GREATER || NETCOREAPP + if (items == null || items.IsEmpty) +#else + if (items == null || items.Count == 0) +#endif return; // If the sorted type isn't the same, we want to sort the dictionary accordingly @@ -936,7 +764,8 @@ CREATE TABLE IF NOT EXISTS groups ( mergedBy = DedupeType.None; // First do the initial sort of all of the roms inplace - List oldkeys = Keys.ToList(); + List oldkeys = [.. Keys]; + #if NET452_OR_GREATER || NETCOREAPP Parallel.For(0, oldkeys.Count, Globals.ParallelOptions, k => #elif NET40_OR_GREATER @@ -947,11 +776,7 @@ CREATE TABLE IF NOT EXISTS groups ( { string key = oldkeys[k]; if (this[key] == null) -#if NET40_OR_GREATER || NETCOREAPP - return; -#else - continue; -#endif + Remove(key); // Now add each of the roms to their respective keys for (int i = 0; i < this[key]!.Count; i++) @@ -990,7 +815,7 @@ CREATE TABLE IF NOT EXISTS groups ( // Set the sorted type mergedBy = dedupeType; - List keys = Keys.ToList(); + List keys = [.. Keys]; #if NET452_OR_GREATER || NETCOREAPP Parallel.ForEach(keys, Globals.ParallelOptions, key => #elif NET40_OR_GREATER @@ -1002,7 +827,11 @@ CREATE TABLE IF NOT EXISTS groups ( // Get the possibly unsorted list ConcurrentList? sortedlist = this[key]?.ToConcurrentList(); if (sortedlist == null) +#if NET40_OR_GREATER || NETCOREAPP return; +#else + continue; +#endif // Sort the list of items to be consistent DatItem.Sort(ref sortedlist, false); @@ -1023,7 +852,7 @@ CREATE TABLE IF NOT EXISTS groups ( // If the merge type is the same, we want to sort the dictionary to be consistent else { - List keys = Keys.ToList(); + List keys = [.. Keys]; #if NET452_OR_GREATER || NETCOREAPP Parallel.ForEach(keys, Globals.ParallelOptions, key => #elif NET40_OR_GREATER @@ -1051,20 +880,28 @@ CREATE TABLE IF NOT EXISTS groups ( /// public void ClearEmpty() { - var keys = Keys.Where(k => k != null).ToList(); + var keys = items.Keys.Where(k => k != null).ToList(); foreach (string key in keys) { // If the key doesn't exist, skip - if (!Keys.Contains(key)) + if (!items.ContainsKey(key)) continue; // If the value is null, remove - else if (this[key] == null) - this[key] = null; + else if (items[key] == null) +#if NET40_OR_GREATER || NETCOREAPP + items.TryRemove(key, out _); +#else + items.Remove(key); +#endif // If there are no non-blank items, remove - else if (!this[key]!.Any(i => i != null && i is not Blank)) - this[key] = null; + else if (!items[key]!.Any(i => i != null && i is not Blank)) +#if NET40_OR_GREATER || NETCOREAPP + items.TryRemove(key, out _); +#else + items.Remove(key); +#endif } } @@ -1073,13 +910,10 @@ CREATE TABLE IF NOT EXISTS groups ( /// public void ClearMarked() { - if (Keys == null) - return; - - var keys = Keys.ToList(); + var keys = items.Keys.ToList(); foreach (string key in keys) { - ConcurrentList? oldItemList = this[key]; + ConcurrentList? oldItemList = items[key]; ConcurrentList? newItemList = oldItemList?.Where(i => i.GetBoolFieldValue(DatItem.RemoveKey) != true)?.ToConcurrentList(); Remove(key); @@ -1095,7 +929,7 @@ CREATE TABLE IF NOT EXISTS groups ( /// List of matched DatItem objects public ConcurrentList GetDuplicates(DatItem datItem, bool sorted = false) { - ConcurrentList output = new(); + ConcurrentList output = []; // Check for an empty rom list first if (TotalCount == 0) @@ -1105,12 +939,15 @@ CREATE TABLE IF NOT EXISTS groups ( string key = SortAndGetKey(datItem, sorted); // If the key doesn't exist, return the empty list - if (!ContainsKey(key) || this[key] == null) + if (!ContainsKey(key)) return output; // Try to find duplicates - ConcurrentList roms = this[key]!; - ConcurrentList left = new(); + ConcurrentList? roms = this[key]; + if (roms == null) + return output; + + ConcurrentList left = []; for (int i = 0; i < roms.Count; i++) { DatItem other = roms[i]; @@ -1169,13 +1006,13 @@ CREATE TABLE IF NOT EXISTS groups ( ResetStatistics(); // If we have a blank Dat in any way, return - if (dbc == null || Keys == null) + if (items == null) return; // Loop through and add - foreach (string key in Keys) + foreach (string key in items.Keys) { - ConcurrentList? datItems = this[key]; + ConcurrentList? datItems = items[key]; if (datItems == null) continue; @@ -1256,30 +1093,52 @@ CREATE TABLE IF NOT EXISTS groups ( #region IDictionary Implementations - public ICollection?> Values => throw new NotImplementedException(); + public ICollection?> Values => ((IDictionary?>)items).Values; - public int Count => throw new NotImplementedException(); + public int Count => ((ICollection?>>)items).Count; - public bool IsReadOnly => throw new NotImplementedException(); + public bool IsReadOnly => ((ICollection?>>)items).IsReadOnly; - public bool TryGetValue(string key, out ConcurrentList? value) => throw new NotImplementedException(); + public bool TryGetValue(string key, out ConcurrentList? value) + { + return ((IDictionary?>)items).TryGetValue(key, out value); + } - public void Add(KeyValuePair?> item) => throw new NotImplementedException(); + public void Add(KeyValuePair?> item) + { + ((ICollection?>>)items).Add(item); + } - public void Clear() => throw new NotImplementedException(); + public void Clear() + { + ((ICollection?>>)items).Clear(); + } - public bool Contains(KeyValuePair?> item) => throw new NotImplementedException(); + public bool Contains(KeyValuePair?> item) + { + return ((ICollection?>>)items).Contains(item); + } - public void CopyTo(KeyValuePair?>[] array, int arrayIndex) => throw new NotImplementedException(); + public void CopyTo(KeyValuePair?>[] array, int arrayIndex) + { + ((ICollection?>>)items).CopyTo(array, arrayIndex); + } - public bool Remove(KeyValuePair?> item) => throw new NotImplementedException(); + public bool Remove(KeyValuePair?> item) + { + return ((ICollection?>>)items).Remove(item); + } - public IEnumerator?>> GetEnumerator() => throw new NotImplementedException(); + public IEnumerator?>> GetEnumerator() + { + return ((IEnumerable?>>)items).GetEnumerator(); + } - IEnumerator IEnumerable.GetEnumerator() => throw new NotImplementedException(); + IEnumerator IEnumerable.GetEnumerator() + { + return ((IEnumerable)items).GetEnumerator(); + } #endregion } } - -#endif diff --git a/SabreTools.DatFiles/SabreTools.DatFiles.csproj b/SabreTools.DatFiles/SabreTools.DatFiles.csproj index 13803880..493131ba 100644 --- a/SabreTools.DatFiles/SabreTools.DatFiles.csproj +++ b/SabreTools.DatFiles/SabreTools.DatFiles.csproj @@ -26,11 +26,6 @@ - - - - -