[Workers] Implement database locking in DatImporter for thread safety

This commit is contained in:
2025-07-11 18:40:14 +01:00
parent 32bbb55e61
commit 8eaca3556a

View File

@@ -53,10 +53,11 @@ namespace RomRepoMgr.Core.Workers;
public sealed class DatImporter public sealed class DatImporter
{ {
readonly string _category; private static readonly object _dbLock = new();
readonly string _datFilesPath; readonly string _category;
readonly string _datPath; readonly string _datFilesPath;
bool _aborted; readonly string _datPath;
bool _aborted;
public DatImporter(string datPath, string category) public DatImporter(string datPath, string category)
{ {
@@ -132,8 +133,11 @@ public sealed class DatImporter
Category = _category Category = _category
}; };
ctx.RomSets.Add(romSet); lock(_dbLock)
ctx.SaveChanges(); {
ctx.RomSets.Add(romSet);
ctx.SaveChanges();
}
SetMessage?.Invoke(this, SetMessage?.Invoke(this,
new MessageEventArgs new MessageEventArgs
@@ -199,8 +203,11 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
ctx.BulkInsert(machines.Values.ToList(), b => b.SetOutputIdentity = true); lock(_dbLock)
ctx.SaveChanges(); {
ctx.BulkInsert(machines.Values.ToList(), b => b.SetOutputIdentity = true);
ctx.SaveChanges();
}
SetMessage?.Invoke(this, SetMessage?.Invoke(this,
new MessageEventArgs new MessageEventArgs
@@ -248,358 +255,329 @@ public sealed class DatImporter
Maximum = datFile.Items.SortedKeys.Length Maximum = datFile.Items.SortedKeys.Length
}); });
using(DbTransaction dbTransaction = dbConnection.BeginTransaction()) List<DbFile> pendingFilesByCrcList;
List<DbFile> pendingFilesByMd5List;
List<DbFile> pendingFilesBySha1List;
List<DbFile> pendingFilesBySha256List;
List<DbFile> pendingFilesBySha384List;
List<DbFile> pendingFilesBySha512List;
Dictionary<string, DbDisk> pendingDisksByMd5;
Dictionary<string, DbDisk> pendingDisksBySha1;
Dictionary<string, DbMedia> pendingMediasByMd5;
Dictionary<string, DbMedia> pendingMediasBySha1;
Dictionary<string, DbMedia> pendingMediasBySha256;
lock(_dbLock)
{ {
DbCommand dbcc = dbConnection.CreateCommand(); using(DbTransaction dbTransaction = dbConnection.BeginTransaction())
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomCrc32Table}\" (\"Size\" INTEGER NOT NULL, \"Crc32\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomMd5Table}\" (\"Size\" INTEGER NOT NULL, \"Md5\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha1Table}\" (\"Size\" INTEGER NOT NULL, \"Sha1\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha256Table}\" (\"Size\" INTEGER NOT NULL, \"Sha256\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha384Table}\" (\"Size\" INTEGER NOT NULL, \"Sha384\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha512Table}\" (\"Size\" INTEGER NOT NULL, \"Sha512\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpDiskMd5Table}\" (\"Md5\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpDiskSha1Table}\" (\"Sha1\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpMediaMd5Table}\" (\"Md5\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpMediaSha1Table}\" (\"Sha1\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpMediaSha256Table}\" (\"Sha256\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
foreach(string key in datFile.Items.SortedKeys)
{ {
SetProgress?.Invoke(this, DbCommand dbcc = dbConnection.CreateCommand();
new ProgressEventArgs
{
Value = position
});
foreach(DatItem item in datFile.GetItemsForBucket(key)) dbcc.CommandText =
$"CREATE TABLE \"{tmpRomCrc32Table}\" (\"Size\" INTEGER NOT NULL, \"Crc32\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomMd5Table}\" (\"Size\" INTEGER NOT NULL, \"Md5\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha1Table}\" (\"Size\" INTEGER NOT NULL, \"Sha1\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha256Table}\" (\"Size\" INTEGER NOT NULL, \"Sha256\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha384Table}\" (\"Size\" INTEGER NOT NULL, \"Sha384\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText =
$"CREATE TABLE \"{tmpRomSha512Table}\" (\"Size\" INTEGER NOT NULL, \"Sha512\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpDiskMd5Table}\" (\"Md5\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpDiskSha1Table}\" (\"Sha1\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpMediaMd5Table}\" (\"Md5\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpMediaSha1Table}\" (\"Sha1\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = $"CREATE TABLE \"{tmpMediaSha256Table}\" (\"Sha256\" TEXT NOT NULL);";
dbcc.ExecuteNonQuery();
foreach(string key in datFile.Items.SortedKeys)
{ {
switch(item) SetProgress?.Invoke(this,
new ProgressEventArgs
{
Value = position
});
foreach(DatItem item in datFile.GetItemsForBucket(key))
{ {
case Rom rom: switch(item)
if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.CRCKey) != null) {
{ case Rom rom:
dbcc = dbConnection.CreateCommand(); if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.CRCKey) != null)
{
dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpRomCrc32Table}\" (\"Size\", \"Crc32\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.CRCKey)}\");"; $"INSERT INTO \"{tmpRomCrc32Table}\" (\"Size\", \"Crc32\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.CRCKey)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
romsHaveCrc = true; romsHaveCrc = true;
} }
if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.MD5Key) != null) if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.MD5Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpRomMd5Table}\" (\"Size\", \"Md5\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.MD5Key)}\");"; $"INSERT INTO \"{tmpRomMd5Table}\" (\"Size\", \"Md5\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.MD5Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
romsHaveMd5 = true; romsHaveMd5 = true;
} }
if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key) != null) if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpRomSha1Table}\" (\"Size\", \"Sha1\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)}\");"; $"INSERT INTO \"{tmpRomSha1Table}\" (\"Size\", \"Sha1\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA1Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
romsHaveSha1 = true; romsHaveSha1 = true;
} }
if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA256Key) != null) if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA256Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpRomSha256Table}\" (\"Size\", \"Sha256\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA256Key)}\");"; $"INSERT INTO \"{tmpRomSha256Table}\" (\"Size\", \"Sha256\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA256Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
romsHaveSha256 = true; romsHaveSha256 = true;
} }
if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA384Key) != null) if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA384Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpRomSha384Table}\" (\"Size\", \"Sha384\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA384Key)}\");"; $"INSERT INTO \"{tmpRomSha384Table}\" (\"Size\", \"Sha384\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA384Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
romsHaveSha384 = true; romsHaveSha384 = true;
} }
if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA512Key) != null) if(rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA512Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpRomSha512Table}\" (\"Size\", \"Sha512\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA512Key)}\");"; $"INSERT INTO \"{tmpRomSha512Table}\" (\"Size\", \"Sha512\") VALUES (\"{(ulong)rom.GetInt64FieldValue(SabreTools.Models.Metadata.Rom.SizeKey)}\", \"{rom.GetStringFieldValue(SabreTools.Models.Metadata.Rom.SHA512Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
romsHaveSha512 = true; romsHaveSha512 = true;
} }
roms.Add(rom); roms.Add(rom);
continue; continue;
case Disk disk: case Disk disk:
if(disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.MD5Key) != null) if(disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.MD5Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpDiskMd5Table}\" (\"Md5\") VALUES (\"{disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.MD5Key)}\");"; $"INSERT INTO \"{tmpDiskMd5Table}\" (\"Md5\") VALUES (\"{disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.MD5Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
disksHaveMd5 = true; disksHaveMd5 = true;
} }
if(disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.SHA1Key) != null) if(disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.SHA1Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpDiskSha1Table}\" (\"Sha1\") VALUES (\"{disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.SHA1Key)}\");"; $"INSERT INTO \"{tmpDiskSha1Table}\" (\"Sha1\") VALUES (\"{disk.GetStringFieldValue(SabreTools.Models.Metadata.Disk.SHA1Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
disksHaveSha1 = true; disksHaveSha1 = true;
} }
disks.Add(disk); disks.Add(disk);
continue; continue;
case Media media: case Media media:
if(media.GetStringFieldValue(SabreTools.Models.Metadata.Media.MD5Key) != null) if(media.GetStringFieldValue(SabreTools.Models.Metadata.Media.MD5Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpMediaMd5Table}\" (\"Md5\") VALUES (\"{media.GetStringFieldValue(SabreTools.Models.Metadata.Media.MD5Key)}\");"; $"INSERT INTO \"{tmpMediaMd5Table}\" (\"Md5\") VALUES (\"{media.GetStringFieldValue(SabreTools.Models.Metadata.Media.MD5Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
mediasHaveMd5 = true; mediasHaveMd5 = true;
} }
if(media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA1Key) != null) if(media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA1Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpMediaSha1Table}\" (\"Sha1\") VALUES (\"{media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA1Key)}\");"; $"INSERT INTO \"{tmpMediaSha1Table}\" (\"Sha1\") VALUES (\"{media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA1Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
mediasHaveSha1 = true; mediasHaveSha1 = true;
} }
if(media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA256Key) != null) if(media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA256Key) != null)
{ {
dbcc = dbConnection.CreateCommand(); dbcc = dbConnection.CreateCommand();
dbcc.CommandText = dbcc.CommandText =
$"INSERT INTO \"{tmpMediaSha256Table}\" (\"Sha256\") VALUES (\"{media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA256Key)}\");"; $"INSERT INTO \"{tmpMediaSha256Table}\" (\"Sha256\") VALUES (\"{media.GetStringFieldValue(SabreTools.Models.Metadata.Media.SHA256Key)}\");";
dbcc.ExecuteNonQuery(); dbcc.ExecuteNonQuery();
mediasHaveSha256 = true; mediasHaveSha256 = true;
} }
medias.Add(media); medias.Add(media);
continue; continue;
}
} }
position++;
} }
position++; SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
dbTransaction.Commit();
} }
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); pendingFilesByCrcList = romsHaveCrc
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomCrc32Table}] AS t WHERE f.Crc32 = t.Crc32 AND f.Size = t.Size")
.ToList()
: [];
dbTransaction.Commit(); pendingFilesByMd5List = romsHaveMd5
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomMd5Table}] AS t WHERE f.Md5 = t.Md5 AND f.Size = t.Size")
.ToList()
: [];
pendingFilesBySha1List = romsHaveSha1
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha1Table}] AS t WHERE f.Sha1 = t.Sha1 AND f.Size = t.Size")
.ToList()
: [];
pendingFilesBySha256List = romsHaveSha256
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha256Table}] AS t WHERE f.Sha256 = t.Sha256 AND f.Size = t.Size")
.ToList()
: [];
pendingFilesBySha384List = romsHaveSha384
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha384Table}] AS t WHERE f.Sha384 = t.Sha384 AND f.Size = t.Size")
.ToList()
: [];
pendingFilesBySha512List = romsHaveSha512
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha512Table}] AS t WHERE f.Sha512 = t.Sha512 AND f.Size = t.Size")
.ToList()
: [];
pendingDisksByMd5 = disksHaveMd5
? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5)
: [];
pendingDisksBySha1 = disksHaveSha1
? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1)
: [];
pendingMediasByMd5 = mediasHaveMd5
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5)
: [];
pendingMediasBySha1 = mediasHaveSha1
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1)
: [];
pendingMediasBySha256 = mediasHaveSha256
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha256Table}] AS t WHERE f.Sha256 = t.Sha256")
.ToDictionary(f => f.Sha256)
: [];
} }
List<DbFile> pendingFilesByCrcList = romsHaveCrc
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomCrc32Table}] AS t WHERE f.Crc32 = t.Crc32 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesByMd5List = romsHaveMd5
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomMd5Table}] AS t WHERE f.Md5 = t.Md5 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha1List =
romsHaveSha1
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha1Table}] AS t WHERE f.Sha1 = t.Sha1 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha256List =
romsHaveSha256
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha256Table}] AS t WHERE f.Sha256 = t.Sha256 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha384List =
romsHaveSha384
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha384Table}] AS t WHERE f.Sha384 = t.Sha384 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha512List =
romsHaveSha512
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha512Table}] AS t WHERE f.Sha512 = t.Sha512 AND f.Size = t.Size")
.ToList()
: [];
Dictionary<string, DbDisk> pendingDisksByMd5 =
disksHaveMd5
? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5)
: [];
Dictionary<string, DbDisk> pendingDisksBySha1 =
disksHaveSha1
? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1)
: [];
Dictionary<string, DbMedia> pendingMediasByMd5 =
mediasHaveMd5
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5)
: [];
Dictionary<string, DbMedia> pendingMediasBySha1 =
mediasHaveSha1
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1)
: [];
Dictionary<string, DbMedia> pendingMediasBySha256 =
mediasHaveSha256
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha256Table}] AS t WHERE f.Sha256 = t.Sha256")
.ToDictionary(f => f.Sha256)
: [];
var pendingFilesByCrc = new Dictionary<string, DbFile>(); var pendingFilesByCrc = new Dictionary<string, DbFile>();
var pendingFilesByMd5 = new Dictionary<string, DbFile>(); var pendingFilesByMd5 = new Dictionary<string, DbFile>();
var pendingFilesBySha1 = new Dictionary<string, DbFile>(); var pendingFilesBySha1 = new Dictionary<string, DbFile>();
var pendingFilesBySha256 = new Dictionary<string, DbFile>(); var pendingFilesBySha256 = new Dictionary<string, DbFile>();
var pendingFilesBySha384 = new Dictionary<string, DbFile>(); var pendingFilesBySha384 = new Dictionary<string, DbFile>();
var pendingFilesBySha512 = new Dictionary<string, DbFile>(); var pendingFilesBySha512 = new Dictionary<string, DbFile>();
var pendingFiles = new List<DbFile>();
// This is because of hash collisions. var pendingFiles = pendingFilesByCrcList.Where(item => !pendingFilesByCrc.TryAdd(item.Crc32, item))
foreach(DbFile item in pendingFilesByCrcList) .ToList();
{
if(pendingFilesByCrc.ContainsKey(item.Crc32))
pendingFiles.Add(item);
else
pendingFilesByCrc[item.Crc32] = item;
}
foreach(DbFile item in pendingFilesByMd5List) pendingFiles.AddRange(pendingFilesByMd5List.Where(item => !pendingFilesByMd5.TryAdd(item.Md5, item)));
{ pendingFiles.AddRange(pendingFilesBySha1List.Where(item => !pendingFilesBySha1.TryAdd(item.Sha1, item)));
if(pendingFilesByMd5.ContainsKey(item.Md5))
pendingFiles.Add(item);
else
pendingFilesByMd5[item.Md5] = item;
}
foreach(DbFile item in pendingFilesBySha1List) pendingFiles.AddRange(pendingFilesBySha256List.Where(item => !pendingFilesBySha256
{ .TryAdd(item.Sha256, item)));
if(pendingFilesBySha1.ContainsKey(item.Sha1))
pendingFiles.Add(item);
else
pendingFilesBySha1[item.Sha1] = item;
}
foreach(DbFile item in pendingFilesBySha256List) pendingFiles.AddRange(pendingFilesBySha384List.Where(item => !pendingFilesBySha384
{ .TryAdd(item.Sha384, item)));
if(pendingFilesBySha256.ContainsKey(item.Sha256))
pendingFiles.Add(item);
else
pendingFilesBySha256[item.Sha256] = item;
}
foreach(DbFile item in pendingFilesBySha384List) pendingFiles.AddRange(pendingFilesBySha512List.Where(item => !pendingFilesBySha512
{ .TryAdd(item.Sha512, item)));
if(pendingFilesBySha384.ContainsKey(item.Sha384))
pendingFiles.Add(item);
else
pendingFilesBySha384[item.Sha384] = item;
}
foreach(DbFile item in pendingFilesBySha512List)
{
if(pendingFilesBySha512.ContainsKey(item.Sha512))
pendingFiles.Add(item);
else
pendingFilesBySha512[item.Sha512] = item;
}
// Clear some memory // Clear some memory
pendingFilesByCrcList.Clear(); pendingFilesByCrcList.Clear();
@@ -609,17 +587,20 @@ public sealed class DatImporter
pendingFilesBySha384List.Clear(); pendingFilesBySha384List.Clear();
pendingFilesBySha512List.Clear(); pendingFilesBySha512List.Clear();
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomCrc32Table}]"); lock(_dbLock)
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomMd5Table}]"); {
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha1Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomCrc32Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha256Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomMd5Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha384Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha1Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha512Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha256Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpDiskMd5Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha384Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpDiskSha1Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha512Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaMd5Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpDiskMd5Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha1Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpDiskSha1Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha256Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaMd5Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha1Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha256Table}]");
}
SetProgressBounds?.Invoke(this, SetProgressBounds?.Invoke(this,
new ProgressBoundsEventArgs new ProgressBoundsEventArgs
@@ -923,7 +904,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
ctx.BulkInsert(newFiles, b => b.SetOutputIdentity = true); lock(_dbLock)
{
ctx.BulkInsert(newFiles, b => b.SetOutputIdentity = true);
}
foreach(FileByMachine fbm in newFilesByMachine) foreach(FileByMachine fbm in newFilesByMachine)
{ {
@@ -931,9 +915,12 @@ public sealed class DatImporter
fbm.MachineId = fbm.Machine.Id; fbm.MachineId = fbm.Machine.Id;
} }
ctx.BulkInsert(newFilesByMachine); lock(_dbLock)
{
ctx.BulkInsert(newFilesByMachine);
ctx.SaveChanges(); ctx.SaveChanges();
}
pendingFilesBySha512.Clear(); pendingFilesBySha512.Clear();
pendingFilesBySha384.Clear(); pendingFilesBySha384.Clear();
@@ -1056,7 +1043,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
ctx.BulkInsert(newDisks, b => b.SetOutputIdentity = true); lock(_dbLock)
{
ctx.BulkInsert(newDisks, b => b.SetOutputIdentity = true);
}
foreach(DiskByMachine dbm in newDisksByMachine) foreach(DiskByMachine dbm in newDisksByMachine)
{ {
@@ -1064,9 +1054,12 @@ public sealed class DatImporter
dbm.MachineId = dbm.Machine.Id; dbm.MachineId = dbm.Machine.Id;
} }
ctx.BulkInsert(newDisksByMachine); lock(_dbLock)
{
ctx.BulkInsert(newDisksByMachine);
ctx.SaveChanges(); ctx.SaveChanges();
}
pendingDisksBySha1.Clear(); pendingDisksBySha1.Clear();
pendingDisksByMd5.Clear(); pendingDisksByMd5.Clear();
@@ -1203,7 +1196,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
ctx.BulkInsert(newMedias, b => b.SetOutputIdentity = true); lock(_dbLock)
{
ctx.BulkInsert(newMedias, b => b.SetOutputIdentity = true);
}
foreach(MediaByMachine mbm in newMediasByMachine) foreach(MediaByMachine mbm in newMediasByMachine)
{ {
@@ -1211,63 +1207,70 @@ public sealed class DatImporter
mbm.MachineId = mbm.Machine.Id; mbm.MachineId = mbm.Machine.Id;
} }
ctx.BulkInsert(newMediasByMachine); lock(_dbLock)
{
ctx.BulkInsert(newMediasByMachine);
ctx.SaveChanges(); ctx.SaveChanges();
}
pendingMediasBySha256.Clear(); pendingMediasBySha256.Clear();
pendingMediasBySha1.Clear(); pendingMediasBySha1.Clear();
pendingMediasByMd5.Clear(); pendingMediasByMd5.Clear();
newMedias.Clear(); newMedias.Clear();
newMediasByMachine.Clear(); newMediasByMachine.Clear();
RomSetStat stats;
RomSetStat stats = ctx.RomSets.Where(r => r.Id == romSet.Id) lock(_dbLock)
.Select(r => new RomSetStat {
{ stats = ctx.RomSets.Where(r => r.Id == romSet.Id)
RomSetId = r.Id, .Select(r => new RomSetStat
TotalMachines = r.Machines.Count, {
CompleteMachines = RomSetId = r.Id,
r.Machines.Count(m => m.Files.Count > 0 && TotalMachines = r.Machines.Count,
m.Disks.Count == 0 && CompleteMachines =
m.Files.All(f => f.File.IsInRepo)) + r.Machines.Count(m => m.Files.Count > 0 &&
r.Machines.Count(m => m.Disks.Count > 0 && m.Disks.Count == 0 &&
m.Files.Count == 0 && m.Files.All(f => f.File.IsInRepo)) +
m.Disks.All(f => f.Disk.IsInRepo)) + r.Machines.Count(m => m.Disks.Count > 0 &&
r.Machines.Count(m => m.Files.Count > 0 && m.Files.Count == 0 &&
m.Disks.Count > 0 && m.Disks.All(f => f.Disk.IsInRepo)) +
m.Files.All(f => f.File.IsInRepo) && r.Machines.Count(m => m.Files.Count > 0 &&
m.Disks.All(f => f.Disk.IsInRepo)), m.Disks.Count > 0 &&
IncompleteMachines = m.Files.All(f => f.File.IsInRepo) &&
r.Machines.Count(m => m.Files.Count > 0 && m.Disks.All(f => f.Disk.IsInRepo)),
m.Disks.Count == 0 && IncompleteMachines =
m.Files.Any(f => !f.File.IsInRepo)) + r.Machines.Count(m => m.Files.Count > 0 &&
r.Machines.Count(m => m.Disks.Count > 0 && m.Disks.Count == 0 &&
m.Files.Count == 0 && m.Files.Any(f => !f.File.IsInRepo)) +
m.Disks.Any(f => !f.Disk.IsInRepo)) + r.Machines.Count(m => m.Disks.Count > 0 &&
r.Machines.Count(m => m.Files.Count > 0 && m.Files.Count == 0 &&
m.Disks.Count > 0 && m.Disks.Any(f => !f.Disk.IsInRepo)) +
(m.Files.Any(f => !f.File.IsInRepo) || r.Machines.Count(m => m.Files.Count > 0 &&
m.Disks.Any(f => !f.Disk.IsInRepo))), m.Disks.Count > 0 &&
TotalRoms = (m.Files.Any(f => !f.File.IsInRepo) ||
r.Machines.Sum(m => m.Files.Count) + m.Disks.Any(f => !f.Disk.IsInRepo))),
r.Machines.Sum(m => m.Disks.Count) + TotalRoms =
r.Machines.Sum(m => m.Medias.Count), r.Machines.Sum(m => m.Files.Count) +
HaveRoms = r.Machines.Sum(m => m.Files.Count(f => f.File.IsInRepo)) + r.Machines.Sum(m => m.Disks.Count) +
r.Machines.Sum(m => m.Disks.Count(f => f.Disk.IsInRepo)) + r.Machines.Sum(m => m.Medias.Count),
r.Machines.Sum(m => m.Medias.Count(f => f.Media.IsInRepo)), HaveRoms = r.Machines.Sum(m => m.Files.Count(f => f.File.IsInRepo)) +
MissRoms = r.Machines.Sum(m => m.Files.Count(f => !f.File.IsInRepo)) + r.Machines.Sum(m => m.Disks.Count(f => f.Disk.IsInRepo)) +
r.Machines.Sum(m => m.Disks.Count(f => !f.Disk.IsInRepo)) + r.Machines.Sum(m => m.Medias.Count(f => f.Media.IsInRepo)),
r.Machines.Sum(m => m.Medias.Count(f => !f.Media.IsInRepo)) MissRoms = r.Machines.Sum(m => m.Files.Count(f => !f.File.IsInRepo)) +
}) r.Machines.Sum(m => m.Disks.Count(f => !f.Disk.IsInRepo)) +
.FirstOrDefault(); r.Machines.Sum(m => m.Medias.Count(f => !f.Media.IsInRepo))
})
.FirstOrDefault();
RomSetStat oldStats = ctx.RomSetStats.Find(stats.RomSetId); RomSetStat oldStats = ctx.RomSetStats.Find(stats.RomSetId);
if(oldStats != null) ctx.Remove(oldStats); if(oldStats != null) ctx.Remove(oldStats);
ctx.RomSetStats.Add(stats); ctx.RomSetStats.Add(stats);
ctx.SaveChanges(); ctx.SaveChanges();
}
WorkFinished?.Invoke(this, WorkFinished?.Invoke(this,
new MessageEventArgs new MessageEventArgs