claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 19:24:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Workers] Implement database locking in DatImporter for thread safety

Browse Source
This commit is contained in:
Natalia Portillo
2025-07-11 18:40:14 +01:00
parent 32bbb55e61
commit 8eaca3556a
1 changed files with 355 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
RomRepoMgr.Core/Workers/DatImporter.cs
View File

@@ -53,6 +53,7 @@ namespace RomRepoMgr.Core.Workers;
public sealed class DatImporter public sealed class DatImporter
{ {
private static readonly object _dbLock = new();
readonly string _category; readonly string _category;
readonly string _datFilesPath; readonly string _datFilesPath;
readonly string _datPath; readonly string _datPath;
@@ -132,8 +133,11 @@ public sealed class DatImporter
Category = _category Category = _category
}; };
lock(_dbLock)
{
ctx.RomSets.Add(romSet); ctx.RomSets.Add(romSet);
ctx.SaveChanges(); ctx.SaveChanges();
}
SetMessage?.Invoke(this, SetMessage?.Invoke(this,
new MessageEventArgs new MessageEventArgs
@@ -199,8 +203,11 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(machines.Values.ToList(), b => b.SetOutputIdentity = true); ctx.BulkInsert(machines.Values.ToList(), b => b.SetOutputIdentity = true);
ctx.SaveChanges(); ctx.SaveChanges();
}
SetMessage?.Invoke(this, SetMessage?.Invoke(this,
new MessageEventArgs new MessageEventArgs
@@ -248,6 +255,20 @@ public sealed class DatImporter
Maximum = datFile.Items.SortedKeys.Length Maximum = datFile.Items.SortedKeys.Length
}); });
List<DbFile> pendingFilesByCrcList;
List<DbFile> pendingFilesByMd5List;
List<DbFile> pendingFilesBySha1List;
List<DbFile> pendingFilesBySha256List;
List<DbFile> pendingFilesBySha384List;
List<DbFile> pendingFilesBySha512List;
Dictionary<string, DbDisk> pendingDisksByMd5;
Dictionary<string, DbDisk> pendingDisksBySha1;
Dictionary<string, DbMedia> pendingMediasByMd5;
Dictionary<string, DbMedia> pendingMediasBySha1;
Dictionary<string, DbMedia> pendingMediasBySha256;
lock(_dbLock)
{
using(DbTransaction dbTransaction = dbConnection.BeginTransaction()) using(DbTransaction dbTransaction = dbConnection.BeginTransaction())
{ {
DbCommand dbcc = dbConnection.CreateCommand(); DbCommand dbcc = dbConnection.CreateCommand();
@@ -469,80 +490,72 @@ public sealed class DatImporter
dbTransaction.Commit(); dbTransaction.Commit();
} }
List<DbFile> pendingFilesByCrcList = romsHaveCrc pendingFilesByCrcList = romsHaveCrc
? ctx.Files ? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomCrc32Table}] AS t WHERE f.Crc32 = t.Crc32 AND f.Size = t.Size") .FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomCrc32Table}] AS t WHERE f.Crc32 = t.Crc32 AND f.Size = t.Size")
.ToList() .ToList()
: []; : [];
List<DbFile> pendingFilesByMd5List = romsHaveMd5 pendingFilesByMd5List = romsHaveMd5
? ctx.Files ? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomMd5Table}] AS t WHERE f.Md5 = t.Md5 AND f.Size = t.Size") .FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomMd5Table}] AS t WHERE f.Md5 = t.Md5 AND f.Size = t.Size")
.ToList() .ToList()
: []; : [];
List<DbFile> pendingFilesBySha1List = pendingFilesBySha1List = romsHaveSha1
romsHaveSha1
? ctx.Files ? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha1Table}] AS t WHERE f.Sha1 = t.Sha1 AND f.Size = t.Size") .FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha1Table}] AS t WHERE f.Sha1 = t.Sha1 AND f.Size = t.Size")
.ToList() .ToList()
: []; : [];
List<DbFile> pendingFilesBySha256List = pendingFilesBySha256List = romsHaveSha256
romsHaveSha256
? ctx.Files ? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha256Table}] AS t WHERE f.Sha256 = t.Sha256 AND f.Size = t.Size") .FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha256Table}] AS t WHERE f.Sha256 = t.Sha256 AND f.Size = t.Size")
.ToList() .ToList()
: []; : [];
List<DbFile> pendingFilesBySha384List = pendingFilesBySha384List = romsHaveSha384
romsHaveSha384
? ctx.Files ? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha384Table}] AS t WHERE f.Sha384 = t.Sha384 AND f.Size = t.Size") .FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha384Table}] AS t WHERE f.Sha384 = t.Sha384 AND f.Size = t.Size")
.ToList() .ToList()
: []; : [];
List<DbFile> pendingFilesBySha512List = pendingFilesBySha512List = romsHaveSha512
romsHaveSha512
? ctx.Files ? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha512Table}] AS t WHERE f.Sha512 = t.Sha512 AND f.Size = t.Size") .FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha512Table}] AS t WHERE f.Sha512 = t.Sha512 AND f.Size = t.Size")
.ToList() .ToList()
: []; : [];
Dictionary<string, DbDisk> pendingDisksByMd5 = pendingDisksByMd5 = disksHaveMd5
disksHaveMd5
? ctx.Disks ? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskMd5Table}] AS t WHERE f.Md5 = t.Md5") .FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5) .ToDictionary(f => f.Md5)
: []; : [];
Dictionary<string, DbDisk> pendingDisksBySha1 = pendingDisksBySha1 = disksHaveSha1
disksHaveSha1
? ctx.Disks ? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskSha1Table}] AS t WHERE f.Sha1 = t.Sha1") .FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1) .ToDictionary(f => f.Sha1)
: []; : [];
Dictionary<string, DbMedia> pendingMediasByMd5 = pendingMediasByMd5 = mediasHaveMd5
mediasHaveMd5
? ctx.Medias ? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaMd5Table}] AS t WHERE f.Md5 = t.Md5") .FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5) .ToDictionary(f => f.Md5)
: []; : [];
Dictionary<string, DbMedia> pendingMediasBySha1 = pendingMediasBySha1 = mediasHaveSha1
mediasHaveSha1
? ctx.Medias ? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha1Table}] AS t WHERE f.Sha1 = t.Sha1") .FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1) .ToDictionary(f => f.Sha1)
: []; : [];
Dictionary<string, DbMedia> pendingMediasBySha256 = pendingMediasBySha256 = mediasHaveSha256
mediasHaveSha256
? ctx.Medias ? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha256Table}] AS t WHERE f.Sha256 = t.Sha256") .FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha256Table}] AS t WHERE f.Sha256 = t.Sha256")
.ToDictionary(f => f.Sha256) .ToDictionary(f => f.Sha256)
: []; : [];
}
var pendingFilesByCrc = new Dictionary<string, DbFile>(); var pendingFilesByCrc = new Dictionary<string, DbFile>();
var pendingFilesByMd5 = new Dictionary<string, DbFile>(); var pendingFilesByMd5 = new Dictionary<string, DbFile>();
@@ -550,56 +563,21 @@ public sealed class DatImporter
var pendingFilesBySha256 = new Dictionary<string, DbFile>(); var pendingFilesBySha256 = new Dictionary<string, DbFile>();
var pendingFilesBySha384 = new Dictionary<string, DbFile>(); var pendingFilesBySha384 = new Dictionary<string, DbFile>();
var pendingFilesBySha512 = new Dictionary<string, DbFile>(); var pendingFilesBySha512 = new Dictionary<string, DbFile>();
var pendingFiles = new List<DbFile>();
// This is because of hash collisions. var pendingFiles = pendingFilesByCrcList.Where(item => !pendingFilesByCrc.TryAdd(item.Crc32, item))
foreach(DbFile item in pendingFilesByCrcList) .ToList();
{
if(pendingFilesByCrc.ContainsKey(item.Crc32))
pendingFiles.Add(item);
else
pendingFilesByCrc[item.Crc32] = item;
}
foreach(DbFile item in pendingFilesByMd5List) pendingFiles.AddRange(pendingFilesByMd5List.Where(item => !pendingFilesByMd5.TryAdd(item.Md5, item)));
{ pendingFiles.AddRange(pendingFilesBySha1List.Where(item => !pendingFilesBySha1.TryAdd(item.Sha1, item)));
if(pendingFilesByMd5.ContainsKey(item.Md5))
pendingFiles.Add(item);
else
pendingFilesByMd5[item.Md5] = item;
}
foreach(DbFile item in pendingFilesBySha1List) pendingFiles.AddRange(pendingFilesBySha256List.Where(item => !pendingFilesBySha256
{ .TryAdd(item.Sha256, item)));
if(pendingFilesBySha1.ContainsKey(item.Sha1))
pendingFiles.Add(item);
else
pendingFilesBySha1[item.Sha1] = item;
}
foreach(DbFile item in pendingFilesBySha256List) pendingFiles.AddRange(pendingFilesBySha384List.Where(item => !pendingFilesBySha384
{ .TryAdd(item.Sha384, item)));
if(pendingFilesBySha256.ContainsKey(item.Sha256))
pendingFiles.Add(item);
else
pendingFilesBySha256[item.Sha256] = item;
}
foreach(DbFile item in pendingFilesBySha384List) pendingFiles.AddRange(pendingFilesBySha512List.Where(item => !pendingFilesBySha512
{ .TryAdd(item.Sha512, item)));
if(pendingFilesBySha384.ContainsKey(item.Sha384))
pendingFiles.Add(item);
else
pendingFilesBySha384[item.Sha384] = item;
}
foreach(DbFile item in pendingFilesBySha512List)
{
if(pendingFilesBySha512.ContainsKey(item.Sha512))
pendingFiles.Add(item);
else
pendingFilesBySha512[item.Sha512] = item;
}
// Clear some memory // Clear some memory
pendingFilesByCrcList.Clear(); pendingFilesByCrcList.Clear();
@@ -609,6 +587,8 @@ public sealed class DatImporter
pendingFilesBySha384List.Clear(); pendingFilesBySha384List.Clear();
pendingFilesBySha512List.Clear(); pendingFilesBySha512List.Clear();
lock(_dbLock)
{
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomCrc32Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomCrc32Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomMd5Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomMd5Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha1Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha1Table}]");
@@ -620,6 +600,7 @@ public sealed class DatImporter
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaMd5Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaMd5Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha1Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha1Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha256Table}]"); ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha256Table}]");
}
SetProgressBounds?.Invoke(this, SetProgressBounds?.Invoke(this,
new ProgressBoundsEventArgs new ProgressBoundsEventArgs
@@ -923,7 +904,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(newFiles, b => b.SetOutputIdentity = true); ctx.BulkInsert(newFiles, b => b.SetOutputIdentity = true);
}
foreach(FileByMachine fbm in newFilesByMachine) foreach(FileByMachine fbm in newFilesByMachine)
{ {
@@ -931,9 +915,12 @@ public sealed class DatImporter
fbm.MachineId = fbm.Machine.Id; fbm.MachineId = fbm.Machine.Id;
} }
lock(_dbLock)
{
ctx.BulkInsert(newFilesByMachine); ctx.BulkInsert(newFilesByMachine);
ctx.SaveChanges(); ctx.SaveChanges();
}
pendingFilesBySha512.Clear(); pendingFilesBySha512.Clear();
pendingFilesBySha384.Clear(); pendingFilesBySha384.Clear();
@@ -1056,7 +1043,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(newDisks, b => b.SetOutputIdentity = true); ctx.BulkInsert(newDisks, b => b.SetOutputIdentity = true);
}
foreach(DiskByMachine dbm in newDisksByMachine) foreach(DiskByMachine dbm in newDisksByMachine)
{ {
@@ -1064,9 +1054,12 @@ public sealed class DatImporter
dbm.MachineId = dbm.Machine.Id; dbm.MachineId = dbm.Machine.Id;
} }
lock(_dbLock)
{
ctx.BulkInsert(newDisksByMachine); ctx.BulkInsert(newDisksByMachine);
ctx.SaveChanges(); ctx.SaveChanges();
}
pendingDisksBySha1.Clear(); pendingDisksBySha1.Clear();
pendingDisksByMd5.Clear(); pendingDisksByMd5.Clear();
@@ -1203,7 +1196,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(newMedias, b => b.SetOutputIdentity = true); ctx.BulkInsert(newMedias, b => b.SetOutputIdentity = true);
}
foreach(MediaByMachine mbm in newMediasByMachine) foreach(MediaByMachine mbm in newMediasByMachine)
{ {
@@ -1211,17 +1207,23 @@ public sealed class DatImporter
mbm.MachineId = mbm.Machine.Id; mbm.MachineId = mbm.Machine.Id;
} }
lock(_dbLock)
{
ctx.BulkInsert(newMediasByMachine); ctx.BulkInsert(newMediasByMachine);
ctx.SaveChanges(); ctx.SaveChanges();
}
pendingMediasBySha256.Clear(); pendingMediasBySha256.Clear();
pendingMediasBySha1.Clear(); pendingMediasBySha1.Clear();
pendingMediasByMd5.Clear(); pendingMediasByMd5.Clear();
newMedias.Clear(); newMedias.Clear();
newMediasByMachine.Clear(); newMediasByMachine.Clear();
RomSetStat stats;
RomSetStat stats = ctx.RomSets.Where(r => r.Id == romSet.Id) lock(_dbLock)
{
stats = ctx.RomSets.Where(r => r.Id == romSet.Id)
.Select(r => new RomSetStat .Select(r => new RomSetStat
{ {
RomSetId = r.Id, RomSetId = r.Id,
@@ -1268,6 +1270,7 @@ public sealed class DatImporter
ctx.RomSetStats.Add(stats); ctx.RomSetStats.Add(stats);
ctx.SaveChanges(); ctx.SaveChanges();
}
WorkFinished?.Invoke(this, WorkFinished?.Invoke(this,
new MessageEventArgs new MessageEventArgs