claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 19:24:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

[Workers] Implement database locking in DatImporter for thread safety

Browse Source
This commit is contained in:
Natalia Portillo
2025-07-11 18:40:14 +01:00
parent 32bbb55e61
commit 8eaca3556a
1 changed files with 355 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

135
RomRepoMgr.Core/Workers/DatImporter.cs
View File

@@ -53,6 +53,7 @@ namespace RomRepoMgr.Core.Workers;
public sealed class DatImporter
{
private static readonly object _dbLock = new();
readonly string _category;
readonly string _datFilesPath;
readonly string _datPath;
@@ -132,8 +133,11 @@ public sealed class DatImporter
Category = _category
};
lock(_dbLock)
{
ctx.RomSets.Add(romSet);
ctx.SaveChanges();
}
SetMessage?.Invoke(this,
new MessageEventArgs
@@ -199,8 +203,11 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(machines.Values.ToList(), b => b.SetOutputIdentity = true);
ctx.SaveChanges();
}
SetMessage?.Invoke(this,
new MessageEventArgs
@@ -248,6 +255,20 @@ public sealed class DatImporter
Maximum = datFile.Items.SortedKeys.Length
});
List<DbFile> pendingFilesByCrcList;
List<DbFile> pendingFilesByMd5List;
List<DbFile> pendingFilesBySha1List;
List<DbFile> pendingFilesBySha256List;
List<DbFile> pendingFilesBySha384List;
List<DbFile> pendingFilesBySha512List;
Dictionary<string, DbDisk> pendingDisksByMd5;
Dictionary<string, DbDisk> pendingDisksBySha1;
Dictionary<string, DbMedia> pendingMediasByMd5;
Dictionary<string, DbMedia> pendingMediasBySha1;
Dictionary<string, DbMedia> pendingMediasBySha256;
lock(_dbLock)
{
using(DbTransaction dbTransaction = dbConnection.BeginTransaction())
{
DbCommand dbcc = dbConnection.CreateCommand();
@@ -469,80 +490,72 @@ public sealed class DatImporter
dbTransaction.Commit();
}
List<DbFile> pendingFilesByCrcList = romsHaveCrc
pendingFilesByCrcList = romsHaveCrc
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomCrc32Table}] AS t WHERE f.Crc32 = t.Crc32 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesByMd5List = romsHaveMd5
pendingFilesByMd5List = romsHaveMd5
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomMd5Table}] AS t WHERE f.Md5 = t.Md5 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha1List =
romsHaveSha1
pendingFilesBySha1List = romsHaveSha1
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha1Table}] AS t WHERE f.Sha1 = t.Sha1 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha256List =
romsHaveSha256
pendingFilesBySha256List = romsHaveSha256
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha256Table}] AS t WHERE f.Sha256 = t.Sha256 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha384List =
romsHaveSha384
pendingFilesBySha384List = romsHaveSha384
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha384Table}] AS t WHERE f.Sha384 = t.Sha384 AND f.Size = t.Size")
.ToList()
: [];
List<DbFile> pendingFilesBySha512List =
romsHaveSha512
pendingFilesBySha512List = romsHaveSha512
? ctx.Files
.FromSqlRaw($"SELECT DISTINCT f.* FROM Files AS f, [{tmpRomSha512Table}] AS t WHERE f.Sha512 = t.Sha512 AND f.Size = t.Size")
.ToList()
: [];
Dictionary<string, DbDisk> pendingDisksByMd5 =
disksHaveMd5
pendingDisksByMd5 = disksHaveMd5
? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5)
: [];
Dictionary<string, DbDisk> pendingDisksBySha1 =
disksHaveSha1
pendingDisksBySha1 = disksHaveSha1
? ctx.Disks
.FromSqlRaw($"SELECT DISTINCT f.* FROM Disks AS f, [{tmpDiskSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1)
: [];
Dictionary<string, DbMedia> pendingMediasByMd5 =
mediasHaveMd5
pendingMediasByMd5 = mediasHaveMd5
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaMd5Table}] AS t WHERE f.Md5 = t.Md5")
.ToDictionary(f => f.Md5)
: [];
Dictionary<string, DbMedia> pendingMediasBySha1 =
mediasHaveSha1
pendingMediasBySha1 = mediasHaveSha1
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha1Table}] AS t WHERE f.Sha1 = t.Sha1")
.ToDictionary(f => f.Sha1)
: [];
Dictionary<string, DbMedia> pendingMediasBySha256 =
mediasHaveSha256
pendingMediasBySha256 = mediasHaveSha256
? ctx.Medias
.FromSqlRaw($"SELECT DISTINCT f.* FROM Medias AS f, [{tmpMediaSha256Table}] AS t WHERE f.Sha256 = t.Sha256")
.ToDictionary(f => f.Sha256)
: [];
}
var pendingFilesByCrc = new Dictionary<string, DbFile>();
var pendingFilesByMd5 = new Dictionary<string, DbFile>();
@@ -550,56 +563,21 @@ public sealed class DatImporter
var pendingFilesBySha256 = new Dictionary<string, DbFile>();
var pendingFilesBySha384 = new Dictionary<string, DbFile>();
var pendingFilesBySha512 = new Dictionary<string, DbFile>();
var pendingFiles = new List<DbFile>();
// This is because of hash collisions.
foreach(DbFile item in pendingFilesByCrcList)
{
if(pendingFilesByCrc.ContainsKey(item.Crc32))
pendingFiles.Add(item);
else
pendingFilesByCrc[item.Crc32] = item;
}
var pendingFiles = pendingFilesByCrcList.Where(item => !pendingFilesByCrc.TryAdd(item.Crc32, item))
.ToList();
foreach(DbFile item in pendingFilesByMd5List)
{
if(pendingFilesByMd5.ContainsKey(item.Md5))
pendingFiles.Add(item);
else
pendingFilesByMd5[item.Md5] = item;
}
pendingFiles.AddRange(pendingFilesByMd5List.Where(item => !pendingFilesByMd5.TryAdd(item.Md5, item)));
pendingFiles.AddRange(pendingFilesBySha1List.Where(item => !pendingFilesBySha1.TryAdd(item.Sha1, item)));
foreach(DbFile item in pendingFilesBySha1List)
{
if(pendingFilesBySha1.ContainsKey(item.Sha1))
pendingFiles.Add(item);
else
pendingFilesBySha1[item.Sha1] = item;
}
pendingFiles.AddRange(pendingFilesBySha256List.Where(item => !pendingFilesBySha256
.TryAdd(item.Sha256, item)));
foreach(DbFile item in pendingFilesBySha256List)
{
if(pendingFilesBySha256.ContainsKey(item.Sha256))
pendingFiles.Add(item);
else
pendingFilesBySha256[item.Sha256] = item;
}
pendingFiles.AddRange(pendingFilesBySha384List.Where(item => !pendingFilesBySha384
.TryAdd(item.Sha384, item)));
foreach(DbFile item in pendingFilesBySha384List)
{
if(pendingFilesBySha384.ContainsKey(item.Sha384))
pendingFiles.Add(item);
else
pendingFilesBySha384[item.Sha384] = item;
}
foreach(DbFile item in pendingFilesBySha512List)
{
if(pendingFilesBySha512.ContainsKey(item.Sha512))
pendingFiles.Add(item);
else
pendingFilesBySha512[item.Sha512] = item;
}
pendingFiles.AddRange(pendingFilesBySha512List.Where(item => !pendingFilesBySha512
.TryAdd(item.Sha512, item)));
// Clear some memory
pendingFilesByCrcList.Clear();
@@ -609,6 +587,8 @@ public sealed class DatImporter
pendingFilesBySha384List.Clear();
pendingFilesBySha512List.Clear();
lock(_dbLock)
{
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomCrc32Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomMd5Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpRomSha1Table}]");
@@ -620,6 +600,7 @@ public sealed class DatImporter
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaMd5Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha1Table}]");
ctx.Database.ExecuteSqlRaw($"DROP TABLE [{tmpMediaSha256Table}]");
}
SetProgressBounds?.Invoke(this,
new ProgressBoundsEventArgs
@@ -923,7 +904,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(newFiles, b => b.SetOutputIdentity = true);
}
foreach(FileByMachine fbm in newFilesByMachine)
{
@@ -931,9 +915,12 @@ public sealed class DatImporter
fbm.MachineId = fbm.Machine.Id;
}
lock(_dbLock)
{
ctx.BulkInsert(newFilesByMachine);
ctx.SaveChanges();
}
pendingFilesBySha512.Clear();
pendingFilesBySha384.Clear();
@@ -1056,7 +1043,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(newDisks, b => b.SetOutputIdentity = true);
}
foreach(DiskByMachine dbm in newDisksByMachine)
{
@@ -1064,9 +1054,12 @@ public sealed class DatImporter
dbm.MachineId = dbm.Machine.Id;
}
lock(_dbLock)
{
ctx.BulkInsert(newDisksByMachine);
ctx.SaveChanges();
}
pendingDisksBySha1.Clear();
pendingDisksByMd5.Clear();
@@ -1203,7 +1196,10 @@ public sealed class DatImporter
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
lock(_dbLock)
{
ctx.BulkInsert(newMedias, b => b.SetOutputIdentity = true);
}
foreach(MediaByMachine mbm in newMediasByMachine)
{
@@ -1211,17 +1207,23 @@ public sealed class DatImporter
mbm.MachineId = mbm.Machine.Id;
}
lock(_dbLock)
{
ctx.BulkInsert(newMediasByMachine);
ctx.SaveChanges();
}
pendingMediasBySha256.Clear();
pendingMediasBySha1.Clear();
pendingMediasByMd5.Clear();
newMedias.Clear();
newMediasByMachine.Clear();
RomSetStat stats;
RomSetStat stats = ctx.RomSets.Where(r => r.Id == romSet.Id)
lock(_dbLock)
{
stats = ctx.RomSets.Where(r => r.Id == romSet.Id)
.Select(r => new RomSetStat
{
RomSetId = r.Id,
@@ -1268,6 +1270,7 @@ public sealed class DatImporter
ctx.RomSetStats.Add(stats);
ctx.SaveChanges();
}
WorkFinished?.Invoke(this,
new MessageEventArgs