claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 19:24:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Optimize SQL calls when importing.

Browse Source
This commit is contained in:
Natalia Portillo
2020-09-03 19:45:39 +01:00
parent 57bda701f2
commit 7999e99c2d
2 changed files with 32 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
RomRepoMgr.Core/Workers/DatImporter.cs
View File

@@ -224,6 +224,8 @@ namespace RomRepoMgr.Core.Workers
Dictionary<string, DbFile> pendingFilesByMd5 = new Dictionary<string, DbFile>(); Dictionary<string, DbFile> pendingFilesByMd5 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesByCrc = new Dictionary<string, DbFile>(); Dictionary<string, DbFile> pendingFilesByCrc = new Dictionary<string, DbFile>();
List<DbFile> pendingFiles = new List<DbFile>(); List<DbFile> pendingFiles = new List<DbFile>();
List<DbFile> newFiles = new List<DbFile>();
List<FileByMachine> newFilesByMachine = new List<FileByMachine>();
foreach(Rom rom in roms) foreach(Rom rom in roms)
{ {
@@ -327,29 +329,12 @@ namespace RomRepoMgr.Core.Workers
file = pendingFiles.FirstOrDefault(f => f.Crc32 == rom.CRC && f.Size == uSize); file = pendingFiles.FirstOrDefault(f => f.Crc32 == rom.CRC && f.Size == uSize);
} }
if(file == null && file ??= Context.Singleton.Files.FirstOrDefault(f => ((rom.SHA512 != null && f.Sha512 == rom.SHA512) ||
rom.SHA512 != null) (rom.SHA384 != null && f.Sha384 == rom.SHA384) ||
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha512 == rom.SHA512 && f.Size == uSize); (rom.SHA256 != null && f.Sha256 == rom.SHA256) ||
(rom.SHA1 != null && f.Sha1 == rom.SHA1) ||
if(file == null && (rom.MD5 != null && f.Md5 == rom.MD5) ||
rom.SHA384 != null) (rom.CRC != null && f.Crc32 == rom.CRC)) && f.Size == uSize);
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha384 == rom.SHA384 && f.Size == uSize);
if(file == null &&
rom.SHA256 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha256 == rom.SHA256 && f.Size == uSize);
if(file == null &&
rom.SHA1 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha1 == rom.SHA1 && f.Size == uSize);
if(file == null &&
rom.MD5 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Md5 == rom.MD5 && f.Size == uSize);
if(file == null &&
rom.CRC != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Crc32 == rom.CRC && f.Size == uSize);
if(file == null) if(file == null)
{ {
@@ -366,7 +351,9 @@ namespace RomRepoMgr.Core.Workers
UpdatedOn = DateTime.UtcNow UpdatedOn = DateTime.UtcNow
}; };
Context.Singleton.Files.Add(file); newFiles.Add(file);
//Context.Singleton.Files.Add(file);
} }
if(string.IsNullOrEmpty(file.Crc32) && if(string.IsNullOrEmpty(file.Crc32) &&
@@ -411,7 +398,7 @@ namespace RomRepoMgr.Core.Workers
file.UpdatedOn = DateTime.UtcNow; file.UpdatedOn = DateTime.UtcNow;
} }
Context.Singleton.FilesByMachines.Add(new FileByMachine newFilesByMachine.Add(new FileByMachine
{ {
File = file, File = file,
Machine = machine, Machine = machine,
@@ -443,6 +430,9 @@ namespace RomRepoMgr.Core.Workers
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
Context.Singleton.Files.AddRange(newFiles);
Context.Singleton.FilesByMachines.AddRange(newFilesByMachine);
Context.Singleton.SaveChanges(); Context.Singleton.SaveChanges();
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs

29
RomRepoMgr.Core/Workers/FileImporter.cs
View File

@@ -19,6 +19,7 @@ namespace RomRepoMgr.Core.Workers
{ {
const long BUFFER_SIZE = 131072; const long BUFFER_SIZE = 131072;
readonly bool _deleteAfterImport; readonly bool _deleteAfterImport;
readonly List<DbFile> _newFiles;
readonly bool _onlyKnown; readonly bool _onlyKnown;
readonly Dictionary<string, DbFile> _pendingFiles; readonly Dictionary<string, DbFile> _pendingFiles;
@@ -30,6 +31,7 @@ namespace RomRepoMgr.Core.Workers
public FileImporter(bool onlyKnown, bool deleteAfterImport) public FileImporter(bool onlyKnown, bool deleteAfterImport)
{ {
_pendingFiles = new Dictionary<string, DbFile>(); _pendingFiles = new Dictionary<string, DbFile>();
_newFiles = new List<DbFile>();
_onlyKnown = onlyKnown; _onlyKnown = onlyKnown;
_deleteAfterImport = deleteAfterImport; _deleteAfterImport = deleteAfterImport;
_position = 0; _position = 0;
@@ -262,19 +264,13 @@ namespace RomRepoMgr.Core.Workers
bool knownFile = _pendingFiles.TryGetValue(checksums[ChecksumType.Sha512], out DbFile dbFile); bool knownFile = _pendingFiles.TryGetValue(checksums[ChecksumType.Sha512], out DbFile dbFile);
dbFile ??= dbFile ??= Context.Singleton.Files.FirstOrDefault(f => (f.Sha512 == checksums[ChecksumType.Sha512] ||
((((Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Sha384 == checksums[ChecksumType.Sha384] ||
f.Sha512 == checksums[ChecksumType.Sha512]) ?? f.Sha256 == checksums[ChecksumType.Sha256] ||
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Sha1 == checksums[ChecksumType.Sha1] ||
f.Sha384 == checksums[ChecksumType.Sha384])) ?? f.Md5 == checksums[ChecksumType.Md5] ||
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Crc32 == checksums[ChecksumType.Crc32]) &&
f.Sha256 == checksums[ChecksumType.Sha256])) ?? f.Size == uSize);
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
f.Sha1 == checksums[ChecksumType.Sha1])) ??
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
f.Md5 == checksums[ChecksumType.Md5])) ??
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
f.Crc32 == checksums[ChecksumType.Crc32]);
if(dbFile == null) if(dbFile == null)
{ {
@@ -384,7 +380,7 @@ namespace RomRepoMgr.Core.Workers
dbFile.UpdatedOn = DateTime.UtcNow; dbFile.UpdatedOn = DateTime.UtcNow;
if(!fileInDb) if(!fileInDb)
Context.Singleton.Files.Add(dbFile); _newFiles.Add(dbFile);
inFs.Close(); inFs.Close();
@@ -449,7 +445,7 @@ namespace RomRepoMgr.Core.Workers
dbFile.UpdatedOn = DateTime.UtcNow; dbFile.UpdatedOn = DateTime.UtcNow;
if(!fileInDb) if(!fileInDb)
Context.Singleton.Files.Add(dbFile); _newFiles.Add(dbFile);
if(_deleteAfterImport) if(_deleteAfterImport)
File.Delete(path); File.Delete(path);
@@ -473,7 +469,10 @@ namespace RomRepoMgr.Core.Workers
Message = Localization.SavingChangesToDatabase Message = Localization.SavingChangesToDatabase
}); });
Context.Singleton.Files.AddRange(_newFiles);
Context.Singleton.SaveChanges(); Context.Singleton.SaveChanges();
_newFiles.Clear();
} }
string GetArchiveFormat(string path, out long counter) string GetArchiveFormat(string path, out long counter)