claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 11:14:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Put database context outside of worker. Reduces memory usage by quite a lot.

Browse Source
This commit is contained in:
Natalia Portillo
2025-07-24 04:26:56 +01:00
parent 134f8c7183
commit 2d54cbdf23
2 changed files with 93 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
RomRepoMgr.Core/Workers/FileImporter.cs
View File

@@ -22,19 +22,23 @@ using CompressionMode = SharpCompress.Compressors.CompressionMode;
namespace RomRepoMgr.Core.Workers; namespace RomRepoMgr.Core.Workers;
public sealed class FileImporter(bool onlyKnown, bool deleteAfterImport) public sealed class FileImporter
(
Context _ctx,
ConcurrentBag<DbFile> _newFiles,
ConcurrentBag<DbDisk> _newDisks,
ConcurrentBag<DbMedia> _newMedias,
bool onlyKnown,
bool deleteAfterImport
)
{ {
const long BUFFER_SIZE = 131072; const long BUFFER_SIZE = 131072;
static readonly Lock DbLock = new(); static readonly Lock DbLock = new();
readonly Context _ctx = Context.Create(Settings.Settings.Current.DatabasePath); readonly Dictionary<string, DbDisk> _pendingDisksByMd5 = [];
readonly List<DbDisk> _newDisks = []; readonly Dictionary<string, DbDisk> _pendingDisksBySha1 = [];
readonly List<DbFile> _newFiles = []; readonly Dictionary<string, DbFile> _pendingFiles = [];
readonly List<DbMedia> _newMedias = []; readonly Dictionary<string, DbMedia> _pendingMediasByMd5 = [];
readonly Dictionary<string, DbDisk> _pendingDisksByMd5 = []; readonly Dictionary<string, DbMedia> _pendingMediasBySha1 = [];
readonly Dictionary<string, DbDisk> _pendingDisksBySha1 = [];
readonly Dictionary<string, DbFile> _pendingFiles = [];
readonly Dictionary<string, DbMedia> _pendingMediasByMd5 = [];
readonly Dictionary<string, DbMedia> _pendingMediasBySha1 = [];
readonly Dictionary<string, DbMedia> _pendingMediasBySha256 = []; readonly Dictionary<string, DbMedia> _pendingMediasBySha256 = [];
string _archiveFolder; string _archiveFolder;
string _lastMessage; string _lastMessage;
@@ -1298,13 +1302,13 @@ public sealed class FileImporter(bool onlyKnown, bool deleteAfterImport)
public void SaveChanges() public void SaveChanges()
{ {
SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty); SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
SetMessage2?.Invoke(this, SetMessage?.Invoke(this,
new MessageEventArgs new MessageEventArgs
{ {
Message = Localization.SavingChangesToDatabase Message = Localization.SavingChangesToDatabase
}); });
lock(DbLock) lock(DbLock)
{ {

122
RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs
View File

@@ -12,6 +12,8 @@ using Avalonia.Threading;
using ReactiveUI; using ReactiveUI;
using RomRepoMgr.Core.EventArgs; using RomRepoMgr.Core.EventArgs;
using RomRepoMgr.Core.Workers; using RomRepoMgr.Core.Workers;
using RomRepoMgr.Database;
using RomRepoMgr.Database.Models;
using RomRepoMgr.Models; using RomRepoMgr.Models;
using RomRepoMgr.Resources; using RomRepoMgr.Resources;
@@ -19,31 +21,36 @@ namespace RomRepoMgr.ViewModels;
public class ImportRomFolderViewModel : ViewModelBase public class ImportRomFolderViewModel : ViewModelBase
{ {
bool _canClose; readonly Context _ctx = Context.Create(Settings.Settings.Current.DatabasePath);
bool _canStart; readonly ConcurrentBag<DbDisk> _newDisks = [];
string _folderPath; readonly ConcurrentBag<DbFile> _newFiles = [];
bool _isImporting; readonly ConcurrentBag<DbMedia> _newMedias = [];
bool _isReady; readonly Stopwatch _stopwatch = new();
bool _knownOnlyChecked; bool _canClose;
int _listPosition; bool _canStart;
bool _progress2IsIndeterminate; string _folderPath;
double _progress2Maximum; bool _isImporting;
double _progress2Minimum; bool _isReady;
double _progress2Value; bool _knownOnlyChecked;
bool _progress2Visible; int _listPosition;
bool _progressIsIndeterminate; bool _progress2IsIndeterminate;
double _progressMaximum; double _progress2Maximum;
double _progressMinimum; double _progress2Minimum;
double _progressValue; double _progress2Value;
bool _progressVisible; bool _progress2Visible;
bool _recurseArchivesChecked; bool _progressIsIndeterminate;
bool _removeFilesChecked; double _progressMaximum;
bool _removeFilesEnabled; double _progressMinimum;
FileImporter _rootImporter; double _progressValue;
string _statusMessage; bool _progressVisible;
string _statusMessage2; bool _recurseArchivesChecked;
bool _statusMessage2Visible; bool _removeFilesChecked;
readonly Stopwatch _stopwatch = new(); bool _removeFilesEnabled;
FileImporter _rootImporter;
string _statusMessage;
string _statusMessage2;
bool _statusMessage2Visible;
public ImportRomFolderViewModel() public ImportRomFolderViewModel()
{ {
@@ -207,17 +214,17 @@ public class ImportRomFolderViewModel : ViewModelBase
void Start() void Start()
{ {
_rootImporter = new FileImporter(KnownOnlyChecked, RemoveFilesChecked); _rootImporter = new FileImporter(_ctx, _newFiles, _newDisks, _newMedias, KnownOnlyChecked, RemoveFilesChecked);
_rootImporter.SetMessage += SetMessage; _rootImporter.SetMessage += SetMessage;
_rootImporter.SetIndeterminateProgress += SetIndeterminateProgress; _rootImporter.SetIndeterminateProgress += SetIndeterminateProgress;
_rootImporter.SetProgress += SetProgress; _rootImporter.SetProgress += SetProgress;
_rootImporter.SetProgressBounds += SetProgressBounds; _rootImporter.SetProgressBounds += SetProgressBounds;
_rootImporter.Finished += EnumeratingFilesFinished; _rootImporter.Finished += EnumeratingFilesFinished;
ProgressIsIndeterminate = true; ProgressIsIndeterminate = true;
ProgressVisible = true; ProgressVisible = true;
CanClose = false; CanClose = false;
CanStart = false; CanStart = false;
IsImporting = true; IsImporting = true;
_ = Task.Run(() => _rootImporter.FindFiles(FolderPath)); _ = Task.Run(() => _rootImporter.FindFiles(FolderPath));
} }
@@ -310,7 +317,13 @@ public class ImportRomFolderViewModel : ViewModelBase
Indeterminate = true Indeterminate = true
}; };
var worker = new FileImporter(KnownOnlyChecked, RemoveFilesChecked); var worker = new FileImporter(_ctx,
_newFiles,
_newDisks,
_newMedias,
KnownOnlyChecked,
RemoveFilesChecked);
worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress; worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress;
worker.SetMessage2 += model.OnSetMessage; worker.SetMessage2 += model.OnSetMessage;
worker.SetProgress2 += model.OnSetProgress; worker.SetProgress2 += model.OnSetProgress;
@@ -322,13 +335,14 @@ public class ImportRomFolderViewModel : ViewModelBase
worker.ImportFile(file); worker.ImportFile(file);
worker.SaveChanges();
Interlocked.Increment(ref _listPosition); Interlocked.Increment(ref _listPosition);
}); });
_stopwatch.Stop(); _stopwatch.Stop();
Console.WriteLine("Took " + _stopwatch.Elapsed.TotalSeconds + " seconds to process files."); Console.WriteLine("Took " + _stopwatch.Elapsed.TotalSeconds + " seconds to process files.");
_rootImporter.SaveChanges();
_rootImporter.UpdateRomStats(); _rootImporter.UpdateRomStats();
_listPosition = 0; _listPosition = 0;
@@ -351,8 +365,8 @@ public class ImportRomFolderViewModel : ViewModelBase
ProgressMinimum = 0; ProgressMinimum = 0;
ProgressValue = 0; ProgressValue = 0;
ProgressIsIndeterminate = false; ProgressIsIndeterminate = false;
Progress2Visible = true; Progress2Visible = false;
StatusMessage2Visible = true; StatusMessage2Visible = false;
_listPosition = 0; _listPosition = 0;
_stopwatch.Restart(); _stopwatch.Restart();
@@ -365,16 +379,16 @@ public class ImportRomFolderViewModel : ViewModelBase
ProgressValue = _listPosition; ProgressValue = _listPosition;
}); });
// Create FileImporter // Create FileImporter
var archiveImporter = new FileImporter(KnownOnlyChecked, RemoveFilesChecked); var archiveImporter = new FileImporter(_ctx,
_newFiles,
_newDisks,
_newMedias,
KnownOnlyChecked,
RemoveFilesChecked);
archiveImporter.SetIndeterminateProgress2 += SetIndeterminateProgress2; // Extract archive
archiveImporter.SetMessage2 += SetMessage2; bool ret = archiveImporter.ExtractArchive(archive);
archiveImporter.SetProgress2 += SetProgress2;
archiveImporter.SetProgressBounds2 += SetProgress2Bounds;
// Extract archive
bool ret = archiveImporter.ExtractArchive(archive);
if(!ret) return; if(!ret) return;
@@ -387,7 +401,13 @@ public class ImportRomFolderViewModel : ViewModelBase
Indeterminate = true Indeterminate = true
}; };
var worker = new FileImporter(KnownOnlyChecked, RemoveFilesChecked); var worker = new FileImporter(_ctx,
_newFiles,
_newDisks,
_newMedias,
KnownOnlyChecked,
RemoveFilesChecked);
worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress; worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress;
worker.SetMessage2 += model.OnSetMessage; worker.SetMessage2 += model.OnSetMessage;
worker.SetProgress2 += model.OnSetProgress; worker.SetProgress2 += model.OnSetProgress;
@@ -402,8 +422,8 @@ public class ImportRomFolderViewModel : ViewModelBase
worker.Files.Clear(); worker.Files.Clear();
} }
// Remove temporary files // Remove temporary files
archiveImporter.CleanupExtractedArchive(); archiveImporter.CleanupExtractedArchive();
Interlocked.Increment(ref _listPosition); Interlocked.Increment(ref _listPosition);
}); });