claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 11:14:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

Put database context outside of worker. Reduces memory usage by quite a lot.

Browse Source
This commit is contained in:
Natalia Portillo
2025-07-24 04:26:56 +01:00
parent 134f8c7183
commit 2d54cbdf23
2 changed files with 93 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
RomRepoMgr.Core/Workers/FileImporter.cs
View File

@@ -22,19 +22,23 @@ using CompressionMode = SharpCompress.Compressors.CompressionMode;
namespace RomRepoMgr.Core.Workers;
public sealed class FileImporter(bool onlyKnown, bool deleteAfterImport)
public sealed class FileImporter
(
Context _ctx,
ConcurrentBag<DbFile> _newFiles,
ConcurrentBag<DbDisk> _newDisks,
ConcurrentBag<DbMedia> _newMedias,
bool onlyKnown,
bool deleteAfterImport
)
{
const long BUFFER_SIZE = 131072;
static readonly Lock DbLock = new();
readonly Context _ctx = Context.Create(Settings.Settings.Current.DatabasePath);
readonly List<DbDisk> _newDisks = [];
readonly List<DbFile> _newFiles = [];
readonly List<DbMedia> _newMedias = [];
readonly Dictionary<string, DbDisk> _pendingDisksByMd5 = [];
readonly Dictionary<string, DbDisk> _pendingDisksBySha1 = [];
readonly Dictionary<string, DbFile> _pendingFiles = [];
readonly Dictionary<string, DbMedia> _pendingMediasByMd5 = [];
readonly Dictionary<string, DbMedia> _pendingMediasBySha1 = [];
const long BUFFER_SIZE = 131072;
static readonly Lock DbLock = new();
readonly Dictionary<string, DbDisk> _pendingDisksByMd5 = [];
readonly Dictionary<string, DbDisk> _pendingDisksBySha1 = [];
readonly Dictionary<string, DbFile> _pendingFiles = [];
readonly Dictionary<string, DbMedia> _pendingMediasByMd5 = [];
readonly Dictionary<string, DbMedia> _pendingMediasBySha1 = [];
readonly Dictionary<string, DbMedia> _pendingMediasBySha256 = [];
string _archiveFolder;
string _lastMessage;
@@ -1298,13 +1302,13 @@ public sealed class FileImporter(bool onlyKnown, bool deleteAfterImport)
public void SaveChanges()
{
SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty);
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
SetMessage2?.Invoke(this,
new MessageEventArgs
{
Message = Localization.SavingChangesToDatabase
});
SetMessage?.Invoke(this,
new MessageEventArgs
{
Message = Localization.SavingChangesToDatabase
});
lock(DbLock)
{

122
RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs
View File

@@ -12,6 +12,8 @@ using Avalonia.Threading;
using ReactiveUI;
using RomRepoMgr.Core.EventArgs;
using RomRepoMgr.Core.Workers;
using RomRepoMgr.Database;
using RomRepoMgr.Database.Models;
using RomRepoMgr.Models;
using RomRepoMgr.Resources;
@@ -19,31 +21,36 @@ namespace RomRepoMgr.ViewModels;
public class ImportRomFolderViewModel : ViewModelBase
{
bool _canClose;
bool _canStart;
string _folderPath;
bool _isImporting;
bool _isReady;
bool _knownOnlyChecked;
int _listPosition;
bool _progress2IsIndeterminate;
double _progress2Maximum;
double _progress2Minimum;
double _progress2Value;
bool _progress2Visible;
bool _progressIsIndeterminate;
double _progressMaximum;
double _progressMinimum;
double _progressValue;
bool _progressVisible;
bool _recurseArchivesChecked;
bool _removeFilesChecked;
bool _removeFilesEnabled;
FileImporter _rootImporter;
string _statusMessage;
string _statusMessage2;
bool _statusMessage2Visible;
readonly Stopwatch _stopwatch = new();
readonly Context _ctx = Context.Create(Settings.Settings.Current.DatabasePath);
readonly ConcurrentBag<DbDisk> _newDisks = [];
readonly ConcurrentBag<DbFile> _newFiles = [];
readonly ConcurrentBag<DbMedia> _newMedias = [];
readonly Stopwatch _stopwatch = new();
bool _canClose;
bool _canStart;
string _folderPath;
bool _isImporting;
bool _isReady;
bool _knownOnlyChecked;
int _listPosition;
bool _progress2IsIndeterminate;
double _progress2Maximum;
double _progress2Minimum;
double _progress2Value;
bool _progress2Visible;
bool _progressIsIndeterminate;
double _progressMaximum;
double _progressMinimum;
double _progressValue;
bool _progressVisible;
bool _recurseArchivesChecked;
bool _removeFilesChecked;
bool _removeFilesEnabled;
FileImporter _rootImporter;
string _statusMessage;
string _statusMessage2;
bool _statusMessage2Visible;
public ImportRomFolderViewModel()
{
@@ -207,17 +214,17 @@ public class ImportRomFolderViewModel : ViewModelBase
void Start()
{
_rootImporter = new FileImporter(KnownOnlyChecked, RemoveFilesChecked);
_rootImporter.SetMessage += SetMessage;
_rootImporter = new FileImporter(_ctx, _newFiles, _newDisks, _newMedias, KnownOnlyChecked, RemoveFilesChecked);
_rootImporter.SetMessage += SetMessage;
_rootImporter.SetIndeterminateProgress += SetIndeterminateProgress;
_rootImporter.SetProgress += SetProgress;
_rootImporter.SetProgressBounds += SetProgressBounds;
_rootImporter.Finished += EnumeratingFilesFinished;
ProgressIsIndeterminate = true;
ProgressVisible = true;
CanClose = false;
CanStart = false;
IsImporting = true;
_rootImporter.SetProgress += SetProgress;
_rootImporter.SetProgressBounds += SetProgressBounds;
_rootImporter.Finished += EnumeratingFilesFinished;
ProgressIsIndeterminate = true;
ProgressVisible = true;
CanClose = false;
CanStart = false;
IsImporting = true;
_ = Task.Run(() => _rootImporter.FindFiles(FolderPath));
}
@@ -310,7 +317,13 @@ public class ImportRomFolderViewModel : ViewModelBase
Indeterminate = true
};
var worker = new FileImporter(KnownOnlyChecked, RemoveFilesChecked);
var worker = new FileImporter(_ctx,
_newFiles,
_newDisks,
_newMedias,
KnownOnlyChecked,
RemoveFilesChecked);
worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress;
worker.SetMessage2 += model.OnSetMessage;
worker.SetProgress2 += model.OnSetProgress;
@@ -322,13 +335,14 @@ public class ImportRomFolderViewModel : ViewModelBase
worker.ImportFile(file);
worker.SaveChanges();
Interlocked.Increment(ref _listPosition);
});
_stopwatch.Stop();
Console.WriteLine("Took " + _stopwatch.Elapsed.TotalSeconds + " seconds to process files.");
_rootImporter.SaveChanges();
_rootImporter.UpdateRomStats();
_listPosition = 0;
@@ -351,8 +365,8 @@ public class ImportRomFolderViewModel : ViewModelBase
ProgressMinimum = 0;
ProgressValue = 0;
ProgressIsIndeterminate = false;
Progress2Visible = true;
StatusMessage2Visible = true;
Progress2Visible = false;
StatusMessage2Visible = false;
_listPosition = 0;
_stopwatch.Restart();
@@ -365,16 +379,16 @@ public class ImportRomFolderViewModel : ViewModelBase
ProgressValue = _listPosition;
});
// Create FileImporter
var archiveImporter = new FileImporter(KnownOnlyChecked, RemoveFilesChecked);
// Create FileImporter
var archiveImporter = new FileImporter(_ctx,
_newFiles,
_newDisks,
_newMedias,
KnownOnlyChecked,
RemoveFilesChecked);
archiveImporter.SetIndeterminateProgress2 += SetIndeterminateProgress2;
archiveImporter.SetMessage2 += SetMessage2;
archiveImporter.SetProgress2 += SetProgress2;
archiveImporter.SetProgressBounds2 += SetProgress2Bounds;
// Extract archive
bool ret = archiveImporter.ExtractArchive(archive);
// Extract archive
bool ret = archiveImporter.ExtractArchive(archive);
if(!ret) return;
@@ -387,7 +401,13 @@ public class ImportRomFolderViewModel : ViewModelBase
Indeterminate = true
};
var worker = new FileImporter(KnownOnlyChecked, RemoveFilesChecked);
var worker = new FileImporter(_ctx,
_newFiles,
_newDisks,
_newMedias,
KnownOnlyChecked,
RemoveFilesChecked);
worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress;
worker.SetMessage2 += model.OnSetMessage;
worker.SetProgress2 += model.OnSetProgress;
@@ -402,8 +422,8 @@ public class ImportRomFolderViewModel : ViewModelBase
worker.Files.Clear();
}
// Remove temporary files
archiveImporter.CleanupExtractedArchive();
// Remove temporary files
archiveImporter.CleanupExtractedArchive();
Interlocked.Increment(ref _listPosition);
});