claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 19:24:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Compare commits

base: claunia:0903d2b5ccc34431316017b1ee65d78adf1a715a
Branches Tags
claunia:master claunia:snyk-fix-236718b002b55491642dd31b5e7fafde
claunia:v1.0.0-beta.1
...
compare: claunia:b68f6f792fe176a4469b025432a49a92acd8b5cd
Branches Tags
claunia:master claunia:snyk-fix-236718b002b55491642dd31b5e7fafde
claunia:v1.0.0-beta.1

5 Commits
0903d2b5cc ... b68f6f792f

Author SHA1 Message Date
Natalia Portillo
b68f6f792f [App] Remove possible leftover temporary files. 2025-07-29 21:41:15 +01:00
Natalia Portillo
cb4511c09d Instruct parallel foreach to not go over the number of processor threads, it's counter productive. 
And it should not do it by default but apparently, it does.
 2025-07-29 21:36:36 +01:00
Natalia Portillo
2382187960 [App] Refactor importing DAT folder to use parallel foreach. 2025-07-29 21:33:57 +01:00
Natalia Portillo
475da761a5 [App] When an error occurs in a DAT importer worker, keep raising workers. 2025-07-29 16:39:34 +01:00
Natalia Portillo
5d1ef578a1 [App] Ensure DAT importer model events run on UI thread. 2025-07-29 16:38:59 +01:00
5 changed files with 103 additions and 81 deletions
Expand all files Collapse all files

16
RomRepoMgr.Blazor/Components/Dialogs/ImportRoms.razor.cs
View File

@@ -202,6 +202,10 @@ public partial class ImportRoms : ComponentBase
_stopwatch.Restart();
Parallel.ForEach(_rootImporter.Files,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
file =>
{
_ = InvokeAsync(() =>
@@ -284,6 +288,10 @@ public partial class ImportRoms : ComponentBase
// For each archive
Parallel.ForEach(_rootImporter.Archives,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
archive =>
{
_ = InvokeAsync(() =>
@@ -326,11 +334,15 @@ public partial class ImportRoms : ComponentBase
KnownOnlyChecked,
RemoveFilesChecked);
string tmpFile = Path.Combine(Settings.Settings.Current.RepositoryPath,
Path.GetRandomFileName());
worker.ImportAndHashRom(reader.OpenEntryStream(),
reader.Entry.Key,
Path.Combine(Settings.Settings.Current.RepositoryPath,
Path.GetFileName(Path.GetTempFileName())),
tmpFile,
reader.Entry.Size);
if(File.Exists(tmpFile)) File.Delete(tmpFile);
}
}
catch(InvalidOperationException) {}

8
RomRepoMgr.Core/Workers/FileImporter.cs
View File

@@ -99,6 +99,10 @@ public sealed class FileImporter
ConcurrentBag<string> archives = [];
Parallel.ForEach(Files,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
file =>
{
SetProgress?.Invoke(this,
@@ -170,6 +174,10 @@ public sealed class FileImporter
ConcurrentBag<string> archives = [];
Parallel.ForEach(Files,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
file =>
{
try

35
RomRepoMgr/Models/DatImporter.cs
View File

@@ -1,6 +1,6 @@
using System;
using System.Threading.Tasks;
using Avalonia.Media;
using Avalonia.Threading;
using CommunityToolkit.Mvvm.ComponentModel;
using RomRepoMgr.Core.EventArgs;
@@ -21,10 +21,8 @@ public partial class DatImporter : ObservableObject
[ObservableProperty]
string _statusMessage;
public string Filename { get; internal init; }
public Task Task { get; set; }
public bool Running { get; private set; } = true;
internal void OnErrorOccurred(object sender, ErrorEventArgs e)
internal void OnErrorOccurred(object sender, ErrorEventArgs e) => Dispatcher.UIThread.Post(() =>
{
StatusMessage = e.Message;
StatusColor = Colors.Red;
@@ -33,37 +31,30 @@ public partial class DatImporter : ObservableObject
Indeterminate = false;
Progress = 0;
}
});
internal void OnSetIndeterminateProgress(object sender, EventArgs e)
{
Indeterminate = true;
}
internal void OnSetIndeterminateProgress(object sender, EventArgs e) =>
Dispatcher.UIThread.Post(() => Indeterminate = true);
internal void OnSetMessage(object sender, MessageEventArgs e)
{
StatusMessage = e.Message;
}
internal void OnSetMessage(object sender, MessageEventArgs e) =>
Dispatcher.UIThread.Post(() => StatusMessage = e.Message);
internal void OnSetProgress(object sender, ProgressEventArgs e)
{
Progress = e.Value;
}
internal void OnSetProgress(object sender, ProgressEventArgs e) =>
Dispatcher.UIThread.Post(() => Progress = e.Value);
internal void OnSetProgressBounds(object sender, ProgressBoundsEventArgs e)
internal void OnSetProgressBounds(object sender, ProgressBoundsEventArgs e) => Dispatcher.UIThread.Post(() =>
{
Indeterminate = false;
Maximum = e.Maximum;
Minimum = e.Minimum;
}
});
internal void OnWorkFinished(object sender, MessageEventArgs e)
internal void OnWorkFinished(object sender, MessageEventArgs e) => Dispatcher.UIThread.Post(() =>
{
Indeterminate = false;
Maximum = 1;
Minimum = 0;
Progress = 1;
StatusMessage = e.Message;
Running = false;
}
});
}

105
RomRepoMgr/ViewModels/ImportDatFolderViewModel.cs
View File

@@ -4,6 +4,7 @@ using System.Collections.ObjectModel;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Input;
using Avalonia.Controls;
@@ -50,7 +51,6 @@ public sealed partial class ImportDatFolderViewModel : ViewModelBase
bool _recursiveChecked;
[ObservableProperty]
string _statusMessage;
int _workers;
public ImportDatFolderViewModel()
{
@@ -100,68 +100,63 @@ public sealed partial class ImportDatFolderViewModel : ViewModelBase
CanStart = false;
IsReady = false;
IsImporting = true;
_workers = 0;
_stopwatch.Restart();
Import();
_ = Task.Run(Import);
}
void Import()
{
Parallel.ForEach(_datFiles,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
datFile =>
{
Dispatcher.UIThread.Post(() =>
{
StatusMessage = string.Format(Localization.ImportingItem, Path.GetFileName(datFile));
ProgressValue = _listPosition;
});
var model = new DatImporter
{
Filename = Path.GetFileName(datFile),
Minimum = 0,
Maximum = _datFiles.Length,
Progress = 0,
Indeterminate = false
};
var worker = new Core.Workers.DatImporter(datFile,
Category,
new SerilogLoggerFactory(Log.Logger));
worker.ErrorOccurred += model.OnErrorOccurred;
worker.SetIndeterminateProgress += model.OnSetIndeterminateProgress;
worker.SetMessage += model.OnSetMessage;
worker.SetProgress += model.OnSetProgress;
worker.SetProgressBounds += model.OnSetProgressBounds;
worker.WorkFinished += model.OnWorkFinished;
worker.RomSetAdded += RomSetAdded;
Dispatcher.UIThread.Post(() => Importers.Add(model));
worker.Import();
Interlocked.Increment(ref _listPosition);
});
Dispatcher.UIThread.Post(() =>
{
if(_listPosition >= _datFiles.Length)
{
if(_workers != 0) return;
ProgressVisible = false;
StatusMessage = Localization.Finished;
CanClose = true;
CanStart = false;
IsReady = true;
_stopwatch.Stop();
return;
}
StatusMessage = string.Format(Localization.ImportingItem, Path.GetFileName(_datFiles[_listPosition]));
ProgressValue = _listPosition;
var model = new DatImporter
{
Filename = Path.GetFileName(_datFiles[_listPosition]),
Minimum = 0,
Maximum = _datFiles.Length,
Progress = 0,
Indeterminate = false
};
var worker =
new Core.Workers.DatImporter(_datFiles[_listPosition], Category, new SerilogLoggerFactory(Log.Logger));
worker.ErrorOccurred += model.OnErrorOccurred;
worker.SetIndeterminateProgress += model.OnSetIndeterminateProgress;
worker.SetMessage += model.OnSetMessage;
worker.SetProgress += model.OnSetProgress;
worker.SetProgressBounds += model.OnSetProgressBounds;
worker.WorkFinished += model.OnWorkFinished;
worker.RomSetAdded += RomSetAdded;
worker.WorkFinished += (_, _) =>
{
_workers--;
if(_workers < Environment.ProcessorCount) Import();
};
Importers.Add(model);
model.Task = Task.Run(worker.Import);
_workers++;
_listPosition++;
if(_workers < Environment.ProcessorCount) Import();
ProgressVisible = false;
StatusMessage = Localization.Finished;
CanClose = true;
CanStart = false;
IsReady = true;
_stopwatch.Stop();
});
}

20
RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs
View File

@@ -213,6 +213,10 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
_stopwatch.Restart();
Parallel.ForEach(_rootImporter.Files,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
file =>
{
Dispatcher.UIThread.Post(() =>
@@ -284,6 +288,10 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
_stopwatch.Restart();
Parallel.ForEach(_rootImporter.Archives,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
archive =>
{
Dispatcher.UIThread.Post(() =>
@@ -365,6 +373,10 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
_stopwatch.Restart();
Parallel.ForEach(_rootImporter.Archives,
new ParallelOptions
{
MaxDegreeOfParallelism = Environment.ProcessorCount
},
archive =>
{
Dispatcher.UIThread.Post(() =>
@@ -420,11 +432,15 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
Dispatcher.UIThread.Post(() => Importers.Add(model));
string tmpFile = Path.Combine(Settings.Settings.Current.RepositoryPath,
Path.GetRandomFileName());
worker.ImportAndHashRom(reader.OpenEntryStream(),
reader.Entry.Key,
Path.Combine(Settings.Settings.Current.RepositoryPath,
Path.GetFileName(Path.GetTempFileName())),
tmpFile,
reader.Entry.Size);
if(File.Exists(tmpFile)) File.Delete(tmpFile);
}
}
catch(InvalidOperationException) {}