From 054b9ffd0dc48ee53668992d82ae29cf541af47e Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Sat, 26 Jul 2025 17:01:30 +0100 Subject: [PATCH] When using internal decompressor, bypass extracting files to temporary folder. Not needed and is slower. --- .../Resources/Localization.Designer.cs | 6 + .../Resources/Localization.es.resx | 3 + RomRepoMgr.Core/Resources/Localization.resx | 3 + RomRepoMgr.Core/Workers/FileImporter.cs | 291 +++++++++++++++++- RomRepoMgr/Resources/Localization.Designer.cs | 6 + RomRepoMgr/Resources/Localization.es.resx | 3 + RomRepoMgr/Resources/Localization.resx | 3 + .../ViewModels/ImportRomFolderViewModel.cs | 101 +++++- 8 files changed, 412 insertions(+), 4 deletions(-) diff --git a/RomRepoMgr.Core/Resources/Localization.Designer.cs b/RomRepoMgr.Core/Resources/Localization.Designer.cs index 00ecd66..3df0588 100644 --- a/RomRepoMgr.Core/Resources/Localization.Designer.cs +++ b/RomRepoMgr.Core/Resources/Localization.Designer.cs @@ -392,5 +392,11 @@ namespace RomRepoMgr.Core.Resources { return ResourceManager.GetString("DatImportSuccess", resourceCulture); } } + + internal static string ImportingFile { + get { + return ResourceManager.GetString("ImportingFile", resourceCulture); + } + } } } diff --git a/RomRepoMgr.Core/Resources/Localization.es.resx b/RomRepoMgr.Core/Resources/Localization.es.resx index ebdacd0..0e99fb8 100644 --- a/RomRepoMgr.Core/Resources/Localization.es.resx +++ b/RomRepoMgr.Core/Resources/Localization.es.resx @@ -186,4 +186,7 @@ Añadiendo medios... + + Importando fichero... + \ No newline at end of file diff --git a/RomRepoMgr.Core/Resources/Localization.resx b/RomRepoMgr.Core/Resources/Localization.resx index f86ef79..4634a7f 100644 --- a/RomRepoMgr.Core/Resources/Localization.resx +++ b/RomRepoMgr.Core/Resources/Localization.resx @@ -196,4 +196,7 @@ Imported {0} machines with {1} ROMs. + + Importing file... + \ No newline at end of file diff --git a/RomRepoMgr.Core/Workers/FileImporter.cs b/RomRepoMgr.Core/Workers/FileImporter.cs index d1df0d3..2084f33 100644 --- a/RomRepoMgr.Core/Workers/FileImporter.cs +++ b/RomRepoMgr.Core/Workers/FileImporter.cs @@ -540,6 +540,295 @@ public sealed class FileImporter } } + public bool IsCrcInDb(long crc32) + { + lock(DbLock) + { + return _ctx.Files.Any(f => f.Crc32 == crc32.ToString("x8")); + } + } + + public void ImportAndHashRom(Stream stream, string filename, string tempPath, long size) + { + try + { + var outFs = new FileStream(tempPath, FileMode.Create, FileAccess.Write); + + SetMessage2?.Invoke(this, + new MessageEventArgs + { + Message = Localization.ImportingFile + }); + + byte[] buffer; + var checksumWorker = new Checksum(); + Stream zStream; + + switch(Settings.Settings.Current.Compression) + { + case CompressionType.Zstd: + { + var zstdStream = new CompressionStream(outFs, 15); + zstdStream.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, Environment.ProcessorCount); + zStream = zstdStream; + + break; + } + case CompressionType.None: + zStream = outFs; + + break; + default: + zStream = new LZipStream(outFs, CompressionMode.Compress); + + break; + } + + + if(size > BUFFER_SIZE) + { + SetProgressBounds2?.Invoke(this, + new ProgressBoundsEventArgs + { + Minimum = 0, + Maximum = size + }); + + long offset; + long remainder = size % BUFFER_SIZE; + + for(offset = 0; offset < size - remainder; offset += (int)BUFFER_SIZE) + { + SetProgress2?.Invoke(this, + new ProgressEventArgs + { + Value = offset + }); + + buffer = new byte[BUFFER_SIZE]; + stream.EnsureRead(buffer, 0, (int)BUFFER_SIZE); + checksumWorker.Update(buffer); + zStream.Write(buffer, 0, buffer.Length); + } + + SetProgress2?.Invoke(this, + new ProgressEventArgs + { + Value = offset + }); + + buffer = new byte[remainder]; + stream.EnsureRead(buffer, 0, (int)remainder); + } + else + { + SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty); + buffer = new byte[size]; + stream.EnsureRead(buffer, 0, (int)size); + } + + checksumWorker.Update(buffer); + zStream.Write(buffer, 0, buffer.Length); + + Dictionary checksums = checksumWorker.End(); + + ulong uSize = (ulong)size; + bool fileInDb = true; + + bool knownFile = _pendingFiles.TryGetValue(checksums[ChecksumType.Sha512], out DbFile dbFile); + + lock(DbLock) + { + dbFile ??= _ctx.Files.FirstOrDefault(f => (f.Sha512 == checksums[ChecksumType.Sha512] || + f.Sha384 == checksums[ChecksumType.Sha384] || + f.Sha256 == checksums[ChecksumType.Sha256] || + f.Sha1 == checksums[ChecksumType.Sha1] || + f.Md5 == checksums[ChecksumType.Md5] || + f.Crc32 == checksums[ChecksumType.Crc32]) && + f.Size == uSize); + } + + if(dbFile == null) + { + if(onlyKnown) + { + ImportedRom?.Invoke(this, + new ImportedRomItemEventArgs + { + Item = new ImportRomItem + { + Filename = filename, + Status = Localization.UnknownFile + } + }); + + return; + } + + dbFile = new DbFile + { + Crc32 = checksums[ChecksumType.Crc32], + Md5 = checksums[ChecksumType.Md5], + Sha1 = checksums[ChecksumType.Sha1], + Sha256 = checksums[ChecksumType.Sha256], + Sha384 = checksums[ChecksumType.Sha384], + Sha512 = checksums[ChecksumType.Sha512], + Size = uSize, + CreatedOn = DateTime.UtcNow, + UpdatedOn = DateTime.UtcNow, + OriginalFileName = Path.GetFileName(filename) + }; + + fileInDb = false; + } + + if(!knownFile) _pendingFiles[checksums[ChecksumType.Sha512]] = dbFile; + + byte[] sha384Bytes = new byte[48]; + string sha384 = checksums[ChecksumType.Sha384]; + + for(int i = 0; i < 48; i++) + { + if(sha384[i * 2] >= 0x30 && sha384[i * 2] <= 0x39) + sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10); + else if(sha384[i * 2] >= 0x41 && sha384[i * 2] <= 0x46) + sha384Bytes[i] = (byte)((sha384[i * 2] - 0x37) * 0x10); + else if(sha384[i * 2] >= 0x61 && sha384[i * 2] <= 0x66) + sha384Bytes[i] = (byte)((sha384[i * 2] - 0x57) * 0x10); + + if(sha384[i * 2 + 1] >= 0x30 && sha384[i * 2 + 1] <= 0x39) + sha384Bytes[i] += (byte)(sha384[i * 2 + 1] - 0x30); + else if(sha384[i * 2 + 1] >= 0x41 && sha384[i * 2 + 1] <= 0x46) + sha384Bytes[i] += (byte)(sha384[i * 2 + 1] - 0x37); + else if(sha384[i * 2 + 1] >= 0x61 && sha384[i * 2 + 1] <= 0x66) + sha384Bytes[i] += (byte)(sha384[i * 2 + 1] - 0x57); + } + + string sha384B32 = Base32.ToBase32String(sha384Bytes); + + string repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, + "files", + sha384B32[0].ToString(), + sha384B32[1].ToString(), + sha384B32[2].ToString(), + sha384B32[3].ToString(), + sha384B32[4].ToString()); + + if(!Directory.Exists(repoPath)) Directory.CreateDirectory(repoPath); + + repoPath = Settings.Settings.Current.Compression switch + { + CompressionType.Zstd => Path.Combine(repoPath, sha384B32 + ".zst"), + CompressionType.None => Path.Combine(repoPath, sha384B32), + _ => Path.Combine(repoPath, sha384B32 + ".lz") + }; + + if(dbFile.Crc32 == null) + { + dbFile.Crc32 = checksums[ChecksumType.Crc32]; + dbFile.UpdatedOn = DateTime.UtcNow; + } + + if(dbFile.Md5 == null) + { + dbFile.Md5 = checksums[ChecksumType.Md5]; + dbFile.UpdatedOn = DateTime.UtcNow; + } + + if(dbFile.Sha1 == null) + { + dbFile.Sha1 = checksums[ChecksumType.Sha1]; + dbFile.UpdatedOn = DateTime.UtcNow; + } + + if(dbFile.Sha256 == null) + { + dbFile.Sha256 = checksums[ChecksumType.Sha256]; + dbFile.UpdatedOn = DateTime.UtcNow; + } + + if(dbFile.Sha384 == null) + { + dbFile.Sha384 = checksums[ChecksumType.Sha384]; + dbFile.UpdatedOn = DateTime.UtcNow; + } + + if(dbFile.Sha512 == null) + { + dbFile.Sha512 = checksums[ChecksumType.Sha512]; + dbFile.UpdatedOn = DateTime.UtcNow; + } + + if(File.Exists(repoPath)) + { + dbFile.IsInRepo = true; + dbFile.UpdatedOn = DateTime.UtcNow; + + if(!fileInDb) _newFiles.Add(dbFile); + + zStream.Close(); + outFs.Dispose(); + + File.Delete(tempPath); + + ImportedRom?.Invoke(this, + new ImportedRomItemEventArgs + { + Item = new ImportRomItem + { + Filename = filename, + Status = Localization.OK + } + }); + + return; + } + + SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty); + + SetMessage2?.Invoke(this, + new MessageEventArgs + { + Message = Localization.Finishing + }); + + zStream.Close(); + outFs.Dispose(); + + File.Move(tempPath, repoPath, true); + + dbFile.IsInRepo = true; + dbFile.UpdatedOn = DateTime.UtcNow; + + if(!fileInDb) _newFiles.Add(dbFile); + + ImportedRom?.Invoke(this, + new ImportedRomItemEventArgs + { + Item = new ImportRomItem + { + Filename = filename, + Status = Localization.OK + } + }); + } + catch(Exception) + { + ImportedRom?.Invoke(this, + new ImportedRomItemEventArgs + { + Item = new ImportRomItem + { + Filename = filename, + Status = Localization.UnhandledExceptionWhenImporting + } + }); + +#pragma warning disable ERP022 + } +#pragma warning restore ERP022 + } + bool ImportRom(string path) { try @@ -819,7 +1108,7 @@ public sealed class FileImporter return true; } - catch(Exception ex) + catch(Exception) { _lastMessage = Localization.UnhandledExceptionWhenImporting; diff --git a/RomRepoMgr/Resources/Localization.Designer.cs b/RomRepoMgr/Resources/Localization.Designer.cs index e430422..2710d95 100644 --- a/RomRepoMgr/Resources/Localization.Designer.cs +++ b/RomRepoMgr/Resources/Localization.Designer.cs @@ -782,5 +782,11 @@ namespace RomRepoMgr.Resources { return ResourceManager.GetString("UseInternalDecompressorLabel", resourceCulture); } } + + public static string ProcessingArchive { + get { + return ResourceManager.GetString("ProcessingArchive", resourceCulture); + } + } } } diff --git a/RomRepoMgr/Resources/Localization.es.resx b/RomRepoMgr/Resources/Localization.es.resx index 322f73d..db43096 100644 --- a/RomRepoMgr/Resources/Localization.es.resx +++ b/RomRepoMgr/Resources/Localization.es.resx @@ -387,4 +387,7 @@ Tardará mucho tiempo... Usar decompresor interno (soporta menos formatos) + + Procesando archivo: {0} + \ No newline at end of file diff --git a/RomRepoMgr/Resources/Localization.resx b/RomRepoMgr/Resources/Localization.resx index d0ce6cd..54ca542 100644 --- a/RomRepoMgr/Resources/Localization.resx +++ b/RomRepoMgr/Resources/Localization.resx @@ -395,4 +395,7 @@ This will take a long time... Use internal decompressor (supports less formats) + + Processing archive: {0} + \ No newline at end of file diff --git a/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs b/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs index fe15d6e..9277971 100644 --- a/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs +++ b/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs @@ -20,6 +20,7 @@ using RomRepoMgr.Models; using RomRepoMgr.Resources; using Serilog; using Serilog.Extensions.Logging; +using SharpCompress.Readers; namespace RomRepoMgr.ViewModels; @@ -87,7 +88,7 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase CanClose = true; RemoveFilesChecked = false; KnownOnlyChecked = true; - RecurseArchivesChecked = Settings.Settings.UnArUsable; + RecurseArchivesChecked = Settings.Settings.CanDecompress; RemoveFilesEnabled = false; CanChoose = true; } @@ -97,7 +98,7 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase public ICommand StartCommand { get; } public Window View { get; init; } - public bool RecurseArchivesEnabled => Settings.Settings.UnArUsable; + public bool RecurseArchivesEnabled => Settings.Settings.CanDecompress; public bool RecurseArchivesChecked { @@ -346,6 +347,97 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase ProcessFiles(); } + void ProcessArchivesManaged() + { + // For each archive + ProgressMaximum = _rootImporter.Archives.Count; + ProgressMinimum = 0; + ProgressValue = 0; + ProgressIsIndeterminate = false; + Progress2Visible = false; + StatusMessage2Visible = false; + _listPosition = 0; + _stopwatch.Restart(); + + Parallel.ForEach(_rootImporter.Archives, + archive => + { + Dispatcher.UIThread.Post(() => + { + StatusMessage = + string.Format(Localization.ProcessingArchive, Path.GetFileName(archive)); + + ProgressValue = _listPosition; + }); + + // Create FileImporter + var archiveImporter = new FileImporter(_ctx, + _newFiles, + _newDisks, + _newMedias, + KnownOnlyChecked, + RemoveFilesChecked); + + // Open archive + try + { + using var fs = new FileStream(archive, FileMode.Open, FileAccess.Read); + using IReader reader = ReaderFactory.Open(fs); + + // Process files in archive + while(reader.MoveToNextEntry()) + { + if(reader.Entry.IsDirectory) continue; + + if(reader.Entry.Crc == 0 && KnownOnlyChecked) continue; + + if(!archiveImporter.IsCrcInDb(reader.Entry.Crc) && KnownOnlyChecked) continue; + + var model = new RomImporter + { + Filename = Path.GetFileName(reader.Entry.Key), + Indeterminate = true + }; + + var worker = new FileImporter(_ctx, + _newFiles, + _newDisks, + _newMedias, + KnownOnlyChecked, + RemoveFilesChecked); + + worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress; + worker.SetMessage2 += model.OnSetMessage; + worker.SetProgress2 += model.OnSetProgress; + worker.SetProgressBounds2 += model.OnSetProgressBounds; + worker.ImportedRom += model.OnImportedRom; + worker.WorkFinished += model.OnWorkFinished; + + Dispatcher.UIThread.Post(() => Importers.Add(model)); + + worker.ImportAndHashRom(reader.OpenEntryStream(), + reader.Entry.Key, + Path.Combine(Settings.Settings.Current.RepositoryPath, + Path.GetFileName(Path.GetTempFileName())), + reader.Entry.Size); + } + } + catch(InvalidOperationException) {} + finally + { + Interlocked.Increment(ref _listPosition); + } + }); + + _stopwatch.Stop(); + Log.Debug("Took {TotalSeconds} seconds to process archives", _stopwatch.Elapsed.TotalSeconds); + + Progress2Visible = false; + StatusMessage2Visible = false; + + ProcessFiles(); + } + void CheckArchivesFinished(object sender, EventArgs e) { _stopwatch.Stop(); @@ -356,7 +448,10 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase _rootImporter.Finished -= CheckArchivesFinished; - ProcessArchives(); + if(Settings.Settings.Current.UseInternalDecompressor) + ProcessArchivesManaged(); + else + ProcessArchives(); } void SetMessage(object sender, MessageEventArgs e)