diff --git a/RomRepoMgr.Core/Resources/Localization.Designer.cs b/RomRepoMgr.Core/Resources/Localization.Designer.cs
index 00ecd66..3df0588 100644
--- a/RomRepoMgr.Core/Resources/Localization.Designer.cs
+++ b/RomRepoMgr.Core/Resources/Localization.Designer.cs
@@ -392,5 +392,11 @@ namespace RomRepoMgr.Core.Resources {
return ResourceManager.GetString("DatImportSuccess", resourceCulture);
}
}
+
+ internal static string ImportingFile {
+ get {
+ return ResourceManager.GetString("ImportingFile", resourceCulture);
+ }
+ }
}
}
diff --git a/RomRepoMgr.Core/Resources/Localization.es.resx b/RomRepoMgr.Core/Resources/Localization.es.resx
index ebdacd0..0e99fb8 100644
--- a/RomRepoMgr.Core/Resources/Localization.es.resx
+++ b/RomRepoMgr.Core/Resources/Localization.es.resx
@@ -186,4 +186,7 @@
Añadiendo medios...
+
+ Importando fichero...
+
\ No newline at end of file
diff --git a/RomRepoMgr.Core/Resources/Localization.resx b/RomRepoMgr.Core/Resources/Localization.resx
index f86ef79..4634a7f 100644
--- a/RomRepoMgr.Core/Resources/Localization.resx
+++ b/RomRepoMgr.Core/Resources/Localization.resx
@@ -196,4 +196,7 @@
Imported {0} machines with {1} ROMs.
+
+ Importing file...
+
\ No newline at end of file
diff --git a/RomRepoMgr.Core/Workers/FileImporter.cs b/RomRepoMgr.Core/Workers/FileImporter.cs
index d1df0d3..2084f33 100644
--- a/RomRepoMgr.Core/Workers/FileImporter.cs
+++ b/RomRepoMgr.Core/Workers/FileImporter.cs
@@ -540,6 +540,295 @@ public sealed class FileImporter
}
}
+ public bool IsCrcInDb(long crc32)
+ {
+ lock(DbLock)
+ {
+ return _ctx.Files.Any(f => f.Crc32 == crc32.ToString("x8"));
+ }
+ }
+
+ public void ImportAndHashRom(Stream stream, string filename, string tempPath, long size)
+ {
+ try
+ {
+ var outFs = new FileStream(tempPath, FileMode.Create, FileAccess.Write);
+
+ SetMessage2?.Invoke(this,
+ new MessageEventArgs
+ {
+ Message = Localization.ImportingFile
+ });
+
+ byte[] buffer;
+ var checksumWorker = new Checksum();
+ Stream zStream;
+
+ switch(Settings.Settings.Current.Compression)
+ {
+ case CompressionType.Zstd:
+ {
+ var zstdStream = new CompressionStream(outFs, 15);
+ zstdStream.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, Environment.ProcessorCount);
+ zStream = zstdStream;
+
+ break;
+ }
+ case CompressionType.None:
+ zStream = outFs;
+
+ break;
+ default:
+ zStream = new LZipStream(outFs, CompressionMode.Compress);
+
+ break;
+ }
+
+
+ if(size > BUFFER_SIZE)
+ {
+ SetProgressBounds2?.Invoke(this,
+ new ProgressBoundsEventArgs
+ {
+ Minimum = 0,
+ Maximum = size
+ });
+
+ long offset;
+ long remainder = size % BUFFER_SIZE;
+
+ for(offset = 0; offset < size - remainder; offset += (int)BUFFER_SIZE)
+ {
+ SetProgress2?.Invoke(this,
+ new ProgressEventArgs
+ {
+ Value = offset
+ });
+
+ buffer = new byte[BUFFER_SIZE];
+ stream.EnsureRead(buffer, 0, (int)BUFFER_SIZE);
+ checksumWorker.Update(buffer);
+ zStream.Write(buffer, 0, buffer.Length);
+ }
+
+ SetProgress2?.Invoke(this,
+ new ProgressEventArgs
+ {
+ Value = offset
+ });
+
+ buffer = new byte[remainder];
+ stream.EnsureRead(buffer, 0, (int)remainder);
+ }
+ else
+ {
+ SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty);
+ buffer = new byte[size];
+ stream.EnsureRead(buffer, 0, (int)size);
+ }
+
+ checksumWorker.Update(buffer);
+ zStream.Write(buffer, 0, buffer.Length);
+
+ Dictionary checksums = checksumWorker.End();
+
+ ulong uSize = (ulong)size;
+ bool fileInDb = true;
+
+ bool knownFile = _pendingFiles.TryGetValue(checksums[ChecksumType.Sha512], out DbFile dbFile);
+
+ lock(DbLock)
+ {
+ dbFile ??= _ctx.Files.FirstOrDefault(f => (f.Sha512 == checksums[ChecksumType.Sha512] ||
+ f.Sha384 == checksums[ChecksumType.Sha384] ||
+ f.Sha256 == checksums[ChecksumType.Sha256] ||
+ f.Sha1 == checksums[ChecksumType.Sha1] ||
+ f.Md5 == checksums[ChecksumType.Md5] ||
+ f.Crc32 == checksums[ChecksumType.Crc32]) &&
+ f.Size == uSize);
+ }
+
+ if(dbFile == null)
+ {
+ if(onlyKnown)
+ {
+ ImportedRom?.Invoke(this,
+ new ImportedRomItemEventArgs
+ {
+ Item = new ImportRomItem
+ {
+ Filename = filename,
+ Status = Localization.UnknownFile
+ }
+ });
+
+ return;
+ }
+
+ dbFile = new DbFile
+ {
+ Crc32 = checksums[ChecksumType.Crc32],
+ Md5 = checksums[ChecksumType.Md5],
+ Sha1 = checksums[ChecksumType.Sha1],
+ Sha256 = checksums[ChecksumType.Sha256],
+ Sha384 = checksums[ChecksumType.Sha384],
+ Sha512 = checksums[ChecksumType.Sha512],
+ Size = uSize,
+ CreatedOn = DateTime.UtcNow,
+ UpdatedOn = DateTime.UtcNow,
+ OriginalFileName = Path.GetFileName(filename)
+ };
+
+ fileInDb = false;
+ }
+
+ if(!knownFile) _pendingFiles[checksums[ChecksumType.Sha512]] = dbFile;
+
+ byte[] sha384Bytes = new byte[48];
+ string sha384 = checksums[ChecksumType.Sha384];
+
+ for(int i = 0; i < 48; i++)
+ {
+ if(sha384[i * 2] >= 0x30 && sha384[i * 2] <= 0x39)
+ sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10);
+ else if(sha384[i * 2] >= 0x41 && sha384[i * 2] <= 0x46)
+ sha384Bytes[i] = (byte)((sha384[i * 2] - 0x37) * 0x10);
+ else if(sha384[i * 2] >= 0x61 && sha384[i * 2] <= 0x66)
+ sha384Bytes[i] = (byte)((sha384[i * 2] - 0x57) * 0x10);
+
+ if(sha384[i * 2 + 1] >= 0x30 && sha384[i * 2 + 1] <= 0x39)
+ sha384Bytes[i] += (byte)(sha384[i * 2 + 1] - 0x30);
+ else if(sha384[i * 2 + 1] >= 0x41 && sha384[i * 2 + 1] <= 0x46)
+ sha384Bytes[i] += (byte)(sha384[i * 2 + 1] - 0x37);
+ else if(sha384[i * 2 + 1] >= 0x61 && sha384[i * 2 + 1] <= 0x66)
+ sha384Bytes[i] += (byte)(sha384[i * 2 + 1] - 0x57);
+ }
+
+ string sha384B32 = Base32.ToBase32String(sha384Bytes);
+
+ string repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath,
+ "files",
+ sha384B32[0].ToString(),
+ sha384B32[1].ToString(),
+ sha384B32[2].ToString(),
+ sha384B32[3].ToString(),
+ sha384B32[4].ToString());
+
+ if(!Directory.Exists(repoPath)) Directory.CreateDirectory(repoPath);
+
+ repoPath = Settings.Settings.Current.Compression switch
+ {
+ CompressionType.Zstd => Path.Combine(repoPath, sha384B32 + ".zst"),
+ CompressionType.None => Path.Combine(repoPath, sha384B32),
+ _ => Path.Combine(repoPath, sha384B32 + ".lz")
+ };
+
+ if(dbFile.Crc32 == null)
+ {
+ dbFile.Crc32 = checksums[ChecksumType.Crc32];
+ dbFile.UpdatedOn = DateTime.UtcNow;
+ }
+
+ if(dbFile.Md5 == null)
+ {
+ dbFile.Md5 = checksums[ChecksumType.Md5];
+ dbFile.UpdatedOn = DateTime.UtcNow;
+ }
+
+ if(dbFile.Sha1 == null)
+ {
+ dbFile.Sha1 = checksums[ChecksumType.Sha1];
+ dbFile.UpdatedOn = DateTime.UtcNow;
+ }
+
+ if(dbFile.Sha256 == null)
+ {
+ dbFile.Sha256 = checksums[ChecksumType.Sha256];
+ dbFile.UpdatedOn = DateTime.UtcNow;
+ }
+
+ if(dbFile.Sha384 == null)
+ {
+ dbFile.Sha384 = checksums[ChecksumType.Sha384];
+ dbFile.UpdatedOn = DateTime.UtcNow;
+ }
+
+ if(dbFile.Sha512 == null)
+ {
+ dbFile.Sha512 = checksums[ChecksumType.Sha512];
+ dbFile.UpdatedOn = DateTime.UtcNow;
+ }
+
+ if(File.Exists(repoPath))
+ {
+ dbFile.IsInRepo = true;
+ dbFile.UpdatedOn = DateTime.UtcNow;
+
+ if(!fileInDb) _newFiles.Add(dbFile);
+
+ zStream.Close();
+ outFs.Dispose();
+
+ File.Delete(tempPath);
+
+ ImportedRom?.Invoke(this,
+ new ImportedRomItemEventArgs
+ {
+ Item = new ImportRomItem
+ {
+ Filename = filename,
+ Status = Localization.OK
+ }
+ });
+
+ return;
+ }
+
+ SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty);
+
+ SetMessage2?.Invoke(this,
+ new MessageEventArgs
+ {
+ Message = Localization.Finishing
+ });
+
+ zStream.Close();
+ outFs.Dispose();
+
+ File.Move(tempPath, repoPath, true);
+
+ dbFile.IsInRepo = true;
+ dbFile.UpdatedOn = DateTime.UtcNow;
+
+ if(!fileInDb) _newFiles.Add(dbFile);
+
+ ImportedRom?.Invoke(this,
+ new ImportedRomItemEventArgs
+ {
+ Item = new ImportRomItem
+ {
+ Filename = filename,
+ Status = Localization.OK
+ }
+ });
+ }
+ catch(Exception)
+ {
+ ImportedRom?.Invoke(this,
+ new ImportedRomItemEventArgs
+ {
+ Item = new ImportRomItem
+ {
+ Filename = filename,
+ Status = Localization.UnhandledExceptionWhenImporting
+ }
+ });
+
+#pragma warning disable ERP022
+ }
+#pragma warning restore ERP022
+ }
+
bool ImportRom(string path)
{
try
@@ -819,7 +1108,7 @@ public sealed class FileImporter
return true;
}
- catch(Exception ex)
+ catch(Exception)
{
_lastMessage = Localization.UnhandledExceptionWhenImporting;
diff --git a/RomRepoMgr/Resources/Localization.Designer.cs b/RomRepoMgr/Resources/Localization.Designer.cs
index e430422..2710d95 100644
--- a/RomRepoMgr/Resources/Localization.Designer.cs
+++ b/RomRepoMgr/Resources/Localization.Designer.cs
@@ -782,5 +782,11 @@ namespace RomRepoMgr.Resources {
return ResourceManager.GetString("UseInternalDecompressorLabel", resourceCulture);
}
}
+
+ public static string ProcessingArchive {
+ get {
+ return ResourceManager.GetString("ProcessingArchive", resourceCulture);
+ }
+ }
}
}
diff --git a/RomRepoMgr/Resources/Localization.es.resx b/RomRepoMgr/Resources/Localization.es.resx
index 322f73d..db43096 100644
--- a/RomRepoMgr/Resources/Localization.es.resx
+++ b/RomRepoMgr/Resources/Localization.es.resx
@@ -387,4 +387,7 @@ Tardará mucho tiempo...
Usar decompresor interno (soporta menos formatos)
+
+ Procesando archivo: {0}
+
\ No newline at end of file
diff --git a/RomRepoMgr/Resources/Localization.resx b/RomRepoMgr/Resources/Localization.resx
index d0ce6cd..54ca542 100644
--- a/RomRepoMgr/Resources/Localization.resx
+++ b/RomRepoMgr/Resources/Localization.resx
@@ -395,4 +395,7 @@ This will take a long time...
Use internal decompressor (supports less formats)
+
+ Processing archive: {0}
+
\ No newline at end of file
diff --git a/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs b/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs
index fe15d6e..9277971 100644
--- a/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs
+++ b/RomRepoMgr/ViewModels/ImportRomFolderViewModel.cs
@@ -20,6 +20,7 @@ using RomRepoMgr.Models;
using RomRepoMgr.Resources;
using Serilog;
using Serilog.Extensions.Logging;
+using SharpCompress.Readers;
namespace RomRepoMgr.ViewModels;
@@ -87,7 +88,7 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
CanClose = true;
RemoveFilesChecked = false;
KnownOnlyChecked = true;
- RecurseArchivesChecked = Settings.Settings.UnArUsable;
+ RecurseArchivesChecked = Settings.Settings.CanDecompress;
RemoveFilesEnabled = false;
CanChoose = true;
}
@@ -97,7 +98,7 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
public ICommand StartCommand { get; }
public Window View { get; init; }
- public bool RecurseArchivesEnabled => Settings.Settings.UnArUsable;
+ public bool RecurseArchivesEnabled => Settings.Settings.CanDecompress;
public bool RecurseArchivesChecked
{
@@ -346,6 +347,97 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
ProcessFiles();
}
+ void ProcessArchivesManaged()
+ {
+ // For each archive
+ ProgressMaximum = _rootImporter.Archives.Count;
+ ProgressMinimum = 0;
+ ProgressValue = 0;
+ ProgressIsIndeterminate = false;
+ Progress2Visible = false;
+ StatusMessage2Visible = false;
+ _listPosition = 0;
+ _stopwatch.Restart();
+
+ Parallel.ForEach(_rootImporter.Archives,
+ archive =>
+ {
+ Dispatcher.UIThread.Post(() =>
+ {
+ StatusMessage =
+ string.Format(Localization.ProcessingArchive, Path.GetFileName(archive));
+
+ ProgressValue = _listPosition;
+ });
+
+ // Create FileImporter
+ var archiveImporter = new FileImporter(_ctx,
+ _newFiles,
+ _newDisks,
+ _newMedias,
+ KnownOnlyChecked,
+ RemoveFilesChecked);
+
+ // Open archive
+ try
+ {
+ using var fs = new FileStream(archive, FileMode.Open, FileAccess.Read);
+ using IReader reader = ReaderFactory.Open(fs);
+
+ // Process files in archive
+ while(reader.MoveToNextEntry())
+ {
+ if(reader.Entry.IsDirectory) continue;
+
+ if(reader.Entry.Crc == 0 && KnownOnlyChecked) continue;
+
+ if(!archiveImporter.IsCrcInDb(reader.Entry.Crc) && KnownOnlyChecked) continue;
+
+ var model = new RomImporter
+ {
+ Filename = Path.GetFileName(reader.Entry.Key),
+ Indeterminate = true
+ };
+
+ var worker = new FileImporter(_ctx,
+ _newFiles,
+ _newDisks,
+ _newMedias,
+ KnownOnlyChecked,
+ RemoveFilesChecked);
+
+ worker.SetIndeterminateProgress2 += model.OnSetIndeterminateProgress;
+ worker.SetMessage2 += model.OnSetMessage;
+ worker.SetProgress2 += model.OnSetProgress;
+ worker.SetProgressBounds2 += model.OnSetProgressBounds;
+ worker.ImportedRom += model.OnImportedRom;
+ worker.WorkFinished += model.OnWorkFinished;
+
+ Dispatcher.UIThread.Post(() => Importers.Add(model));
+
+ worker.ImportAndHashRom(reader.OpenEntryStream(),
+ reader.Entry.Key,
+ Path.Combine(Settings.Settings.Current.RepositoryPath,
+ Path.GetFileName(Path.GetTempFileName())),
+ reader.Entry.Size);
+ }
+ }
+ catch(InvalidOperationException) {}
+ finally
+ {
+ Interlocked.Increment(ref _listPosition);
+ }
+ });
+
+ _stopwatch.Stop();
+ Log.Debug("Took {TotalSeconds} seconds to process archives", _stopwatch.Elapsed.TotalSeconds);
+
+ Progress2Visible = false;
+ StatusMessage2Visible = false;
+
+ ProcessFiles();
+ }
+
void CheckArchivesFinished(object sender, EventArgs e)
{
_stopwatch.Stop();
@@ -356,7 +448,10 @@ public sealed partial class ImportRomFolderViewModel : ViewModelBase
_rootImporter.Finished -= CheckArchivesFinished;
- ProcessArchives();
+ if(Settings.Settings.Current.UseInternalDecompressor)
+ ProcessArchivesManaged();
+ else
+ ProcessArchives();
}
void SetMessage(object sender, MessageEventArgs e)