diff --git a/Directory.Packages.props b/Directory.Packages.props index 78f37df..903f739 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -45,6 +45,6 @@ - + \ No newline at end of file diff --git a/RomRepoMgr.Core/Filesystem/Vfs.cs b/RomRepoMgr.Core/Filesystem/Vfs.cs index 31840fa..bb39d0e 100644 --- a/RomRepoMgr.Core/Filesystem/Vfs.cs +++ b/RomRepoMgr.Core/Filesystem/Vfs.cs @@ -12,6 +12,7 @@ using RomRepoMgr.Database; using RomRepoMgr.Database.Models; using SharpCompress.Compressors; using SharpCompress.Compressors.LZMA; +using ZstdSharp; namespace RomRepoMgr.Core.Filesystem; @@ -365,9 +366,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable internal long Open(string sha384, long fileSize) { - var sha384Bytes = new byte[48]; + byte[] sha384Bytes = new byte[48]; - for(var i = 0; i < 48; i++) + for(int i = 0; i < 48; i++) { if(sha384[i * 2] >= 0x30 && sha384[i * 2] <= 0x39) sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10); @@ -386,19 +387,47 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable string sha384B32 = Base32.ToBase32String(sha384Bytes); - string repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, - "files", - sha384B32[0].ToString(), - sha384B32[1].ToString(), - sha384B32[2].ToString(), - sha384B32[3].ToString(), - sha384B32[4].ToString(), - sha384B32 + ".lz"); - if(!File.Exists(repoPath)) return -1; + string repoPath; + + repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, + "files", + sha384B32[0].ToString(), + sha384B32[1].ToString(), + sha384B32[2].ToString(), + sha384B32[3].ToString(), + sha384B32[4].ToString(), + sha384B32 + ".lz"); + + long handle; + + if(!File.Exists(repoPath)) + { + repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, + "files", + sha384B32[0].ToString(), + sha384B32[1].ToString(), + sha384B32[2].ToString(), + sha384B32[3].ToString(), + sha384B32[4].ToString(), + sha384B32 + ".zst"); + + if(!File.Exists(repoPath)) return -1; + + _lastHandle++; + handle = _lastHandle; + + _streamsCache[handle] = + Stream.Synchronized(new ForcedSeekStream(fileSize, + new FileStream(repoPath, + FileMode.Open, + FileAccess.Read))); + + return handle; + } _lastHandle++; - long handle = _lastHandle; + handle = _lastHandle; _streamsCache[handle] = Stream.Synchronized(new ForcedSeekStream(fileSize, @@ -457,9 +486,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable if(sha1 != null) { - var sha1Bytes = new byte[20]; + byte[] sha1Bytes = new byte[20]; - for(var i = 0; i < 20; i++) + for(int i = 0; i < 20; i++) { if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39) sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10); @@ -490,9 +519,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable if(md5 != null) { - var md5Bytes = new byte[16]; + byte[] md5Bytes = new byte[16]; - for(var i = 0; i < 16; i++) + for(int i = 0; i < 16; i++) { if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39) md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10); @@ -545,9 +574,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable if(sha256 != null) { - var sha256Bytes = new byte[32]; + byte[] sha256Bytes = new byte[32]; - for(var i = 0; i < 32; i++) + for(int i = 0; i < 32; i++) { if(sha256[i * 2] >= 0x30 && sha256[i * 2] <= 0x39) sha256Bytes[i] = (byte)((sha256[i * 2] - 0x30) * 0x10); @@ -579,9 +608,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable if(sha1 != null) { - var sha1Bytes = new byte[20]; + byte[] sha1Bytes = new byte[20]; - for(var i = 0; i < 20; i++) + for(int i = 0; i < 20; i++) { if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39) sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10); @@ -612,9 +641,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable if(md5 != null) { - var md5Bytes = new byte[16]; + byte[] md5Bytes = new byte[16]; - for(var i = 0; i < 16; i++) + for(int i = 0; i < 16; i++) { if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39) md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10); diff --git a/RomRepoMgr.Core/RomRepoMgr.Core.csproj b/RomRepoMgr.Core/RomRepoMgr.Core.csproj index 74f800b..ccc7d46 100644 --- a/RomRepoMgr.Core/RomRepoMgr.Core.csproj +++ b/RomRepoMgr.Core/RomRepoMgr.Core.csproj @@ -1,7 +1,7 @@ - + @@ -19,6 +19,7 @@ + diff --git a/RomRepoMgr.Core/Workers/Compression.cs b/RomRepoMgr.Core/Workers/Compression.cs index 0f2f232..6233082 100644 --- a/RomRepoMgr.Core/Workers/Compression.cs +++ b/RomRepoMgr.Core/Workers/Compression.cs @@ -28,8 +28,11 @@ using System.Diagnostics; using System.IO; using RomRepoMgr.Core.EventArgs; using RomRepoMgr.Core.Resources; +using RomRepoMgr.Settings; using SharpCompress.Compressors; using SharpCompress.Compressors.LZMA; +using ZstdSharp; +using ZstdSharp.Unsafe; using ErrorEventArgs = RomRepoMgr.Core.EventArgs.ErrorEventArgs; namespace RomRepoMgr.Core.Workers; @@ -45,11 +48,21 @@ public sealed class Compression public void CompressFile(string source, string destination) { - var inFs = new FileStream(source, FileMode.Open, FileAccess.Read); - var outFs = new FileStream(destination, FileMode.CreateNew, FileAccess.Write); - Stream zStream = new LZipStream(outFs, CompressionMode.Compress); + var inFs = new FileStream(source, FileMode.Open, FileAccess.Read); + var outFs = new FileStream(destination, FileMode.CreateNew, FileAccess.Write); - var buffer = new byte[BUFFER_SIZE]; + Stream zStream; + + if(Settings.Settings.Current.Compression == CompressionType.Zstd) + { + var zstdStream = new CompressionStream(outFs, 15); + zstdStream.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, Environment.ProcessorCount); + zStream = zstdStream; + } + else + zStream = new LZipStream(outFs, CompressionMode.Compress); + + byte[] buffer = new byte[BUFFER_SIZE]; SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs @@ -89,9 +102,17 @@ public sealed class Compression public void DecompressFile(string source, string destination) { - var inFs = new FileStream(source, FileMode.Open, FileAccess.Read); - var outFs = new FileStream(destination, FileMode.Create, FileAccess.Write); - Stream zStream = new LZipStream(inFs, CompressionMode.Decompress); + var inFs = new FileStream(source, FileMode.Open, FileAccess.Read); + var outFs = new FileStream(destination, FileMode.Create, FileAccess.Write); + Stream zStream; + + if(Path.GetExtension(source) == ".zst") + zStream = new DecompressionStream(inFs); + else if(Path.GetExtension(source) == ".lz") + zStream = new LZipStream(inFs, CompressionMode.Decompress); + else + throw new ArgumentException($"Invalid compression extension {Path.GetExtension(source)}"); + zStream.CopyTo(outFs); diff --git a/RomRepoMgr.Core/Workers/FileExporter.cs b/RomRepoMgr.Core/Workers/FileExporter.cs index de3ba3d..f26ce2e 100644 --- a/RomRepoMgr.Core/Workers/FileExporter.cs +++ b/RomRepoMgr.Core/Workers/FileExporter.cs @@ -12,6 +12,7 @@ using RomRepoMgr.Core.Resources; using RomRepoMgr.Database; using RomRepoMgr.Database.Models; using SharpCompress.Compressors.LZMA; +using ZstdSharp; using CompressionMode = SharpCompress.Compressors.CompressionMode; namespace RomRepoMgr.Core.Workers; @@ -155,10 +156,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa if(media.Sha256 != null) { - var sha256Bytes = new byte[32]; + byte[] sha256Bytes = new byte[32]; string sha256 = media.Sha256; - for(var i = 0; i < 32; i++) + for(int i = 0; i < 32; i++) { if(sha256[i * 2] >= 0x30 && sha256[i * 2] <= 0x39) sha256Bytes[i] = (byte)((sha256[i * 2] - 0x30) * 0x10); @@ -190,10 +191,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa if(media.Sha1 != null) { - var sha1Bytes = new byte[20]; + byte[] sha1Bytes = new byte[20]; string sha1 = media.Sha1; - for(var i = 0; i < 20; i++) + for(int i = 0; i < 20; i++) { if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39) sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10); @@ -225,10 +226,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa if(media.Md5 != null) { - var md5Bytes = new byte[16]; + byte[] md5Bytes = new byte[16]; string md5 = media.Md5; - for(var i = 0; i < 16; i++) + for(int i = 0; i < 16; i++) { if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39) md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10); @@ -286,7 +287,7 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa Maximum = inFs.Length }); - var buffer = new byte[BUFFER_SIZE]; + byte[] buffer = new byte[BUFFER_SIZE]; while(inFs.Position + BUFFER_SIZE <= inFs.Length) { @@ -359,10 +360,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa if(disk.Sha1 != null) { - var sha1Bytes = new byte[20]; + byte[] sha1Bytes = new byte[20]; string sha1 = disk.Sha1; - for(var i = 0; i < 20; i++) + for(int i = 0; i < 20; i++) { if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39) sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10); @@ -394,10 +395,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa if(disk.Md5 != null) { - var md5Bytes = new byte[16]; + byte[] md5Bytes = new byte[16]; string md5 = disk.Md5; - for(var i = 0; i < 16; i++) + for(int i = 0; i < 16; i++) { if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39) md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10); @@ -453,7 +454,7 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa Maximum = inFs.Length }); - var buffer = new byte[BUFFER_SIZE]; + byte[] buffer = new byte[BUFFER_SIZE]; while(inFs.Position + BUFFER_SIZE <= inFs.Length) { @@ -558,10 +559,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa // Special case for empty file, as it seems to crash when SharpCompress tries to unLZMA it. if(file.Size == 0) return new MemoryStream(); - var sha384Bytes = new byte[48]; + byte[] sha384Bytes = new byte[48]; string sha384 = file.Sha384; - for(var i = 0; i < 48; i++) + for(int i = 0; i < 48; i++) { if(sha384[i * 2] >= 0x30 && sha384[i * 2] <= 0x39) sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10); @@ -589,10 +590,29 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa sha384B32[4].ToString(), sha384B32 + ".lz"); - if(!File.Exists(repoPath)) - throw new ArgumentException(string.Format(Localization.CannotFindHashInRepository, file.Sha256)); + FileStream inFs; - var inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read); + // Try ZSTD + if(!File.Exists(repoPath)) + { + repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, + "files", + sha384B32[0].ToString(), + sha384B32[1].ToString(), + sha384B32[2].ToString(), + sha384B32[3].ToString(), + sha384B32[4].ToString(), + sha384B32 + ".zst"); + + if(!File.Exists(repoPath)) + throw new ArgumentException(string.Format(Localization.CannotFindHashInRepository, file.Sha256)); + + inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read); + + return new StreamWithLength(new DecompressionStream(inFs), (long)file.Size); + } + + inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read); return new StreamWithLength(new LZipStream(inFs, CompressionMode.Decompress), (long)file.Size); } @@ -612,11 +632,9 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa Value = _filePosition }); - if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName, out FileByMachine fileByMachine)) - { - if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName.Replace('/', '\\'), out fileByMachine)) - throw new ArgumentException(Localization.CannotFindZipEntryInDictionary); - } + if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName, out FileByMachine fileByMachine) && + !_filesByMachine.TryGetValue(e.CurrentEntry.FileName.Replace('/', '\\'), out fileByMachine)) + throw new ArgumentException(Localization.CannotFindZipEntryInDictionary); DbFile currentFile = fileByMachine.File; diff --git a/RomRepoMgr.Core/Workers/FileImporter.cs b/RomRepoMgr.Core/Workers/FileImporter.cs index c873a9c..f9052c0 100644 --- a/RomRepoMgr.Core/Workers/FileImporter.cs +++ b/RomRepoMgr.Core/Workers/FileImporter.cs @@ -15,9 +15,12 @@ using RomRepoMgr.Core.Models; using RomRepoMgr.Core.Resources; using RomRepoMgr.Database; using RomRepoMgr.Database.Models; +using RomRepoMgr.Settings; using SabreTools.FileTypes.Aaru; using SabreTools.FileTypes.CHD; using SharpCompress.Compressors.LZMA; +using ZstdSharp; +using ZstdSharp.Unsafe; using CompressionMode = SharpCompress.Compressors.CompressionMode; namespace RomRepoMgr.Core.Workers; @@ -551,7 +554,9 @@ public sealed class FileImporter if(!Directory.Exists(repoPath)) Directory.CreateDirectory(repoPath); - repoPath = Path.Combine(repoPath, sha384B32 + ".lz"); + repoPath = Settings.Settings.Current.Compression == CompressionType.Zstd + ? Path.Combine(repoPath, sha384B32 + ".zst") + : Path.Combine(repoPath, sha384B32 + ".lz"); if(dbFile.Crc32 == null) { @@ -605,8 +610,18 @@ public sealed class FileImporter inFs.Position = 0; - var outFs = new FileStream(repoPath, FileMode.CreateNew, FileAccess.Write); - Stream zStream = new LZipStream(outFs, CompressionMode.Compress); + var outFs = new FileStream(repoPath, FileMode.CreateNew, FileAccess.Write); + + Stream zStream; + + if(Settings.Settings.Current.Compression == CompressionType.Zstd) + { + var zstdStream = new CompressionStream(outFs, 15); + zstdStream.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, Environment.ProcessorCount); + zStream = zstdStream; + } + else + zStream = new LZipStream(outFs, CompressionMode.Compress); SetProgressBounds2?.Invoke(this, new ProgressBoundsEventArgs diff --git a/RomRepoMgr.Settings/Settings.cs b/RomRepoMgr.Settings/Settings.cs index b37460b..876ebc6 100644 --- a/RomRepoMgr.Settings/Settings.cs +++ b/RomRepoMgr.Settings/Settings.cs @@ -33,12 +33,19 @@ using PlatformID = Aaru.CommonTypes.Interop.PlatformID; namespace RomRepoMgr.Settings; +public enum CompressionType +{ + Lzip = 0, + Zstd +} + public sealed class SetSettings { - public string DatabasePath { get; set; } - public string RepositoryPath { get; set; } - public string TemporaryFolder { get; set; } - public string UnArchiverPath { get; set; } + public string DatabasePath { get; set; } + public string RepositoryPath { get; set; } + public string TemporaryFolder { get; set; } + public string UnArchiverPath { get; set; } + public CompressionType Compression { get; set; } } /// Manages statistics @@ -105,6 +112,11 @@ public static class Settings ? ((NSString)obj).ToString() : null; + Current.Compression = parsedPreferences.TryGetValue("Compression", out obj) + ? (CompressionType)Enum.Parse(typeof(CompressionType), + ((NSNumber)obj).ToString()) + : CompressionType.Lzip; + prefsFs.Close(); } else @@ -150,6 +162,11 @@ public static class Settings Current.RepositoryPath = key.GetValue("RepositoryPath") as string; Current.TemporaryFolder = key.GetValue("TemporaryFolder") as string; Current.UnArchiverPath = key.GetValue("UnArchiverPath") as string; + + if(key.GetValue("Compression") is int compression) + Current.Compression = (CompressionType)compression; + else + Current.Compression = CompressionType.Lzip; } break; @@ -222,6 +239,9 @@ public static class Settings }, { "UnArchiverPath", Current.UnArchiverPath + }, + { + "Compression", (NSNumber)(int)Current.Compression } }; @@ -256,6 +276,7 @@ public static class Settings key?.SetValue("RepositoryPath", Current.RepositoryPath); key?.SetValue("TemporaryFolder", Current.TemporaryFolder); key?.SetValue("UnArchiverPath", Current.UnArchiverPath); + key?.SetValue("Compression", (int)Current.Compression); } break; @@ -309,7 +330,8 @@ public static class Settings { DatabasePath = Path.Combine(dataPath, "romrepo.db"), RepositoryPath = Path.Combine(dataPath, "repo"), - TemporaryFolder = Path.GetTempPath() + TemporaryFolder = Path.GetTempPath(), + Compression = CompressionType.Lzip }; } } \ No newline at end of file diff --git a/RomRepoMgr/Resources/Localization.Designer.cs b/RomRepoMgr/Resources/Localization.Designer.cs index adb2134..d066071 100644 --- a/RomRepoMgr/Resources/Localization.Designer.cs +++ b/RomRepoMgr/Resources/Localization.Designer.cs @@ -770,5 +770,11 @@ namespace RomRepoMgr.Resources { return ResourceManager.GetString("ProgressLabel", resourceCulture); } } + + public static string CompressionType { + get { + return ResourceManager.GetString("CompressionType", resourceCulture); + } + } } } diff --git a/RomRepoMgr/Resources/Localization.es.resx b/RomRepoMgr/Resources/Localization.es.resx index 3afe466..428df29 100644 --- a/RomRepoMgr/Resources/Localization.es.resx +++ b/RomRepoMgr/Resources/Localization.es.resx @@ -381,4 +381,7 @@ Tardará mucho tiempo... Progreso + + Compresión + \ No newline at end of file diff --git a/RomRepoMgr/Resources/Localization.resx b/RomRepoMgr/Resources/Localization.resx index 89c9059..45c6aa7 100644 --- a/RomRepoMgr/Resources/Localization.resx +++ b/RomRepoMgr/Resources/Localization.resx @@ -389,4 +389,7 @@ This will take a long time... Progress + + Compression + \ No newline at end of file diff --git a/RomRepoMgr/ViewModels/SettingsViewModel.cs b/RomRepoMgr/ViewModels/SettingsViewModel.cs index 1908188..092de10 100644 --- a/RomRepoMgr/ViewModels/SettingsViewModel.cs +++ b/RomRepoMgr/ViewModels/SettingsViewModel.cs @@ -26,6 +26,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Threading.Tasks; using System.Windows.Input; using Avalonia.Platform.Storage; @@ -39,6 +40,7 @@ using RomRepoMgr.Core.EventArgs; using RomRepoMgr.Core.Workers; using RomRepoMgr.Database; using RomRepoMgr.Resources; +using RomRepoMgr.Settings; using RomRepoMgr.Views; using Serilog; using Serilog.Extensions.Logging; @@ -49,13 +51,16 @@ namespace RomRepoMgr.ViewModels; public sealed partial class SettingsViewModel : ViewModelBase { readonly SettingsDialog _view; - bool _databaseChanged; - string _databasePath; - bool _repositoryChanged; - string _repositoryPath; - bool _temporaryChanged; - string _temporaryPath; - bool _unArChanged; + + CompressionType _compression; + bool _compressionChanged; + bool _databaseChanged; + string _databasePath; + bool _repositoryChanged; + string _repositoryPath; + bool _temporaryChanged; + string _temporaryPath; + bool _unArChanged; [ObservableProperty] string _unArPath; [ObservableProperty] @@ -83,10 +88,14 @@ public sealed partial class SettingsViewModel : ViewModelBase RepositoryPath = Settings.Settings.Current.RepositoryPath; TemporaryPath = Settings.Settings.Current.TemporaryFolder; UnArPath = Settings.Settings.Current.UnArchiverPath; + Compression = Settings.Settings.Current.Compression; if(!string.IsNullOrWhiteSpace(UnArPath)) CheckUnAr(); } + public List CompressionTypes { get; } = + Enum.GetValues(typeof(CompressionType)).Cast().ToList(); + public ICommand UnArCommand { get; } public ICommand TemporaryCommand { get; } public ICommand RepositoryCommand { get; } @@ -126,6 +135,16 @@ public sealed partial class SettingsViewModel : ViewModelBase } } + public CompressionType Compression + { + get => _compression; + set + { + SetProperty(ref _compression, value); + _compressionChanged = true; + } + } + void CheckUnAr() { var worker = new Compression(); @@ -331,7 +350,9 @@ public sealed partial class SettingsViewModel : ViewModelBase Settings.Settings.UnArUsable = true; } - if(_databaseChanged || _repositoryChanged || _temporaryChanged || _unArChanged) + if(_compressionChanged) Settings.Settings.Current.Compression = Compression; + + if(_databaseChanged || _repositoryChanged || _temporaryChanged || _unArChanged || _compressionChanged) Settings.Settings.SaveSettings(); _view.Close(); diff --git a/RomRepoMgr/Views/SettingsDialog.axaml b/RomRepoMgr/Views/SettingsDialog.axaml index c1d8120..25d0c68 100644 --- a/RomRepoMgr/Views/SettingsDialog.axaml +++ b/RomRepoMgr/Views/SettingsDialog.axaml @@ -41,7 +41,7 @@ - + - + + + +