claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 19:24:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support to compress repository with zstd.

Browse Source
This commit is contained in:
Natalia Portillo
2025-07-25 17:49:36 +01:00
parent bf19439e49
commit 0bda03afee
12 changed files with 224 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
RomRepoMgr.Core/Filesystem/Vfs.cs
View File

@@ -12,6 +12,7 @@ using RomRepoMgr.Database;
using RomRepoMgr.Database.Models;
using SharpCompress.Compressors;
using SharpCompress.Compressors.LZMA;
using ZstdSharp;
namespace RomRepoMgr.Core.Filesystem;
@@ -365,9 +366,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable
internal long Open(string sha384, long fileSize)
{
var sha384Bytes = new byte[48];
byte[] sha384Bytes = new byte[48];
for(var i = 0; i < 48; i++)
for(int i = 0; i < 48; i++)
{
if(sha384[i * 2] >= 0x30 && sha384[i * 2] <= 0x39)
sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10);
@@ -386,19 +387,47 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable
string sha384B32 = Base32.ToBase32String(sha384Bytes);
string repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath,
"files",
sha384B32[0].ToString(),
sha384B32[1].ToString(),
sha384B32[2].ToString(),
sha384B32[3].ToString(),
sha384B32[4].ToString(),
sha384B32 + ".lz");
if(!File.Exists(repoPath)) return -1;
string repoPath;
repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath,
"files",
sha384B32[0].ToString(),
sha384B32[1].ToString(),
sha384B32[2].ToString(),
sha384B32[3].ToString(),
sha384B32[4].ToString(),
sha384B32 + ".lz");
long handle;
if(!File.Exists(repoPath))
{
repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath,
"files",
sha384B32[0].ToString(),
sha384B32[1].ToString(),
sha384B32[2].ToString(),
sha384B32[3].ToString(),
sha384B32[4].ToString(),
sha384B32 + ".zst");
if(!File.Exists(repoPath)) return -1;
_lastHandle++;
handle = _lastHandle;
_streamsCache[handle] =
Stream.Synchronized(new ForcedSeekStream<DecompressionStream>(fileSize,
new FileStream(repoPath,
FileMode.Open,
FileAccess.Read)));
return handle;
}
_lastHandle++;
long handle = _lastHandle;
handle = _lastHandle;
_streamsCache[handle] =
Stream.Synchronized(new ForcedSeekStream<LZipStream>(fileSize,
@@ -457,9 +486,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable
if(sha1 != null)
{
var sha1Bytes = new byte[20];
byte[] sha1Bytes = new byte[20];
for(var i = 0; i < 20; i++)
for(int i = 0; i < 20; i++)
{
if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39)
sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10);
@@ -490,9 +519,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable
if(md5 != null)
{
var md5Bytes = new byte[16];
byte[] md5Bytes = new byte[16];
for(var i = 0; i < 16; i++)
for(int i = 0; i < 16; i++)
{
if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39)
md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10);
@@ -545,9 +574,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable
if(sha256 != null)
{
var sha256Bytes = new byte[32];
byte[] sha256Bytes = new byte[32];
for(var i = 0; i < 32; i++)
for(int i = 0; i < 32; i++)
{
if(sha256[i * 2] >= 0x30 && sha256[i * 2] <= 0x39)
sha256Bytes[i] = (byte)((sha256[i * 2] - 0x30) * 0x10);
@@ -579,9 +608,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable
if(sha1 != null)
{
var sha1Bytes = new byte[20];
byte[] sha1Bytes = new byte[20];
for(var i = 0; i < 20; i++)
for(int i = 0; i < 20; i++)
{
if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39)
sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10);
@@ -612,9 +641,9 @@ public class Vfs(ILoggerFactory loggerFactory) : IDisposable
if(md5 != null)
{
var md5Bytes = new byte[16];
byte[] md5Bytes = new byte[16];
for(var i = 0; i < 16; i++)
for(int i = 0; i < 16; i++)
{
if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39)
md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10);

3
RomRepoMgr.Core/RomRepoMgr.Core.csproj
View File

@@ -1,7 +1,7 @@
<Project Sdk="Microsoft.NET.Sdk">
<ItemGroup>
<PackageReference Include="Aaru.Checksums.Native" />
<PackageReference Include="Aaru.Checksums.Native"/>
<PackageReference Include="DotNetZip"/>
<PackageReference Include="EFCore.BulkExtensions"/>
<PackageReference Include="Mono.Fuse.NETStandard"/>
@@ -19,6 +19,7 @@
<PackageReference Include="Roslynator.Formatting.Analyzers"/>
<PackageReference Include="SmartAnalyzers.MultithreadingAnalyzer"/>
<PackageReference Include="Text.Analyzers"/>
<PackageReference Include="ZstdSharp.Port"/>
</ItemGroup>
<ItemGroup>

35
RomRepoMgr.Core/Workers/Compression.cs
View File

@@ -28,8 +28,11 @@ using System.Diagnostics;
using System.IO;
using RomRepoMgr.Core.EventArgs;
using RomRepoMgr.Core.Resources;
using RomRepoMgr.Settings;
using SharpCompress.Compressors;
using SharpCompress.Compressors.LZMA;
using ZstdSharp;
using ZstdSharp.Unsafe;
using ErrorEventArgs = RomRepoMgr.Core.EventArgs.ErrorEventArgs;
namespace RomRepoMgr.Core.Workers;
@@ -45,11 +48,21 @@ public sealed class Compression
public void CompressFile(string source, string destination)
{
var inFs = new FileStream(source, FileMode.Open, FileAccess.Read);
var outFs = new FileStream(destination, FileMode.CreateNew, FileAccess.Write);
Stream zStream = new LZipStream(outFs, CompressionMode.Compress);
var inFs = new FileStream(source, FileMode.Open, FileAccess.Read);
var outFs = new FileStream(destination, FileMode.CreateNew, FileAccess.Write);
var buffer = new byte[BUFFER_SIZE];
Stream zStream;
if(Settings.Settings.Current.Compression == CompressionType.Zstd)
{
var zstdStream = new CompressionStream(outFs, 15);
zstdStream.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, Environment.ProcessorCount);
zStream = zstdStream;
}
else
zStream = new LZipStream(outFs, CompressionMode.Compress);
byte[] buffer = new byte[BUFFER_SIZE];
SetProgressBounds?.Invoke(this,
new ProgressBoundsEventArgs
@@ -89,9 +102,17 @@ public sealed class Compression
public void DecompressFile(string source, string destination)
{
var inFs = new FileStream(source, FileMode.Open, FileAccess.Read);
var outFs = new FileStream(destination, FileMode.Create, FileAccess.Write);
Stream zStream = new LZipStream(inFs, CompressionMode.Decompress);
var inFs = new FileStream(source, FileMode.Open, FileAccess.Read);
var outFs = new FileStream(destination, FileMode.Create, FileAccess.Write);
Stream zStream;
if(Path.GetExtension(source) == ".zst")
zStream = new DecompressionStream(inFs);
else if(Path.GetExtension(source) == ".lz")
zStream = new LZipStream(inFs, CompressionMode.Decompress);
else
throw new ArgumentException($"Invalid compression extension {Path.GetExtension(source)}");
zStream.CopyTo(outFs);

62
RomRepoMgr.Core/Workers/FileExporter.cs
View File

@@ -12,6 +12,7 @@ using RomRepoMgr.Core.Resources;
using RomRepoMgr.Database;
using RomRepoMgr.Database.Models;
using SharpCompress.Compressors.LZMA;
using ZstdSharp;
using CompressionMode = SharpCompress.Compressors.CompressionMode;
namespace RomRepoMgr.Core.Workers;
@@ -155,10 +156,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
if(media.Sha256 != null)
{
var sha256Bytes = new byte[32];
byte[] sha256Bytes = new byte[32];
string sha256 = media.Sha256;
for(var i = 0; i < 32; i++)
for(int i = 0; i < 32; i++)
{
if(sha256[i * 2] >= 0x30 && sha256[i * 2] <= 0x39)
sha256Bytes[i] = (byte)((sha256[i * 2] - 0x30) * 0x10);
@@ -190,10 +191,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
if(media.Sha1 != null)
{
var sha1Bytes = new byte[20];
byte[] sha1Bytes = new byte[20];
string sha1 = media.Sha1;
for(var i = 0; i < 20; i++)
for(int i = 0; i < 20; i++)
{
if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39)
sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10);
@@ -225,10 +226,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
if(media.Md5 != null)
{
var md5Bytes = new byte[16];
byte[] md5Bytes = new byte[16];
string md5 = media.Md5;
for(var i = 0; i < 16; i++)
for(int i = 0; i < 16; i++)
{
if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39)
md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10);
@@ -286,7 +287,7 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
Maximum = inFs.Length
});
var buffer = new byte[BUFFER_SIZE];
byte[] buffer = new byte[BUFFER_SIZE];
while(inFs.Position + BUFFER_SIZE <= inFs.Length)
{
@@ -359,10 +360,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
if(disk.Sha1 != null)
{
var sha1Bytes = new byte[20];
byte[] sha1Bytes = new byte[20];
string sha1 = disk.Sha1;
for(var i = 0; i < 20; i++)
for(int i = 0; i < 20; i++)
{
if(sha1[i * 2] >= 0x30 && sha1[i * 2] <= 0x39)
sha1Bytes[i] = (byte)((sha1[i * 2] - 0x30) * 0x10);
@@ -394,10 +395,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
if(disk.Md5 != null)
{
var md5Bytes = new byte[16];
byte[] md5Bytes = new byte[16];
string md5 = disk.Md5;
for(var i = 0; i < 16; i++)
for(int i = 0; i < 16; i++)
{
if(md5[i * 2] >= 0x30 && md5[i * 2] <= 0x39)
md5Bytes[i] = (byte)((md5[i * 2] - 0x30) * 0x10);
@@ -453,7 +454,7 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
Maximum = inFs.Length
});
var buffer = new byte[BUFFER_SIZE];
byte[] buffer = new byte[BUFFER_SIZE];
while(inFs.Position + BUFFER_SIZE <= inFs.Length)
{
@@ -558,10 +559,10 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
// Special case for empty file, as it seems to crash when SharpCompress tries to unLZMA it.
if(file.Size == 0) return new MemoryStream();
var sha384Bytes = new byte[48];
byte[] sha384Bytes = new byte[48];
string sha384 = file.Sha384;
for(var i = 0; i < 48; i++)
for(int i = 0; i < 48; i++)
{
if(sha384[i * 2] >= 0x30 && sha384[i * 2] <= 0x39)
sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10);
@@ -589,10 +590,29 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
sha384B32[4].ToString(),
sha384B32 + ".lz");
if(!File.Exists(repoPath))
throw new ArgumentException(string.Format(Localization.CannotFindHashInRepository, file.Sha256));
FileStream inFs;
var inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read);
// Try ZSTD
if(!File.Exists(repoPath))
{
repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath,
"files",
sha384B32[0].ToString(),
sha384B32[1].ToString(),
sha384B32[2].ToString(),
sha384B32[3].ToString(),
sha384B32[4].ToString(),
sha384B32 + ".zst");
if(!File.Exists(repoPath))
throw new ArgumentException(string.Format(Localization.CannotFindHashInRepository, file.Sha256));
inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read);
return new StreamWithLength(new DecompressionStream(inFs), (long)file.Size);
}
inFs = new FileStream(repoPath, FileMode.Open, FileAccess.Read);
return new StreamWithLength(new LZipStream(inFs, CompressionMode.Decompress), (long)file.Size);
}
@@ -612,11 +632,9 @@ public class FileExporter(long romSetId, string outPath, ILoggerFactory loggerFa
Value = _filePosition
});
if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName, out FileByMachine fileByMachine))
{
if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName.Replace('/', '\\'), out fileByMachine))
throw new ArgumentException(Localization.CannotFindZipEntryInDictionary);
}
if(!_filesByMachine.TryGetValue(e.CurrentEntry.FileName, out FileByMachine fileByMachine) &&
!_filesByMachine.TryGetValue(e.CurrentEntry.FileName.Replace('/', '\\'), out fileByMachine))
throw new ArgumentException(Localization.CannotFindZipEntryInDictionary);
DbFile currentFile = fileByMachine.File;

21
RomRepoMgr.Core/Workers/FileImporter.cs
View File

@@ -15,9 +15,12 @@ using RomRepoMgr.Core.Models;
using RomRepoMgr.Core.Resources;
using RomRepoMgr.Database;
using RomRepoMgr.Database.Models;
using RomRepoMgr.Settings;
using SabreTools.FileTypes.Aaru;
using SabreTools.FileTypes.CHD;
using SharpCompress.Compressors.LZMA;
using ZstdSharp;
using ZstdSharp.Unsafe;
using CompressionMode = SharpCompress.Compressors.CompressionMode;
namespace RomRepoMgr.Core.Workers;
@@ -551,7 +554,9 @@ public sealed class FileImporter
if(!Directory.Exists(repoPath)) Directory.CreateDirectory(repoPath);
repoPath = Path.Combine(repoPath, sha384B32 + ".lz");
repoPath = Settings.Settings.Current.Compression == CompressionType.Zstd
? Path.Combine(repoPath, sha384B32 + ".zst")
: Path.Combine(repoPath, sha384B32 + ".lz");
if(dbFile.Crc32 == null)
{
@@ -605,8 +610,18 @@ public sealed class FileImporter
inFs.Position = 0;
var outFs = new FileStream(repoPath, FileMode.CreateNew, FileAccess.Write);
Stream zStream = new LZipStream(outFs, CompressionMode.Compress);
var outFs = new FileStream(repoPath, FileMode.CreateNew, FileAccess.Write);
Stream zStream;
if(Settings.Settings.Current.Compression == CompressionType.Zstd)
{
var zstdStream = new CompressionStream(outFs, 15);
zstdStream.SetParameter(ZSTD_cParameter.ZSTD_c_nbWorkers, Environment.ProcessorCount);
zStream = zstdStream;
}
else
zStream = new LZipStream(outFs, CompressionMode.Compress);
SetProgressBounds2?.Invoke(this,
new ProgressBoundsEventArgs