claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 19:24:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Move getting files to VFS.

Browse Source
This commit is contained in:
Natalia Portillo
2020-09-03 01:42:18 +01:00
parent ff9c5cac19
commit b4c7570492
2 changed files with 80 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
RomRepoMgr.Core/Filesystem/Fuse.cs
View File

@@ -21,7 +21,6 @@ namespace RomRepoMgr.Core.Filesystem
{ {
readonly ConcurrentDictionary<long, List<DirectoryEntry>> _directoryCache; readonly ConcurrentDictionary<long, List<DirectoryEntry>> _directoryCache;
readonly ConcurrentDictionary<long, Stat> _fileStatHandleCache; readonly ConcurrentDictionary<long, Stat> _fileStatHandleCache;
readonly ConcurrentDictionary<ulong, ConcurrentDictionary<string, CachedFile>> _machineFilesCache;
readonly ConcurrentDictionary<long, Stream> _streamsCache; readonly ConcurrentDictionary<long, Stream> _streamsCache;
readonly Vfs _vfs; readonly Vfs _vfs;
long _lastHandle; long _lastHandle;
@@ -31,7 +30,6 @@ namespace RomRepoMgr.Core.Filesystem
{ {
_directoryCache = new ConcurrentDictionary<long, List<DirectoryEntry>>(); _directoryCache = new ConcurrentDictionary<long, List<DirectoryEntry>>();
_lastHandle = 0; _lastHandle = 0;
_machineFilesCache = new ConcurrentDictionary<ulong, ConcurrentDictionary<string, CachedFile>>();
_streamsCache = new ConcurrentDictionary<long, Stream>(); _streamsCache = new ConcurrentDictionary<long, Stream>();
_fileStatHandleCache = new ConcurrentDictionary<long, Stat>(); _fileStatHandleCache = new ConcurrentDictionary<long, Stat>();
Name = "romrepombgrfs"; Name = "romrepombgrfs";
@@ -156,38 +154,9 @@ namespace RomRepoMgr.Core.Filesystem
return 0; return 0;
} }
_machineFilesCache.TryGetValue(machine.Id, out ConcurrentDictionary<string, CachedFile> cachedMachineFiles); CachedFile file = _vfs.GetFile(machine.Id, pieces[2]);
if(cachedMachineFiles == null) if(file == null)
{
using var ctx = Context.Create(Settings.Settings.Current.DatabasePath);
cachedMachineFiles = new ConcurrentDictionary<string, CachedFile>();
foreach(FileByMachine machineFile in ctx.FilesByMachines.Where(fbm => fbm.Machine.Id == machine.Id &&
fbm.File.IsInRepo))
{
var cachedFile = new CachedFile
{
Id = machineFile.File.Id,
Crc32 = machineFile.File.Crc32,
Md5 = machineFile.File.Md5,
Sha1 = machineFile.File.Sha1,
Sha256 = machineFile.File.Sha256,
Sha384 = machineFile.File.Sha384,
Sha512 = machineFile.File.Sha512,
Size = machineFile.File.Size,
CreatedOn = machineFile.File.CreatedOn,
UpdatedOn = machineFile.File.UpdatedOn
};
cachedMachineFiles[machineFile.Name] = cachedFile;
}
_machineFilesCache[machine.Id] = cachedMachineFiles;
}
if(!cachedMachineFiles.TryGetValue(pieces[2], out CachedFile file))
return Errno.ENOENT; return Errno.ENOENT;
if(pieces.Length == 3) if(pieces.Length == 3)
@@ -267,38 +236,9 @@ namespace RomRepoMgr.Core.Filesystem
if(pieces.Length == 2) if(pieces.Length == 2)
return Errno.EISDIR; return Errno.EISDIR;
_machineFilesCache.TryGetValue(machine.Id, out ConcurrentDictionary<string, CachedFile> cachedMachineFiles); CachedFile file = _vfs.GetFile(machine.Id, pieces[2]);
if(cachedMachineFiles == null) if(file == null)
{
using var ctx = Context.Create(Settings.Settings.Current.DatabasePath);
cachedMachineFiles = new ConcurrentDictionary<string, CachedFile>();
foreach(FileByMachine machineFile in ctx.FilesByMachines.Where(fbm => fbm.Machine.Id == machine.Id &&
fbm.File.IsInRepo))
{
var cachedFile = new CachedFile
{
Id = machineFile.File.Id,
Crc32 = machineFile.File.Crc32,
Md5 = machineFile.File.Md5,
Sha1 = machineFile.File.Sha1,
Sha256 = machineFile.File.Sha256,
Sha384 = machineFile.File.Sha384,
Sha512 = machineFile.File.Sha512,
Size = machineFile.File.Size,
CreatedOn = machineFile.File.CreatedOn,
UpdatedOn = machineFile.File.UpdatedOn
};
cachedMachineFiles[machineFile.Name] = cachedFile;
}
_machineFilesCache[machine.Id] = cachedMachineFiles;
}
if(!cachedMachineFiles.TryGetValue(pieces[2], out CachedFile file))
return Errno.ENOENT; return Errno.ENOENT;
if(pieces.Length > 3) if(pieces.Length > 3)
@@ -483,38 +423,9 @@ namespace RomRepoMgr.Core.Filesystem
if(pieces.Length == 2) if(pieces.Length == 2)
return Errno.ENODATA; return Errno.ENODATA;
_machineFilesCache.TryGetValue(machine.Id, out ConcurrentDictionary<string, CachedFile> cachedMachineFiles); CachedFile file = _vfs.GetFile(machine.Id, pieces[2]);
if(cachedMachineFiles == null) if(file == null)
{
using var ctx = Context.Create(Settings.Settings.Current.DatabasePath);
cachedMachineFiles = new ConcurrentDictionary<string, CachedFile>();
foreach(FileByMachine machineFile in ctx.FilesByMachines.Where(fbm => fbm.Machine.Id == machine.Id &&
fbm.File.IsInRepo))
{
var cachedFile = new CachedFile
{
Id = machineFile.File.Id,
Crc32 = machineFile.File.Crc32,
Md5 = machineFile.File.Md5,
Sha1 = machineFile.File.Sha1,
Sha256 = machineFile.File.Sha256,
Sha384 = machineFile.File.Sha384,
Sha512 = machineFile.File.Sha512,
Size = machineFile.File.Size,
CreatedOn = machineFile.File.CreatedOn,
UpdatedOn = machineFile.File.UpdatedOn
};
cachedMachineFiles[machineFile.Name] = cachedFile;
}
_machineFilesCache[machine.Id] = cachedMachineFiles;
}
if(!cachedMachineFiles.TryGetValue(pieces[2], out CachedFile file))
return Errno.ENOENT; return Errno.ENOENT;
if(pieces.Length > 3) if(pieces.Length > 3)
@@ -623,38 +534,9 @@ namespace RomRepoMgr.Core.Filesystem
if(pieces.Length == 2) if(pieces.Length == 2)
return 0; return 0;
_machineFilesCache.TryGetValue(machine.Id, out ConcurrentDictionary<string, CachedFile> cachedMachineFiles); CachedFile file = _vfs.GetFile(machine.Id, pieces[2]);
if(cachedMachineFiles == null) if(file == null)
{
using var ctx = Context.Create(Settings.Settings.Current.DatabasePath);
cachedMachineFiles = new ConcurrentDictionary<string, CachedFile>();
foreach(FileByMachine machineFile in ctx.FilesByMachines.Where(fbm => fbm.Machine.Id == machine.Id &&
fbm.File.IsInRepo))
{
var cachedFile = new CachedFile
{
Id = machineFile.File.Id,
Crc32 = machineFile.File.Crc32,
Md5 = machineFile.File.Md5,
Sha1 = machineFile.File.Sha1,
Sha256 = machineFile.File.Sha256,
Sha384 = machineFile.File.Sha384,
Sha512 = machineFile.File.Sha512,
Size = machineFile.File.Size,
CreatedOn = machineFile.File.CreatedOn,
UpdatedOn = machineFile.File.UpdatedOn
};
cachedMachineFiles[machineFile.Name] = cachedFile;
}
_machineFilesCache[machine.Id] = cachedMachineFiles;
}
if(!cachedMachineFiles.TryGetValue(pieces[2], out CachedFile file))
return Errno.ENOENT; return Errno.ENOENT;
if(pieces.Length > 3) if(pieces.Length > 3)
@@ -769,37 +651,7 @@ namespace RomRepoMgr.Core.Filesystem
if(machine == null) if(machine == null)
return Errno.ENOENT; return Errno.ENOENT;
_machineFilesCache.TryGetValue(machine.Id, ConcurrentDictionary<string, CachedFile> cachedMachineFiles = _vfs.GetFilesFromMachine(machine.Id);
out ConcurrentDictionary<string, CachedFile> cachedMachineFiles);
if(cachedMachineFiles == null)
{
using var ctx = Context.Create(Settings.Settings.Current.DatabasePath);
cachedMachineFiles = new ConcurrentDictionary<string, CachedFile>();
foreach(FileByMachine machineFile in
ctx.FilesByMachines.Where(fbm => fbm.Machine.Id == machine.Id && fbm.File.IsInRepo))
{
var file = new CachedFile
{
Id = machineFile.File.Id,
Crc32 = machineFile.File.Crc32,
Md5 = machineFile.File.Md5,
Sha1 = machineFile.File.Sha1,
Sha256 = machineFile.File.Sha256,
Sha384 = machineFile.File.Sha384,
Sha512 = machineFile.File.Sha512,
Size = machineFile.File.Size,
CreatedOn = machineFile.File.CreatedOn,
UpdatedOn = machineFile.File.UpdatedOn
};
cachedMachineFiles[machineFile.Name] = file;
}
_machineFilesCache[machine.Id] = cachedMachineFiles;
}
if(pieces.Length == 2) if(pieces.Length == 2)
{ {
@@ -887,38 +739,9 @@ namespace RomRepoMgr.Core.Filesystem
if(pieces.Length == 2) if(pieces.Length == 2)
return mode.HasFlag(AccessModes.W_OK) ? Errno.EROFS : 0; return mode.HasFlag(AccessModes.W_OK) ? Errno.EROFS : 0;
_machineFilesCache.TryGetValue(machine.Id, out ConcurrentDictionary<string, CachedFile> cachedMachineFiles); CachedFile file = _vfs.GetFile(machine.Id, pieces[2]);
if(cachedMachineFiles == null) if(file == null)
{
using var ctx = Context.Create(Settings.Settings.Current.DatabasePath);
cachedMachineFiles = new ConcurrentDictionary<string, CachedFile>();
foreach(FileByMachine machineFile in ctx.FilesByMachines.Where(fbm => fbm.Machine.Id == machine.Id &&
fbm.File.IsInRepo))
{
var cachedFile = new CachedFile
{
Id = machineFile.File.Id,
Crc32 = machineFile.File.Crc32,
Md5 = machineFile.File.Md5,
Sha1 = machineFile.File.Sha1,
Sha256 = machineFile.File.Sha256,
Sha384 = machineFile.File.Sha384,
Sha512 = machineFile.File.Sha512,
Size = machineFile.File.Size,
CreatedOn = machineFile.File.CreatedOn,
UpdatedOn = machineFile.File.UpdatedOn
};
cachedMachineFiles[machineFile.Name] = cachedFile;
}
_machineFilesCache[machine.Id] = cachedMachineFiles;
}
if(!cachedMachineFiles.TryGetValue(pieces[2], out CachedFile _))
return Errno.ENOENT; return Errno.ENOENT;
if(pieces.Length > 3) if(pieces.Length > 3)
@@ -964,19 +787,5 @@ namespace RomRepoMgr.Core.Filesystem
_umountToken = Base32.ToBase32String(token); _umountToken = Base32.ToBase32String(token);
setxattr(Path.Combine(MountPoint, ".fuse_umount"), _umountToken, IntPtr.Zero, 0, 0); setxattr(Path.Combine(MountPoint, ".fuse_umount"), _umountToken, IntPtr.Zero, 0, 0);
} }
sealed class CachedFile
{
public ulong Id { get; set; }
public ulong Size { get; set; }
public string Crc32 { get; set; }
public string Md5 { get; set; }
public string Sha1 { get; set; }
public string Sha256 { get; set; }
public string Sha384 { get; set; }
public string Sha512 { get; set; }
public DateTime CreatedOn { get; set; }
public DateTime UpdatedOn { get; set; }
}
} }
} }

63
RomRepoMgr.Core/Filesystem/Vfs.cs
View File

@@ -11,6 +11,7 @@ namespace RomRepoMgr.Core.Filesystem
{ {
public class Vfs : IDisposable public class Vfs : IDisposable
{ {
readonly ConcurrentDictionary<ulong, ConcurrentDictionary<string, CachedFile>> _machineFilesCache;
readonly ConcurrentDictionary<long, ConcurrentDictionary<string, CachedMachine>> _machinesStatCache; readonly ConcurrentDictionary<long, ConcurrentDictionary<string, CachedMachine>> _machinesStatCache;
readonly ConcurrentDictionary<long, RomSet> _romSetsCache; readonly ConcurrentDictionary<long, RomSet> _romSetsCache;
Fuse _fuse; Fuse _fuse;
@@ -22,6 +23,7 @@ namespace RomRepoMgr.Core.Filesystem
_rootDirectoryCache = new ConcurrentDictionary<string, long>(); _rootDirectoryCache = new ConcurrentDictionary<string, long>();
_romSetsCache = new ConcurrentDictionary<long, RomSet>(); _romSetsCache = new ConcurrentDictionary<long, RomSet>();
_machinesStatCache = new ConcurrentDictionary<long, ConcurrentDictionary<string, CachedMachine>>(); _machinesStatCache = new ConcurrentDictionary<long, ConcurrentDictionary<string, CachedMachine>>();
_machineFilesCache = new ConcurrentDictionary<ulong, ConcurrentDictionary<string, CachedFile>>();
} }
public static bool IsAvailable => Winfsp.IsAvailable || Fuse.IsAvailable; public static bool IsAvailable => Winfsp.IsAvailable || Fuse.IsAvailable;
@@ -190,6 +192,53 @@ namespace RomRepoMgr.Core.Filesystem
return machine; return machine;
} }
internal ConcurrentDictionary<string, CachedFile> GetFilesFromMachine(ulong id)
{
_machineFilesCache.TryGetValue(id, out ConcurrentDictionary<string, CachedFile> cachedMachineFiles);
if(cachedMachineFiles != null)
return cachedMachineFiles;
using var ctx = Context.Create(Settings.Settings.Current.DatabasePath);
cachedMachineFiles = new ConcurrentDictionary<string, CachedFile>();
foreach(FileByMachine machineFile in ctx.FilesByMachines.Where(fbm => fbm.Machine.Id == id &&
fbm.File.IsInRepo))
{
var cachedFile = new CachedFile
{
Id = machineFile.File.Id,
Crc32 = machineFile.File.Crc32,
Md5 = machineFile.File.Md5,
Sha1 = machineFile.File.Sha1,
Sha256 = machineFile.File.Sha256,
Sha384 = machineFile.File.Sha384,
Sha512 = machineFile.File.Sha512,
Size = machineFile.File.Size,
CreatedOn = machineFile.File.CreatedOn,
UpdatedOn = machineFile.File.UpdatedOn
};
cachedMachineFiles[machineFile.Name] = cachedFile;
}
_machineFilesCache[id] = cachedMachineFiles;
return cachedMachineFiles;
}
internal CachedFile GetFile(ulong machineId, string name)
{
ConcurrentDictionary<string, CachedFile> cachedFiles = GetFilesFromMachine(machineId);
if(cachedFiles == null ||
!cachedFiles.TryGetValue(name, out CachedFile file))
return null;
return file;
}
} }
internal sealed class CachedMachine internal sealed class CachedMachine
@@ -198,4 +247,18 @@ namespace RomRepoMgr.Core.Filesystem
public DateTime CreationDate { get; set; } public DateTime CreationDate { get; set; }
public DateTime ModificationDate { get; set; } public DateTime ModificationDate { get; set; }
} }
internal sealed class CachedFile
{
public ulong Id { get; set; }
public ulong Size { get; set; }
public string Crc32 { get; set; }
public string Md5 { get; set; }
public string Sha1 { get; set; }
public string Sha256 { get; set; }
public string Sha384 { get; set; }
public string Sha512 { get; set; }
public DateTime CreatedOn { get; set; }
public DateTime UpdatedOn { get; set; }
}
} }