mirror of
https://github.com/claunia/romrepomgr.git
synced 2025-12-16 19:24:51 +00:00
Implement importing files from folder.
This commit is contained in:
187
RomRepoMgr.Core/Workers/Checksum.cs
Normal file
187
RomRepoMgr.Core/Workers/Checksum.cs
Normal file
@@ -0,0 +1,187 @@
|
||||
// /***************************************************************************
|
||||
// The Disc Image Chef
|
||||
// ----------------------------------------------------------------------------
|
||||
//
|
||||
// Filename : Checksum.cs
|
||||
// Author(s) : Natalia Portillo <claunia@claunia.com>
|
||||
//
|
||||
// Component : Core methods.
|
||||
//
|
||||
// --[ Description ] ----------------------------------------------------------
|
||||
//
|
||||
// Methods to checksum data.
|
||||
//
|
||||
// --[ License ] --------------------------------------------------------------
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
//
|
||||
// ----------------------------------------------------------------------------
|
||||
// Copyright © 2011-2016 Natalia Portillo
|
||||
// ****************************************************************************/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using Aaru.Checksums;
|
||||
|
||||
namespace RomRepoMgr.Core.Workers
|
||||
{
|
||||
internal enum ChecksumType
|
||||
{
|
||||
Crc32, Md5, Sha1,
|
||||
Sha256, Sha384, Sha512
|
||||
}
|
||||
|
||||
internal sealed class Checksum
|
||||
{
|
||||
readonly Crc32Context _crc32Ctx;
|
||||
readonly Md5Context _md5Ctx;
|
||||
readonly Sha1Context _sha1Ctx;
|
||||
readonly Sha256Context _sha256Ctx;
|
||||
readonly Sha384Context _sha384Ctx;
|
||||
readonly Sha512Context _sha512Ctx;
|
||||
Crc32Packet _crc32Pkt;
|
||||
Thread _crc32Thread;
|
||||
Md5Packet _md5Pkt;
|
||||
Thread _md5Thread;
|
||||
Sha1Packet _sha1Pkt;
|
||||
Thread _sha1Thread;
|
||||
Sha256Packet _sha256Pkt;
|
||||
Thread _sha256Thread;
|
||||
Sha384Packet _sha384Pkt;
|
||||
Thread _sha384Thread;
|
||||
Sha512Packet _sha512Pkt;
|
||||
Thread _sha512Thread;
|
||||
|
||||
internal Checksum()
|
||||
{
|
||||
_crc32Ctx = new Crc32Context();
|
||||
_md5Ctx = new Md5Context();
|
||||
_sha1Ctx = new Sha1Context();
|
||||
_sha256Ctx = new Sha256Context();
|
||||
_sha384Ctx = new Sha384Context();
|
||||
_sha512Ctx = new Sha512Context();
|
||||
|
||||
_crc32Thread = new Thread(UpdateCrc32);
|
||||
_md5Thread = new Thread(UpdateMd5);
|
||||
_sha1Thread = new Thread(UpdateSha1);
|
||||
_sha256Thread = new Thread(UpdateSha256);
|
||||
_sha384Thread = new Thread(UpdateSha384);
|
||||
_sha512Thread = new Thread(UpdateSha512);
|
||||
|
||||
_crc32Pkt = new Crc32Packet();
|
||||
_md5Pkt = new Md5Packet();
|
||||
_sha1Pkt = new Sha1Packet();
|
||||
_sha256Pkt = new Sha256Packet();
|
||||
_sha384Pkt = new Sha384Packet();
|
||||
_sha512Pkt = new Sha512Packet();
|
||||
|
||||
_crc32Pkt.context = _crc32Ctx;
|
||||
_md5Pkt.context = _md5Ctx;
|
||||
_sha1Pkt.context = _sha1Ctx;
|
||||
_sha256Pkt.context = _sha256Ctx;
|
||||
_sha384Pkt.context = _sha384Ctx;
|
||||
_sha512Pkt.context = _sha512Ctx;
|
||||
}
|
||||
|
||||
internal void Update(byte[] data)
|
||||
{
|
||||
_crc32Pkt.data = data;
|
||||
_crc32Thread.Start(_crc32Pkt);
|
||||
_md5Pkt.data = data;
|
||||
_md5Thread.Start(_md5Pkt);
|
||||
_sha1Pkt.data = data;
|
||||
_sha1Thread.Start(_sha1Pkt);
|
||||
_sha256Pkt.data = data;
|
||||
_sha256Thread.Start(_sha256Pkt);
|
||||
_sha384Pkt.data = data;
|
||||
_sha384Thread.Start(_sha384Pkt);
|
||||
_sha512Pkt.data = data;
|
||||
_sha512Thread.Start(_sha512Pkt);
|
||||
|
||||
while(_crc32Thread.IsAlive ||
|
||||
_md5Thread.IsAlive ||
|
||||
_sha1Thread.IsAlive ||
|
||||
_sha256Thread.IsAlive ||
|
||||
_sha384Thread.IsAlive ||
|
||||
_sha512Thread.IsAlive) {}
|
||||
|
||||
_crc32Thread = new Thread(UpdateCrc32);
|
||||
_md5Thread = new Thread(UpdateMd5);
|
||||
_sha1Thread = new Thread(UpdateSha1);
|
||||
_sha256Thread = new Thread(UpdateSha256);
|
||||
_sha384Thread = new Thread(UpdateSha384);
|
||||
_sha512Thread = new Thread(UpdateSha512);
|
||||
}
|
||||
|
||||
internal Dictionary<ChecksumType, string> End() => new Dictionary<ChecksumType, string>
|
||||
{
|
||||
[ChecksumType.Crc32] = _crc32Ctx.End(),
|
||||
[ChecksumType.Md5] = _md5Ctx.End(),
|
||||
[ChecksumType.Sha1] = _sha1Ctx.End(),
|
||||
[ChecksumType.Sha256] = _sha256Ctx.End(),
|
||||
[ChecksumType.Sha384] = _sha384Ctx.End(),
|
||||
[ChecksumType.Sha512] = _sha512Ctx.End()
|
||||
};
|
||||
|
||||
#region Threading helpers
|
||||
struct Crc32Packet
|
||||
{
|
||||
public Crc32Context context;
|
||||
public byte[] data;
|
||||
}
|
||||
|
||||
struct Md5Packet
|
||||
{
|
||||
public Md5Context context;
|
||||
public byte[] data;
|
||||
}
|
||||
|
||||
struct Sha1Packet
|
||||
{
|
||||
public Sha1Context context;
|
||||
public byte[] data;
|
||||
}
|
||||
|
||||
struct Sha256Packet
|
||||
{
|
||||
public Sha256Context context;
|
||||
public byte[] data;
|
||||
}
|
||||
|
||||
struct Sha384Packet
|
||||
{
|
||||
public Sha384Context context;
|
||||
public byte[] data;
|
||||
}
|
||||
|
||||
struct Sha512Packet
|
||||
{
|
||||
public Sha512Context context;
|
||||
public byte[] data;
|
||||
}
|
||||
|
||||
static void UpdateCrc32(object packet) => ((Crc32Packet)packet).context.Update(((Crc32Packet)packet).data);
|
||||
|
||||
static void UpdateMd5(object packet) => ((Md5Packet)packet).context.Update(((Md5Packet)packet).data);
|
||||
|
||||
static void UpdateSha1(object packet) => ((Sha1Packet)packet).context.Update(((Sha1Packet)packet).data);
|
||||
|
||||
static void UpdateSha256(object packet) => ((Sha256Packet)packet).context.Update(((Sha256Packet)packet).data);
|
||||
|
||||
static void UpdateSha384(object packet) => ((Sha384Packet)packet).context.Update(((Sha384Packet)packet).data);
|
||||
|
||||
static void UpdateSha512(object packet) => ((Sha512Packet)packet).context.Update(((Sha512Packet)packet).data);
|
||||
#endregion Threading helpers
|
||||
}
|
||||
}
|
||||
311
RomRepoMgr.Core/Workers/FileImporter.cs
Normal file
311
RomRepoMgr.Core/Workers/FileImporter.cs
Normal file
@@ -0,0 +1,311 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using RomRepoMgr.Core.EventArgs;
|
||||
using RomRepoMgr.Database;
|
||||
using RomRepoMgr.Database.Models;
|
||||
using SharpCompress.Compressors;
|
||||
using SharpCompress.Compressors.LZMA;
|
||||
|
||||
namespace RomRepoMgr.Core.Workers
|
||||
{
|
||||
public class FileImporter
|
||||
{
|
||||
const long BUFFER_SIZE = 131072;
|
||||
readonly bool _deleteAfterImport;
|
||||
readonly bool _onlyKnown;
|
||||
|
||||
readonly Dictionary<string, DbFile> _pendingFiles;
|
||||
|
||||
public FileImporter(bool onlyKnown, bool deleteAfterImport)
|
||||
{
|
||||
_pendingFiles = new Dictionary<string, DbFile>();
|
||||
_onlyKnown = onlyKnown;
|
||||
_deleteAfterImport = deleteAfterImport;
|
||||
}
|
||||
|
||||
public string LastMessage { get; private set; }
|
||||
|
||||
public event EventHandler SetIndeterminateProgress;
|
||||
public event EventHandler<ProgressBoundsEventArgs> SetProgressBounds;
|
||||
public event EventHandler<ProgressEventArgs> SetProgress;
|
||||
public event EventHandler<MessageEventArgs> SetMessage;
|
||||
|
||||
public bool ImportRom(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var inFs = new FileStream(path, FileMode.Open, FileAccess.Read);
|
||||
|
||||
byte[] dataBuffer;
|
||||
|
||||
SetMessage?.Invoke(this, new MessageEventArgs
|
||||
{
|
||||
Message = "Hashing file..."
|
||||
});
|
||||
|
||||
var checksumWorker = new Checksum();
|
||||
|
||||
if(inFs.Length > BUFFER_SIZE)
|
||||
{
|
||||
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs
|
||||
{
|
||||
Minimum = 0,
|
||||
Maximum = inFs.Length
|
||||
});
|
||||
|
||||
long offset;
|
||||
long remainder = inFs.Length % BUFFER_SIZE;
|
||||
|
||||
for(offset = 0; offset < inFs.Length - remainder; offset += (int)BUFFER_SIZE)
|
||||
{
|
||||
SetProgress?.Invoke(this, new ProgressEventArgs
|
||||
{
|
||||
Value = offset
|
||||
});
|
||||
|
||||
dataBuffer = new byte[BUFFER_SIZE];
|
||||
inFs.Read(dataBuffer, 0, (int)BUFFER_SIZE);
|
||||
checksumWorker.Update(dataBuffer);
|
||||
}
|
||||
|
||||
SetProgress?.Invoke(this, new ProgressEventArgs
|
||||
{
|
||||
Value = offset
|
||||
});
|
||||
|
||||
dataBuffer = new byte[remainder];
|
||||
inFs.Read(dataBuffer, 0, (int)remainder);
|
||||
checksumWorker.Update(dataBuffer);
|
||||
}
|
||||
else
|
||||
{
|
||||
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
|
||||
dataBuffer = new byte[inFs.Length];
|
||||
inFs.Read(dataBuffer, 0, (int)inFs.Length);
|
||||
checksumWorker.Update(dataBuffer);
|
||||
}
|
||||
|
||||
Dictionary<ChecksumType, string> checksums = checksumWorker.End();
|
||||
|
||||
ulong uSize = (ulong)inFs.Length;
|
||||
bool fileInDb = true;
|
||||
|
||||
bool knownFile = _pendingFiles.TryGetValue(checksums[ChecksumType.Sha512], out DbFile dbFile);
|
||||
|
||||
dbFile ??=
|
||||
((((Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
|
||||
f.Sha512 == checksums[ChecksumType.Sha512]) ??
|
||||
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
|
||||
f.Sha384 == checksums[ChecksumType.Sha384])) ??
|
||||
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
|
||||
f.Sha256 == checksums[ChecksumType.Sha256])) ??
|
||||
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
|
||||
f.Sha1 == checksums[ChecksumType.Sha1])) ??
|
||||
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
|
||||
f.Md5 == checksums[ChecksumType.Md5])) ??
|
||||
Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize &&
|
||||
f.Crc32 == checksums[ChecksumType.Crc32]);
|
||||
|
||||
if(dbFile == null)
|
||||
{
|
||||
if(_onlyKnown)
|
||||
{
|
||||
LastMessage = "Unknown file.";
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
dbFile = new DbFile
|
||||
{
|
||||
Crc32 = checksums[ChecksumType.Crc32],
|
||||
Md5 = checksums[ChecksumType.Md5],
|
||||
Sha1 = checksums[ChecksumType.Sha1],
|
||||
Sha256 = checksums[ChecksumType.Sha256],
|
||||
Sha384 = checksums[ChecksumType.Sha384],
|
||||
Sha512 = checksums[ChecksumType.Sha512],
|
||||
Size = uSize,
|
||||
CreatedOn = DateTime.UtcNow,
|
||||
UpdatedOn = DateTime.UtcNow
|
||||
};
|
||||
|
||||
fileInDb = false;
|
||||
}
|
||||
|
||||
if(!knownFile)
|
||||
_pendingFiles[checksums[ChecksumType.Sha512]] = dbFile;
|
||||
|
||||
byte[] sha384Bytes = new byte[48];
|
||||
string sha384 = checksums[ChecksumType.Sha384];
|
||||
|
||||
for(int i = 0; i < 48; i++)
|
||||
{
|
||||
if(sha384[i * 2] >= 0x30 &&
|
||||
sha384[i * 2] <= 0x39)
|
||||
sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10);
|
||||
else if(sha384[i * 2] >= 0x41 &&
|
||||
sha384[i * 2] <= 0x46)
|
||||
sha384Bytes[i] = (byte)((sha384[i * 2] - 0x37) * 0x10);
|
||||
else if(sha384[i * 2] >= 0x61 &&
|
||||
sha384[i * 2] <= 0x66)
|
||||
sha384Bytes[i] = (byte)((sha384[i * 2] - 0x57) * 0x10);
|
||||
|
||||
if(sha384[(i * 2) + 1] >= 0x30 &&
|
||||
sha384[(i * 2) + 1] <= 0x39)
|
||||
sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x30);
|
||||
else if(sha384[(i * 2) + 1] >= 0x41 &&
|
||||
sha384[(i * 2) + 1] <= 0x46)
|
||||
sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x37);
|
||||
else if(sha384[(i * 2) + 1] >= 0x61 &&
|
||||
sha384[(i * 2) + 1] <= 0x66)
|
||||
sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x57);
|
||||
}
|
||||
|
||||
string sha384B32 = Base32.ToBase32String(sha384Bytes);
|
||||
|
||||
string repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, "files",
|
||||
sha384B32[0].ToString(), sha384B32[1].ToString(),
|
||||
sha384B32[2].ToString(), sha384B32[3].ToString(),
|
||||
sha384B32[4].ToString());
|
||||
|
||||
if(!Directory.Exists(repoPath))
|
||||
Directory.CreateDirectory(repoPath);
|
||||
|
||||
repoPath = Path.Combine(repoPath, sha384B32 + ".lz");
|
||||
|
||||
if(dbFile.Crc32 == null)
|
||||
{
|
||||
dbFile.Crc32 = checksums[ChecksumType.Crc32];
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if(dbFile.Md5 == null)
|
||||
{
|
||||
dbFile.Md5 = checksums[ChecksumType.Md5];
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if(dbFile.Sha1 == null)
|
||||
{
|
||||
dbFile.Sha1 = checksums[ChecksumType.Sha1];
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if(dbFile.Sha256 == null)
|
||||
{
|
||||
dbFile.Sha256 = checksums[ChecksumType.Sha256];
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if(dbFile.Sha384 == null)
|
||||
{
|
||||
dbFile.Sha384 = checksums[ChecksumType.Sha384];
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if(dbFile.Sha512 == null)
|
||||
{
|
||||
dbFile.Sha512 = checksums[ChecksumType.Sha512];
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if(File.Exists(repoPath))
|
||||
{
|
||||
dbFile.IsInRepo = true;
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
|
||||
if(!fileInDb)
|
||||
Context.Singleton.Files.Add(dbFile);
|
||||
|
||||
inFs.Close();
|
||||
|
||||
if(_deleteAfterImport)
|
||||
File.Delete(path);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
inFs.Position = 0;
|
||||
|
||||
var outFs = new FileStream(repoPath, FileMode.CreateNew, FileAccess.Write);
|
||||
Stream zStream = null;
|
||||
zStream = new LZipStream(outFs, CompressionMode.Compress);
|
||||
|
||||
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs
|
||||
{
|
||||
Minimum = 0,
|
||||
Maximum = inFs.Length
|
||||
});
|
||||
|
||||
SetMessage?.Invoke(this, new MessageEventArgs
|
||||
{
|
||||
Message = "Compressing file..."
|
||||
});
|
||||
|
||||
byte[] buffer = new byte[BUFFER_SIZE];
|
||||
|
||||
while(inFs.Position + BUFFER_SIZE <= inFs.Length)
|
||||
{
|
||||
SetProgress?.Invoke(this, new ProgressEventArgs
|
||||
{
|
||||
Value = inFs.Position
|
||||
});
|
||||
|
||||
inFs.Read(buffer, 0, buffer.Length);
|
||||
zStream.Write(buffer, 0, buffer.Length);
|
||||
}
|
||||
|
||||
buffer = new byte[inFs.Length - inFs.Position];
|
||||
|
||||
SetProgress?.Invoke(this, new ProgressEventArgs
|
||||
{
|
||||
Value = inFs.Position
|
||||
});
|
||||
|
||||
inFs.Read(buffer, 0, buffer.Length);
|
||||
zStream.Write(buffer, 0, buffer.Length);
|
||||
|
||||
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
|
||||
|
||||
SetMessage?.Invoke(this, new MessageEventArgs
|
||||
{
|
||||
Message = "Finishing..."
|
||||
});
|
||||
|
||||
inFs.Close();
|
||||
zStream.Close();
|
||||
outFs.Dispose();
|
||||
|
||||
dbFile.IsInRepo = true;
|
||||
dbFile.UpdatedOn = DateTime.UtcNow;
|
||||
|
||||
if(!fileInDb)
|
||||
Context.Singleton.Files.Add(dbFile);
|
||||
|
||||
if(_deleteAfterImport)
|
||||
File.Delete(path);
|
||||
|
||||
return true;
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LastMessage = "Unhandled exception when importing file.";
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
public void SaveChanges()
|
||||
{
|
||||
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
|
||||
|
||||
SetMessage?.Invoke(this, new MessageEventArgs
|
||||
{
|
||||
Message = "Saving changes to database..."
|
||||
});
|
||||
|
||||
Context.Singleton.SaveChanges();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user