using System; using System.Collections.Generic; using System.IO; using System.Linq; using RomRepoMgr.Core.EventArgs; using RomRepoMgr.Core.Models; using RomRepoMgr.Database; using RomRepoMgr.Database.Models; using SharpCompress.Compressors; using SharpCompress.Compressors.LZMA; namespace RomRepoMgr.Core.Workers { public class FileImporter { const long BUFFER_SIZE = 131072; readonly bool _deleteAfterImport; readonly bool _onlyKnown; readonly Dictionary _pendingFiles; string _lastMessage; long _position; long _totalFiles; public FileImporter(bool onlyKnown, bool deleteAfterImport) { _pendingFiles = new Dictionary(); _onlyKnown = onlyKnown; _deleteAfterImport = deleteAfterImport; _position = 0; } public event EventHandler SetIndeterminateProgress2; public event EventHandler SetProgressBounds2; public event EventHandler SetProgress2; public event EventHandler SetMessage2; public event EventHandler SetIndeterminateProgress; public event EventHandler SetProgressBounds; public event EventHandler SetProgress; public event EventHandler SetMessage; public event EventHandler Finished; public event EventHandler ImportedRom; public void ProcessPath(string path, bool rootPath, bool removePathOnFinish) { try { SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty); SetMessage?.Invoke(this, new MessageEventArgs { Message = "Enumerating files..." }); string[] files = Directory.GetFiles(path, "*", SearchOption.AllDirectories); _totalFiles += files.LongLength; SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs { Minimum = 0, Maximum = _totalFiles }); foreach(string file in files) { SetProgress?.Invoke(this, new ProgressEventArgs { Value = _position }); SetMessage?.Invoke(this, new MessageEventArgs { Message = string.Format("Importing {0}...", Path.GetFileName(file)) }); bool ret = ImportRom(file); if(ret) { ImportedRom?.Invoke(this, new ImportedRomItemEventArgs { Item = new ImportRomItem { Filename = Path.GetFileName(file), Status = "OK" } }); } else { ImportedRom?.Invoke(this, new ImportedRomItemEventArgs { Item = new ImportRomItem { Filename = Path.GetFileName(file), Status = string.Format("Error: {0}", _lastMessage) } }); } _position++; } if(removePathOnFinish) { SetMessage?.Invoke(this, new MessageEventArgs { Message = "Removing temporary path..." }); Directory.Delete(path, true); } if(!rootPath) return; SaveChanges(); Finished?.Invoke(this, System.EventArgs.Empty); } catch(Exception) { // TODO: Send error back if(rootPath) Finished?.Invoke(this, System.EventArgs.Empty); } } bool ImportRom(string path) { try { var inFs = new FileStream(path, FileMode.Open, FileAccess.Read); byte[] dataBuffer; SetMessage2?.Invoke(this, new MessageEventArgs { Message = "Hashing file..." }); var checksumWorker = new Checksum(); if(inFs.Length > BUFFER_SIZE) { SetProgressBounds2?.Invoke(this, new ProgressBoundsEventArgs { Minimum = 0, Maximum = inFs.Length }); long offset; long remainder = inFs.Length % BUFFER_SIZE; for(offset = 0; offset < inFs.Length - remainder; offset += (int)BUFFER_SIZE) { SetProgress2?.Invoke(this, new ProgressEventArgs { Value = offset }); dataBuffer = new byte[BUFFER_SIZE]; inFs.Read(dataBuffer, 0, (int)BUFFER_SIZE); checksumWorker.Update(dataBuffer); } SetProgress2?.Invoke(this, new ProgressEventArgs { Value = offset }); dataBuffer = new byte[remainder]; inFs.Read(dataBuffer, 0, (int)remainder); checksumWorker.Update(dataBuffer); } else { SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty); dataBuffer = new byte[inFs.Length]; inFs.Read(dataBuffer, 0, (int)inFs.Length); checksumWorker.Update(dataBuffer); } Dictionary checksums = checksumWorker.End(); ulong uSize = (ulong)inFs.Length; bool fileInDb = true; bool knownFile = _pendingFiles.TryGetValue(checksums[ChecksumType.Sha512], out DbFile dbFile); dbFile ??= ((((Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Sha512 == checksums[ChecksumType.Sha512]) ?? Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Sha384 == checksums[ChecksumType.Sha384])) ?? Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Sha256 == checksums[ChecksumType.Sha256])) ?? Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Sha1 == checksums[ChecksumType.Sha1])) ?? Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Md5 == checksums[ChecksumType.Md5])) ?? Context.Singleton.Files.FirstOrDefault(f => f.Size == uSize && f.Crc32 == checksums[ChecksumType.Crc32]); if(dbFile == null) { if(_onlyKnown) { _lastMessage = "Unknown file."; return false; } dbFile = new DbFile { Crc32 = checksums[ChecksumType.Crc32], Md5 = checksums[ChecksumType.Md5], Sha1 = checksums[ChecksumType.Sha1], Sha256 = checksums[ChecksumType.Sha256], Sha384 = checksums[ChecksumType.Sha384], Sha512 = checksums[ChecksumType.Sha512], Size = uSize, CreatedOn = DateTime.UtcNow, UpdatedOn = DateTime.UtcNow }; fileInDb = false; } if(!knownFile) _pendingFiles[checksums[ChecksumType.Sha512]] = dbFile; byte[] sha384Bytes = new byte[48]; string sha384 = checksums[ChecksumType.Sha384]; for(int i = 0; i < 48; i++) { if(sha384[i * 2] >= 0x30 && sha384[i * 2] <= 0x39) sha384Bytes[i] = (byte)((sha384[i * 2] - 0x30) * 0x10); else if(sha384[i * 2] >= 0x41 && sha384[i * 2] <= 0x46) sha384Bytes[i] = (byte)((sha384[i * 2] - 0x37) * 0x10); else if(sha384[i * 2] >= 0x61 && sha384[i * 2] <= 0x66) sha384Bytes[i] = (byte)((sha384[i * 2] - 0x57) * 0x10); if(sha384[(i * 2) + 1] >= 0x30 && sha384[(i * 2) + 1] <= 0x39) sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x30); else if(sha384[(i * 2) + 1] >= 0x41 && sha384[(i * 2) + 1] <= 0x46) sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x37); else if(sha384[(i * 2) + 1] >= 0x61 && sha384[(i * 2) + 1] <= 0x66) sha384Bytes[i] += (byte)(sha384[(i * 2) + 1] - 0x57); } string sha384B32 = Base32.ToBase32String(sha384Bytes); string repoPath = Path.Combine(Settings.Settings.Current.RepositoryPath, "files", sha384B32[0].ToString(), sha384B32[1].ToString(), sha384B32[2].ToString(), sha384B32[3].ToString(), sha384B32[4].ToString()); if(!Directory.Exists(repoPath)) Directory.CreateDirectory(repoPath); repoPath = Path.Combine(repoPath, sha384B32 + ".lz"); if(dbFile.Crc32 == null) { dbFile.Crc32 = checksums[ChecksumType.Crc32]; dbFile.UpdatedOn = DateTime.UtcNow; } if(dbFile.Md5 == null) { dbFile.Md5 = checksums[ChecksumType.Md5]; dbFile.UpdatedOn = DateTime.UtcNow; } if(dbFile.Sha1 == null) { dbFile.Sha1 = checksums[ChecksumType.Sha1]; dbFile.UpdatedOn = DateTime.UtcNow; } if(dbFile.Sha256 == null) { dbFile.Sha256 = checksums[ChecksumType.Sha256]; dbFile.UpdatedOn = DateTime.UtcNow; } if(dbFile.Sha384 == null) { dbFile.Sha384 = checksums[ChecksumType.Sha384]; dbFile.UpdatedOn = DateTime.UtcNow; } if(dbFile.Sha512 == null) { dbFile.Sha512 = checksums[ChecksumType.Sha512]; dbFile.UpdatedOn = DateTime.UtcNow; } if(File.Exists(repoPath)) { dbFile.IsInRepo = true; dbFile.UpdatedOn = DateTime.UtcNow; if(!fileInDb) Context.Singleton.Files.Add(dbFile); inFs.Close(); if(_deleteAfterImport) File.Delete(path); return true; } inFs.Position = 0; var outFs = new FileStream(repoPath, FileMode.CreateNew, FileAccess.Write); Stream zStream = null; zStream = new LZipStream(outFs, CompressionMode.Compress); SetProgressBounds2?.Invoke(this, new ProgressBoundsEventArgs { Minimum = 0, Maximum = inFs.Length }); SetMessage2?.Invoke(this, new MessageEventArgs { Message = "Compressing file..." }); byte[] buffer = new byte[BUFFER_SIZE]; while(inFs.Position + BUFFER_SIZE <= inFs.Length) { SetProgress2?.Invoke(this, new ProgressEventArgs { Value = inFs.Position }); inFs.Read(buffer, 0, buffer.Length); zStream.Write(buffer, 0, buffer.Length); } buffer = new byte[inFs.Length - inFs.Position]; SetProgress2?.Invoke(this, new ProgressEventArgs { Value = inFs.Position }); inFs.Read(buffer, 0, buffer.Length); zStream.Write(buffer, 0, buffer.Length); SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty); SetMessage2?.Invoke(this, new MessageEventArgs { Message = "Finishing..." }); inFs.Close(); zStream.Close(); outFs.Dispose(); dbFile.IsInRepo = true; dbFile.UpdatedOn = DateTime.UtcNow; if(!fileInDb) Context.Singleton.Files.Add(dbFile); if(_deleteAfterImport) File.Delete(path); return true; } catch(Exception e) { _lastMessage = "Unhandled exception when importing file."; return false; } } void SaveChanges() { SetIndeterminateProgress2?.Invoke(this, System.EventArgs.Empty); SetMessage2?.Invoke(this, new MessageEventArgs { Message = "Saving changes to database..." }); Context.Singleton.SaveChanges(); } } }