Files
romrepomgr/RomRepoMgr.Core/Workers/DatImporter.cs

630 lines
25 KiB
C#
Raw Normal View History

2020-08-22 04:40:39 +01:00
/******************************************************************************
// RomRepoMgr - ROM repository manager
// ----------------------------------------------------------------------------
//
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// --[ License ] --------------------------------------------------------------
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2020 Natalia Portillo
*******************************************************************************/
using System;
2020-08-22 14:20:59 +01:00
using System.Collections.Generic;
2020-08-22 04:40:39 +01:00
using System.Diagnostics;
2020-08-22 05:14:52 +01:00
using System.IO;
2020-08-22 14:20:59 +01:00
using System.Linq;
2020-08-22 05:14:52 +01:00
using Aaru.Checksums;
2020-08-22 04:40:39 +01:00
using RomRepoMgr.Core.EventArgs;
using RomRepoMgr.Core.Models;
2020-08-30 03:00:14 +01:00
using RomRepoMgr.Core.Resources;
2020-08-22 05:40:50 +01:00
using RomRepoMgr.Database;
using RomRepoMgr.Database.Models;
2020-08-22 04:40:39 +01:00
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
2020-08-22 05:14:52 +01:00
using ErrorEventArgs = RomRepoMgr.Core.EventArgs.ErrorEventArgs;
using Machine = RomRepoMgr.Database.Models.Machine;
2020-08-22 04:40:39 +01:00
namespace RomRepoMgr.Core.Workers
{
public sealed class DatImporter
{
2020-08-22 05:14:52 +01:00
readonly string _datFilesPath;
2020-08-22 04:40:39 +01:00
readonly string _datPath;
bool _aborted;
2020-08-22 05:14:52 +01:00
public DatImporter(string datPath)
{
_datPath = datPath;
_datFilesPath = Path.Combine(Settings.Settings.Current.RepositoryPath, "datfiles");
}
2020-08-22 04:40:39 +01:00
public void Import()
{
try
{
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.ParsinDatFile
2020-08-22 04:40:39 +01:00
});
var datFile = DatFile.CreateAndParse(_datPath);
2020-08-22 04:40:39 +01:00
2020-08-22 05:14:52 +01:00
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.HashingDatFile
2020-08-22 05:14:52 +01:00
});
2020-08-22 12:38:21 +01:00
string datHash = Sha384Context.File(_datPath, out byte[] datHashBinary);
string datHash32 = Base32.ToBase32String(datHashBinary);
2020-08-22 05:14:52 +01:00
if(!Directory.Exists(_datFilesPath))
Directory.CreateDirectory(_datFilesPath);
2020-08-22 12:38:21 +01:00
string compressedDatPath = Path.Combine(_datFilesPath, datHash32 + ".lz");
2020-08-22 05:14:52 +01:00
if(File.Exists(compressedDatPath))
{
ErrorOccurred?.Invoke(this, new ErrorEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.DatAlreadyInDatabase
2020-08-22 05:14:52 +01:00
});
return;
}
2020-08-22 05:40:50 +01:00
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.AddingDatToDatabase
2020-08-22 05:40:50 +01:00
});
2020-08-22 12:38:21 +01:00
// TODO: Check if there is a hash in database but not in repo
2020-08-22 05:40:50 +01:00
var romSet = new RomSet
{
Author = datFile.Header.Author,
Comment = datFile.Header.Comment,
Date = datFile.Header.Date,
Description = datFile.Header.Description,
Filename = Path.GetFileName(_datPath),
Homepage = datFile.Header.Homepage,
Name = datFile.Header.Name,
Sha384 = datHash,
Version = datFile.Header.Version,
CreatedOn = DateTime.UtcNow,
UpdatedOn = DateTime.UtcNow
};
Context.Singleton.RomSets.Add(romSet);
Context.Singleton.SaveChanges();
2020-08-22 05:14:52 +01:00
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.CompressingDatFile
2020-08-22 05:14:52 +01:00
});
var datCompress = new Compression();
datCompress.SetProgress += SetProgress;
datCompress.SetProgressBounds += SetProgressBounds;
datCompress.CompressFile(_datPath, compressedDatPath);
2020-08-22 14:20:59 +01:00
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.GettingMachineNames
2020-08-22 14:20:59 +01:00
});
List<string> machineNames =
(from value in datFile.Items.Values from item in value select item.Machine.Name).Distinct().
ToList();
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.AddingMachines
2020-08-22 14:20:59 +01:00
});
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs
{
Minimum = 0,
Maximum = machineNames.Count
});
int position = 0;
Dictionary<string, Machine> machines = new Dictionary<string, Machine>();
foreach(string name in machineNames)
{
SetProgress?.Invoke(this, new ProgressEventArgs
{
Value = position
});
var machine = new Machine
{
Name = name,
RomSet = romSet,
CreatedOn = DateTime.UtcNow,
UpdatedOn = DateTime.UtcNow
};
machines[name] = machine;
Context.Singleton.Machines.Add(machine);
position++;
}
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.SavingChangesToDatabase
2020-08-22 14:20:59 +01:00
});
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
Context.Singleton.SaveChanges();
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.RetrievingRomsAndDisks
});
List<Rom> roms = new List<Rom>();
List<Disk> disks = new List<Disk>();
foreach(List<DatItem> values in datFile.Items.Values)
{
foreach(DatItem item in values)
{
switch(item)
{
case Rom rom:
roms.Add(rom);
continue;
case Disk disk:
disks.Add(disk);
continue;
}
}
}
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs
{
Minimum = 0,
Maximum = roms.Count
});
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.AddingRoms
});
position = 0;
Dictionary<string, DbFile> pendingFilesBySha512 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesBySha384 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesBySha256 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesBySha1 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesByMd5 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesByCrc = new Dictionary<string, DbFile>();
List<DbFile> pendingFiles = new List<DbFile>();
2020-09-03 19:45:39 +01:00
List<DbFile> newFiles = new List<DbFile>();
List<FileByMachine> newFilesByMachine = new List<FileByMachine>();
foreach(Rom rom in roms)
{
bool hashCollision = false;
SetProgress?.Invoke(this, new ProgressEventArgs
{
Value = position
});
if(!machines.TryGetValue(rom.Machine.Name, out Machine machine))
{
ErrorOccurred?.Invoke(this, new ErrorEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.FoundRomWithoutMachine
});
return;
}
ulong uSize = (ulong)rom.Size;
DbFile file = null;
if(rom.SHA512 != null)
if(pendingFilesBySha512.TryGetValue(rom.SHA512, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.SHA384 != null &&
file == null)
if(pendingFilesBySha384.TryGetValue(rom.SHA384, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.SHA256 != null &&
file == null)
if(pendingFilesBySha256.TryGetValue(rom.SHA256, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.SHA1 != null &&
file == null)
if(pendingFilesBySha1.TryGetValue(rom.SHA1, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.MD5 != null &&
file == null)
if(pendingFilesByMd5.TryGetValue(rom.MD5, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.CRC != null &&
file == null)
if(pendingFilesByCrc.TryGetValue(rom.CRC, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(file == null && hashCollision)
{
if(rom.SHA512 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha512 == rom.SHA512 && f.Size == uSize);
if(file == null &&
rom.SHA384 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha384 == rom.SHA384 && f.Size == uSize);
if(file == null &&
rom.SHA256 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha256 == rom.SHA256 && f.Size == uSize);
if(file == null &&
rom.SHA1 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha1 == rom.SHA1 && f.Size == uSize);
if(file == null &&
rom.MD5 != null)
file = pendingFiles.FirstOrDefault(f => f.Md5 == rom.MD5 && f.Size == uSize);
if(file == null &&
rom.CRC != null)
file = pendingFiles.FirstOrDefault(f => f.Crc32 == rom.CRC && f.Size == uSize);
}
file ??=
Context.Singleton.Files.FirstOrDefault(f => ((rom.SHA512 != null && f.Sha512 == rom.SHA512) ||
(rom.SHA384 != null && f.Sha384 == rom.SHA384) ||
(rom.SHA256 != null && f.Sha256 == rom.SHA256) ||
(rom.SHA1 != null && f.Sha1 == rom.SHA1) ||
(rom.MD5 != null && f.Md5 == rom.MD5) ||
(rom.CRC != null && f.Crc32 == rom.CRC)) &&
f.Size == uSize);
if(file == null)
{
file = new DbFile
{
Crc32 = rom.CRC,
CreatedOn = DateTime.UtcNow,
Md5 = rom.MD5,
Sha1 = rom.SHA1,
Sha256 = rom.SHA256,
Sha384 = rom.SHA384,
Sha512 = rom.SHA512,
Size = uSize,
UpdatedOn = DateTime.UtcNow
};
2020-09-03 19:45:39 +01:00
newFiles.Add(file);
}
if(string.IsNullOrEmpty(file.Crc32) &&
!string.IsNullOrEmpty(rom.CRC))
{
file.Crc32 = rom.CRC;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Md5) &&
!string.IsNullOrEmpty(rom.MD5))
{
file.Md5 = rom.MD5;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha1) &&
!string.IsNullOrEmpty(rom.SHA1))
{
file.Sha1 = rom.SHA1;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha256) &&
!string.IsNullOrEmpty(rom.SHA256))
{
file.Sha256 = rom.SHA256;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha384) &&
!string.IsNullOrEmpty(rom.SHA384))
{
file.Sha384 = rom.SHA384;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha512) &&
!string.IsNullOrEmpty(rom.SHA512))
{
file.Sha512 = rom.SHA512;
file.UpdatedOn = DateTime.UtcNow;
}
2020-09-03 19:45:39 +01:00
newFilesByMachine.Add(new FileByMachine
{
File = file,
Machine = machine,
Name = rom.Name
});
if(hashCollision)
pendingFiles.Add(file);
else if(file.Sha512 != null)
pendingFilesBySha512[file.Sha512] = file;
else if(file.Sha384 != null)
pendingFilesBySha384[file.Sha384] = file;
else if(file.Sha256 != null)
pendingFilesBySha256[file.Sha256] = file;
else if(file.Sha1 != null)
pendingFilesBySha1[file.Sha1] = file;
else if(file.Md5 != null)
pendingFilesByMd5[file.Md5] = file;
else if(file.Crc32 != null)
pendingFilesByCrc[file.Crc32] = file;
position++;
}
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.SavingChangesToDatabase
});
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
2020-09-03 19:45:39 +01:00
Context.Singleton.Files.AddRange(newFiles);
Context.Singleton.FilesByMachines.AddRange(newFilesByMachine);
Context.Singleton.SaveChanges();
pendingFilesBySha512.Clear();
pendingFilesBySha384.Clear();
pendingFilesBySha256.Clear();
pendingFilesBySha1.Clear();
pendingFilesByMd5.Clear();
pendingFilesByCrc.Clear();
pendingFiles.Clear();
newFiles.Clear();
newFilesByMachine.Clear();
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs
{
Minimum = 0,
Maximum = disks.Count
});
SetMessage?.Invoke(this, new MessageEventArgs
{
2020-09-04 03:15:26 +01:00
Message = Localization.AddingDisks
});
position = 0;
Dictionary<string, DbDisk> pendingDisksBySha1 = new Dictionary<string, DbDisk>();
Dictionary<string, DbDisk> pendingDisksByMd5 = new Dictionary<string, DbDisk>();
List<DbDisk> newDisks = new List<DbDisk>();
List<DiskByMachine> newDisksByMachine = new List<DiskByMachine>();
foreach(Disk disk in disks)
{
SetProgress?.Invoke(this, new ProgressEventArgs
{
Value = position
});
if(!machines.TryGetValue(disk.Machine.Name, out Machine machine))
{
ErrorOccurred?.Invoke(this, new ErrorEventArgs
{
Message = Localization.FoundDiskWithoutMachine
});
return;
}
if(disk.MD5 == null &&
disk.SHA1 == null)
{
position++;
continue;
}
DbDisk dbDisk = null;
if(disk.SHA1 != null &&
dbDisk == null)
pendingDisksBySha1.TryGetValue(disk.SHA1, out dbDisk);
if(disk.MD5 != null &&
dbDisk == null)
pendingDisksByMd5.TryGetValue(disk.MD5, out dbDisk);
dbDisk ??= Context.Singleton.Disks.FirstOrDefault(f => (disk.SHA1 != null && f.Sha1 == disk.SHA1) ||
(disk.MD5 != null && f.Md5 == disk.MD5));
if(dbDisk == null)
{
dbDisk = new DbDisk
{
CreatedOn = DateTime.UtcNow,
Md5 = disk.MD5,
Sha1 = disk.SHA1,
UpdatedOn = DateTime.UtcNow
};
newDisks.Add(dbDisk);
}
if(string.IsNullOrEmpty(dbDisk.Md5) &&
!string.IsNullOrEmpty(disk.MD5))
{
dbDisk.Md5 = disk.MD5;
dbDisk.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(dbDisk.Sha1) &&
!string.IsNullOrEmpty(disk.SHA1))
{
dbDisk.Sha1 = disk.SHA1;
dbDisk.UpdatedOn = DateTime.UtcNow;
}
newDisksByMachine.Add(new DiskByMachine
{
Disk = dbDisk,
Machine = machine,
Name = disk.Name
});
if(dbDisk.Sha1 != null)
pendingDisksBySha1[dbDisk.Sha1] = dbDisk;
if(dbDisk.Md5 != null)
pendingDisksByMd5[dbDisk.Md5] = dbDisk;
position++;
}
SetMessage?.Invoke(this, new MessageEventArgs
{
Message = Localization.SavingChangesToDatabase
});
2020-09-04 03:15:26 +01:00
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
Context.Singleton.Disks.AddRange(newDisks);
Context.Singleton.DisksByMachines.AddRange(newDisksByMachine);
Context.Singleton.SaveChanges();
pendingDisksBySha1.Clear();
pendingDisksByMd5.Clear();
newDisks.Clear();
newDisksByMachine.Clear();
2020-08-22 04:40:39 +01:00
WorkFinished?.Invoke(this, System.EventArgs.Empty);
2020-08-24 01:57:15 +01:00
romSet = Context.Singleton.RomSets.Find(romSet.Id);
RomSetAdded?.Invoke(this, new RomSetEventArgs
{
RomSet = new RomSetModel
{
2020-09-04 02:21:54 +01:00
Id = romSet.Id,
Author = romSet.Author,
Comment = romSet.Comment,
Date = romSet.Date,
Description = romSet.Description,
Filename = romSet.Filename,
Homepage = romSet.Homepage,
Name = romSet.Name,
Sha384 = romSet.Sha384,
Version = romSet.Version,
TotalMachines = romSet.Machines.Count,
CompleteMachines =
2020-09-04 03:27:44 +01:00
romSet.Machines.Count(m => m.Files.Count > 0 && m.Disks.Count == 0 &&
m.Files.All(f => f.File.IsInRepo)) +
romSet.Machines.Count(m => m.Disks.Count > 0 && m.Files.Count == 0 &&
m.Disks.All(f => f.Disk.IsInRepo)) +
romSet.Machines.Count(m => m.Files.Count > 0 && m.Disks.Count > 0 &&
m.Files.All(f => f.File.IsInRepo) &&
2020-09-04 02:21:54 +01:00
m.Disks.All(f => f.Disk.IsInRepo)),
IncompleteMachines =
2020-09-04 03:27:44 +01:00
romSet.Machines.Count(m => m.Files.Count > 0 && m.Disks.Count == 0 &&
m.Files.Any(f => !f.File.IsInRepo)) +
romSet.Machines.Count(m => m.Disks.Count > 0 && m.Files.Count == 0 &&
m.Disks.Any(f => !f.Disk.IsInRepo)) +
romSet.Machines.Count(m => m.Files.Count > 0 && m.Disks.Count > 0 &&
(m.Files.Any(f => !f.File.IsInRepo) ||
m.Disks.Any(f => !f.Disk.IsInRepo))),
2020-09-04 02:21:54 +01:00
TotalRoms = romSet.Machines.Sum(m => m.Files.Count) + romSet.Machines.Sum(m => m.Disks.Count),
HaveRoms = romSet.Machines.Sum(m => m.Files.Count(f => f.File.IsInRepo)) +
romSet.Machines.Sum(m => m.Disks.Count(f => f.Disk.IsInRepo)),
MissRoms = romSet.Machines.Sum(m => m.Files.Count(f => !f.File.IsInRepo)) +
romSet.Machines.Sum(m => m.Disks.Count(f => !f.Disk.IsInRepo))
2020-08-24 01:57:15 +01:00
}
});
2020-08-22 04:40:39 +01:00
}
catch(Exception e)
{
if(Debugger.IsAttached)
throw;
ErrorOccurred?.Invoke(this, new ErrorEventArgs
{
2020-08-30 03:00:14 +01:00
Message = Localization.UnhandledException
2020-08-22 04:40:39 +01:00
});
}
}
// TODO: Cancel and get back
2020-08-22 04:40:39 +01:00
public void Abort() => _aborted = true;
public event EventHandler SetIndeterminateProgress;
public event EventHandler WorkFinished;
public event EventHandler<ErrorEventArgs> ErrorOccurred;
public event EventHandler<ProgressBoundsEventArgs> SetProgressBounds;
public event EventHandler<ProgressEventArgs> SetProgress;
public event EventHandler<MessageEventArgs> SetMessage;
public event EventHandler<RomSetEventArgs> RomSetAdded;
2020-08-22 04:40:39 +01:00
}
}