claunia/romrepomgr
1
0
Fork 0
mirror of https://github.com/claunia/romrepomgr.git synced 2025-12-16 19:24:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add files information from DAT to database.

Browse Source
This commit is contained in:
Natalia Portillo
2020-08-22 17:14:52 +01:00
parent 4b166c2e4a
commit 131b32a3dc
9 changed files with 667 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
RomRepoMgr.Core/Workers/DatImporter.cs
View File

@@ -34,7 +34,9 @@ using RomRepoMgr.Core.Models;
using RomRepoMgr.Database;
using RomRepoMgr.Database.Models;
using SabreTools.Library.DatFiles;
using SabreTools.Library.DatItems;
using ErrorEventArgs = RomRepoMgr.Core.EventArgs.ErrorEventArgs;
using Machine = RomRepoMgr.Database.Models.Machine;
namespace RomRepoMgr.Core.Workers
{
@@ -61,10 +63,7 @@ namespace RomRepoMgr.Core.Workers
Message = "Parsing DAT file..."
});
DateTime start = DateTime.UtcNow;
var datFile = DatFile.CreateAndParse(_datPath);
DateTime end = DateTime.UtcNow;
double elapsed = (end - start).TotalSeconds;
var datFile = DatFile.CreateAndParse(_datPath);
SetMessage?.Invoke(this, new MessageEventArgs
{
@@ -194,6 +193,285 @@ namespace RomRepoMgr.Core.Workers
Context.Singleton.SaveChanges();
SetMessage?.Invoke(this, new MessageEventArgs
{
Message = "Retrieving ROMs and disks..."
});
List<Rom> roms = new List<Rom>();
List<Disk> disks = new List<Disk>();
foreach(List<DatItem> values in datFile.Items.Values)
{
foreach(DatItem item in values)
{
switch(item)
{
case Rom rom:
roms.Add(rom);
continue;
case Disk disk:
disks.Add(disk);
continue;
}
}
}
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs
{
Minimum = 0,
Maximum = roms.Count
});
SetMessage?.Invoke(this, new MessageEventArgs
{
Message = "Adding ROMs..."
});
position = 0;
Dictionary<string, DbFile> pendingFilesBySha512 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesBySha384 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesBySha256 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesBySha1 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesByMd5 = new Dictionary<string, DbFile>();
Dictionary<string, DbFile> pendingFilesByCrc = new Dictionary<string, DbFile>();
List<DbFile> pendingFiles = new List<DbFile>();
foreach(Rom rom in roms)
{
bool hashCollision = false;
SetProgress?.Invoke(this, new ProgressEventArgs
{
Value = position
});
if(!machines.TryGetValue(rom.Machine.Name, out Machine machine))
{
ErrorOccurred?.Invoke(this, new ErrorEventArgs
{
Message = "Found a ROM with an unknown machine, this should not happen."
});
return;
}
ulong uSize = (ulong)rom.Size;
DbFile file = null;
if(rom.SHA512 != null)
if(pendingFilesBySha512.TryGetValue(rom.SHA512, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.SHA384 != null &&
file == null)
if(pendingFilesBySha384.TryGetValue(rom.SHA384, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.SHA256 != null &&
file == null)
if(pendingFilesBySha256.TryGetValue(rom.SHA256, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.SHA1 != null &&
file == null)
if(pendingFilesBySha1.TryGetValue(rom.SHA1, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.MD5 != null &&
file == null)
if(pendingFilesByMd5.TryGetValue(rom.MD5, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(rom.CRC != null &&
file == null)
if(pendingFilesByCrc.TryGetValue(rom.CRC, out file))
if(file.Size != uSize)
{
hashCollision = true;
file = null;
}
if(file == null && hashCollision)
{
if(rom.SHA512 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha512 == rom.SHA512 && f.Size == uSize);
if(file == null &&
rom.SHA384 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha384 == rom.SHA384 && f.Size == uSize);
if(file == null &&
rom.SHA256 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha256 == rom.SHA256 && f.Size == uSize);
if(file == null &&
rom.SHA1 != null)
file = pendingFiles.FirstOrDefault(f => f.Sha1 == rom.SHA1 && f.Size == uSize);
if(file == null &&
rom.MD5 != null)
file = pendingFiles.FirstOrDefault(f => f.Md5 == rom.MD5 && f.Size == uSize);
if(file == null &&
rom.CRC != null)
file = pendingFiles.FirstOrDefault(f => f.Crc32 == rom.CRC && f.Size == uSize);
}
if(file == null &&
rom.SHA512 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha512 == rom.SHA512 && f.Size == uSize);
if(file == null &&
rom.SHA384 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha384 == rom.SHA384 && f.Size == uSize);
if(file == null &&
rom.SHA256 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha256 == rom.SHA256 && f.Size == uSize);
if(file == null &&
rom.SHA1 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Sha1 == rom.SHA1 && f.Size == uSize);
if(file == null &&
rom.MD5 != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Md5 == rom.MD5 && f.Size == uSize);
if(file == null &&
rom.CRC != null)
file = Context.Singleton.Files.FirstOrDefault(f => f.Crc32 == rom.CRC && f.Size == uSize);
if(file == null)
{
file = new DbFile
{
Crc32 = rom.CRC,
CreatedOn = DateTime.UtcNow,
Md5 = rom.MD5,
Sha1 = rom.SHA1,
Sha256 = rom.SHA256,
Sha384 = rom.SHA384,
Sha512 = rom.SHA512,
Size = uSize,
UpdatedOn = DateTime.UtcNow
};
Context.Singleton.Files.Add(file);
}
if(string.IsNullOrEmpty(file.Crc32) &&
!string.IsNullOrEmpty(rom.CRC))
{
file.Crc32 = rom.CRC;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Md5) &&
!string.IsNullOrEmpty(rom.MD5))
{
file.Md5 = rom.MD5;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha1) &&
!string.IsNullOrEmpty(rom.SHA1))
{
file.Sha1 = rom.SHA1;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha256) &&
!string.IsNullOrEmpty(rom.SHA256))
{
file.Sha256 = rom.SHA256;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha384) &&
!string.IsNullOrEmpty(rom.SHA384))
{
file.Sha384 = rom.SHA384;
file.UpdatedOn = DateTime.UtcNow;
}
if(string.IsNullOrEmpty(file.Sha512) &&
!string.IsNullOrEmpty(rom.SHA512))
{
file.Sha512 = rom.SHA512;
file.UpdatedOn = DateTime.UtcNow;
}
Context.Singleton.FilesByMachines.Add(new FileByMachine
{
File = file,
Machine = machine,
Name = rom.Name
});
if(hashCollision)
pendingFiles.Add(file);
else if(file.Sha512 != null)
pendingFilesBySha512[file.Sha512] = file;
else if(file.Sha384 != null)
pendingFilesBySha384[file.Sha384] = file;
else if(file.Sha256 != null)
pendingFilesBySha256[file.Sha256] = file;
else if(file.Sha1 != null)
pendingFilesBySha1[file.Sha1] = file;
else if(file.Md5 != null)
pendingFilesByMd5[file.Md5] = file;
else if(file.Crc32 != null)
pendingFilesByCrc[file.Crc32] = file;
position++;
}
SetMessage?.Invoke(this, new MessageEventArgs
{
Message = "Saving changes to database..."
});
SetIndeterminateProgress?.Invoke(this, System.EventArgs.Empty);
Context.Singleton.SaveChanges();
SetProgressBounds?.Invoke(this, new ProgressBoundsEventArgs
{
Minimum = 0,
Maximum = disks.Count
});
// TODO: Support CHDs
SetMessage?.Invoke(this, new MessageEventArgs
{
Message = "Adding disks..."
});
WorkFinished?.Invoke(this, System.EventArgs.Empty);
}
catch(Exception e)
@@ -208,6 +486,7 @@ namespace RomRepoMgr.Core.Workers
}
}
// TODO: Cancel and get back
public void Abort() => _aborted = true;
public event EventHandler SetIndeterminateProgress;

16
RomRepoMgr.Database/Context.cs
View File

@@ -52,9 +52,10 @@ namespace RomRepoMgr.Database
}
}
public DbSet<DbFile> Files { get; set; }
public DbSet<RomSet> RomSets { get; set; }
public DbSet<Machine> Machines { get; set; }
public DbSet<DbFile> Files { get; set; }
public DbSet<RomSet> RomSets { get; set; }
public DbSet<Machine> Machines { get; set; }
public DbSet<FileByMachine> FilesByMachines { get; set; }
public static Context Create(string dbPath)
{
@@ -110,6 +111,15 @@ namespace RomRepoMgr.Database
entity.HasOne(e => e.RomSet).WithMany(e => e.Machines).OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<FileByMachine>(entity =>
{
entity.HasIndex(e => e.Name);
entity.HasOne(e => e.Machine).WithMany(e => e.Files).OnDelete(DeleteBehavior.Cascade);
entity.HasOne(e => e.File).WithMany(e => e.Machines).OnDelete(DeleteBehavior.Cascade);
});
}
}
}

223
RomRepoMgr.Database/Migrations/20200822141456_AddFilesByMachines.Designer.cs generated Normal file
View File

@@ -0,0 +1,223 @@
// <auto-generated />
using System;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
using RomRepoMgr.Database;
namespace RomRepoMgr.Database.Migrations
{
[DbContext(typeof(Context))]
[Migration("20200822141456_AddFilesByMachines")]
partial class AddFilesByMachines
{
protected override void BuildTargetModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder
.HasAnnotation("ProductVersion", "3.1.7");
modelBuilder.Entity("RomRepoMgr.Database.Models.DbFile", b =>
{
b.Property<ulong>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Crc32")
.HasColumnType("TEXT")
.HasMaxLength(8);
b.Property<DateTime>("CreatedOn")
.HasColumnType("TEXT");
b.Property<string>("Md5")
.HasColumnType("TEXT")
.HasMaxLength(32);
b.Property<string>("Sha1")
.HasColumnType("TEXT")
.HasMaxLength(40);
b.Property<string>("Sha256")
.HasColumnType("TEXT")
.HasMaxLength(64);
b.Property<string>("Sha384")
.HasColumnType("TEXT")
.HasMaxLength(96);
b.Property<string>("Sha512")
.HasColumnType("TEXT")
.HasMaxLength(128);
b.Property<ulong>("Size")
.HasColumnType("INTEGER");
b.Property<DateTime>("UpdatedOn")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("Crc32");
b.HasIndex("Sha1");
b.HasIndex("Sha256");
b.HasIndex("Sha384");
b.HasIndex("Sha512");
b.HasIndex("Size");
b.ToTable("Files");
});
modelBuilder.Entity("RomRepoMgr.Database.Models.FileByMachine", b =>
{
b.Property<ulong>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<ulong>("FileId")
.HasColumnType("INTEGER");
b.Property<ulong>("MachineId")
.HasColumnType("INTEGER");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("FileId");
b.HasIndex("MachineId");
b.HasIndex("Name");
b.ToTable("FilesByMachines");
});
modelBuilder.Entity("RomRepoMgr.Database.Models.Machine", b =>
{
b.Property<ulong>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<DateTime>("CreatedOn")
.HasColumnType("TEXT");
b.Property<string>("Name")
.IsRequired()
.HasColumnType("TEXT");
b.Property<long>("RomSetId")
.HasColumnType("INTEGER");
b.Property<DateTime>("UpdatedOn")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("Name");
b.HasIndex("RomSetId");
b.ToTable("Machines");
});
modelBuilder.Entity("RomRepoMgr.Database.Models.RomSet", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Author")
.HasColumnType("TEXT");
b.Property<string>("Comment")
.HasColumnType("TEXT");
b.Property<DateTime>("CreatedOn")
.HasColumnType("TEXT");
b.Property<string>("Date")
.HasColumnType("TEXT");
b.Property<string>("Description")
.HasColumnType("TEXT");
b.Property<string>("Filename")
.IsRequired()
.HasColumnType("TEXT");
b.Property<string>("Homepage")
.HasColumnType("TEXT");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<string>("Sha384")
.IsRequired()
.HasColumnType("TEXT")
.HasMaxLength(96);
b.Property<DateTime>("UpdatedOn")
.HasColumnType("TEXT");
b.Property<string>("Version")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("Author");
b.HasIndex("Comment");
b.HasIndex("Date");
b.HasIndex("Description");
b.HasIndex("Filename");
b.HasIndex("Homepage");
b.HasIndex("Name");
b.HasIndex("Sha384");
b.HasIndex("Version");
b.ToTable("RomSets");
});
modelBuilder.Entity("RomRepoMgr.Database.Models.FileByMachine", b =>
{
b.HasOne("RomRepoMgr.Database.Models.DbFile", "File")
.WithMany("Machines")
.HasForeignKey("FileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("RomRepoMgr.Database.Models.Machine", "Machine")
.WithMany("Files")
.HasForeignKey("MachineId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("RomRepoMgr.Database.Models.Machine", b =>
{
b.HasOne("RomRepoMgr.Database.Models.RomSet", "RomSet")
.WithMany("Machines")
.HasForeignKey("RomSetId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
#pragma warning restore 612, 618
}
}
}

61
RomRepoMgr.Database/Migrations/20200822141456_AddFilesByMachines.cs Normal file
View File

@@ -0,0 +1,61 @@
/******************************************************************************
// RomRepoMgr - ROM repository manager
// ----------------------------------------------------------------------------
//
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// --[ License ] --------------------------------------------------------------
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2020 Natalia Portillo
*******************************************************************************/
using Microsoft.EntityFrameworkCore.Migrations;
namespace RomRepoMgr.Database.Migrations
{
public partial class AddFilesByMachines : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable("FilesByMachines", table => new
{
Id = table.Column<ulong>(nullable: false).Annotation("Sqlite:Autoincrement", true),
FileId = table.Column<ulong>(nullable: false),
MachineId = table.Column<ulong>(nullable: false),
Name = table.Column<string>(nullable: false)
}, constraints: table =>
{
table.PrimaryKey("PK_FilesByMachines", x => x.Id);
table.ForeignKey("FK_FilesByMachines_Files_FileId", x => x.FileId, "Files", "Id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey("FK_FilesByMachines_Machines_MachineId", x => x.MachineId, "Machines", "Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex("IX_FilesByMachines_FileId", "FilesByMachines", "FileId");
migrationBuilder.CreateIndex("IX_FilesByMachines_MachineId", "FilesByMachines", "MachineId");
migrationBuilder.CreateIndex("IX_FilesByMachines_Name", "FilesByMachines", "Name");
}
protected override void Down(MigrationBuilder migrationBuilder) =>
migrationBuilder.DropTable("FilesByMachines");
}
}

30
RomRepoMgr.Database/Migrations/ContextModelSnapshot.cs
View File

@@ -53,6 +53,27 @@ namespace RomRepoMgr.Database.Migrations
b.ToTable("Files");
});
modelBuilder.Entity("RomRepoMgr.Database.Models.FileByMachine", b =>
{
b.Property<ulong>("Id").ValueGeneratedOnAdd().HasColumnType("INTEGER");
b.Property<ulong>("FileId").HasColumnType("INTEGER");
b.Property<ulong>("MachineId").HasColumnType("INTEGER");
b.Property<string>("Name").IsRequired().HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("FileId");
b.HasIndex("MachineId");
b.HasIndex("Name");
b.ToTable("FilesByMachines");
});
modelBuilder.Entity("RomRepoMgr.Database.Models.Machine", b =>
{
b.Property<ulong>("Id").ValueGeneratedOnAdd().HasColumnType("INTEGER");
@@ -123,6 +144,15 @@ namespace RomRepoMgr.Database.Migrations
b.ToTable("RomSets");
});
modelBuilder.Entity("RomRepoMgr.Database.Models.FileByMachine", b =>
{
b.HasOne("RomRepoMgr.Database.Models.DbFile", "File").WithMany("Machines").HasForeignKey("FileId").
OnDelete(DeleteBehavior.Cascade).IsRequired();
b.HasOne("RomRepoMgr.Database.Models.Machine", "Machine").WithMany("Files").HasForeignKey("MachineId").
OnDelete(DeleteBehavior.Cascade).IsRequired();
});
modelBuilder.Entity("RomRepoMgr.Database.Models.Machine", b =>
{
b.HasOne("RomRepoMgr.Database.Models.RomSet", "RomSet").WithMany("Machines").HasForeignKey("RomSetId").

4
RomRepoMgr.Database/Models/DbFile.cs
View File

@@ -23,6 +23,7 @@
// Copyright © 2020 Natalia Portillo
*******************************************************************************/
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
namespace RomRepoMgr.Database.Models
@@ -42,6 +43,7 @@ namespace RomRepoMgr.Database.Models
[StringLength(96, MinimumLength = 96)]
public string Sha384 { get; set; }
[StringLength(128, MinimumLength = 128)]
public string Sha512 { get; set; }
public string Sha512 { get; set; }
public virtual ICollection<FileByMachine> Machines { get; set; }
}
}

41
RomRepoMgr.Database/Models/FileByMachine.cs Normal file
View File

@@ -0,0 +1,41 @@
/******************************************************************************
// RomRepoMgr - ROM repository manager
// ----------------------------------------------------------------------------
//
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// --[ License ] --------------------------------------------------------------
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2020 Natalia Portillo
*******************************************************************************/
using System.ComponentModel.DataAnnotations;
namespace RomRepoMgr.Database.Models
{
public class FileByMachine
{
[Key]
public ulong Id { get; set; }
[Required]
public virtual DbFile File { get; set; }
[Required]
public virtual Machine Machine { get; set; }
[Required]
public string Name { get; set; }
}
}

4
RomRepoMgr.Database/Models/Machine.cs
View File

@@ -23,6 +23,7 @@
// Copyright © 2020 Natalia Portillo
*******************************************************************************/
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
namespace RomRepoMgr.Database.Models
@@ -32,6 +33,7 @@ namespace RomRepoMgr.Database.Models
[Required]
public string Name { get; set; }
[Required]
public virtual RomSet RomSet { get; set; }
public virtual RomSet RomSet { get; set; }
public virtual ICollection<FileByMachine> Files { get; set; }
}
}

14
RomRepoMgr/ViewModels/ImportDatViewModel.cs
View File

@@ -125,7 +125,12 @@ namespace RomRepoMgr.ViewModels
public string CloseLabel => "Close";
public ReactiveCommand<Unit, Unit> CloseCommand { get; }
void OnWorkerOnWorkFinished(object sender, EventArgs args) => Dispatcher.UIThread.Post(() => CanClose = true);
void OnWorkerOnWorkFinished(object sender, EventArgs args) => Dispatcher.UIThread.Post(() =>
{
StatusMessage = "Finished";
ProgressVisible = false;
CanClose = true;
});
void OnWorkerOnSetProgressBounds(object sender, ProgressBoundsEventArgs args) => Dispatcher.UIThread.Post(() =>
{
@@ -145,9 +150,10 @@ namespace RomRepoMgr.ViewModels
void OnWorkerOnErrorOccurred(object sender, ErrorEventArgs args) => Dispatcher.UIThread.Post(() =>
{
ErrorMessage = args.Message;
ErrorVisible = true;
CanClose = true;
ErrorMessage = args.Message;
ProgressVisible = false;
ErrorVisible = true;
CanClose = true;
});
void ExecuteCloseCommand() => _view.Close();