Add MobyGames import services and country matching functionality

- Implemented CountryMatcher for matching country names with aliases.
- Created GameAssembler to parse and assemble game data from MobyGames.
- Developed ImportService to handle the import process of games, including validation and state management.
- Added PersonMatcher and PlatformMatcher for matching and creating persons and platforms.
- Introduced SoundexHelper for phonetic matching of names.
- Built SourceDatabaseService for database interactions with MobyGames data.
- Established StateService for tracking import states and managing game processing statuses.
- Updated solution file to include MobyGames project.
This commit is contained in:
2026-04-27 21:37:45 +01:00
parent e2fc429401
commit 681fe830f2
38 changed files with 20235 additions and 5 deletions

7
.gitignore vendored
View File

@@ -281,7 +281,7 @@ ClientBin/
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
@@ -377,7 +377,7 @@ __pycache__/
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
@@ -453,4 +453,5 @@ Marechai/wwwroot/assets/**/*.tiff
Marechai/wwwroot/assets/**/*.webp
appsettings.Development.json
appsettings.Production.json
*.sql
*.sql
Marechai.MobyGames/appsettings.json

View File

@@ -21,6 +21,9 @@
<PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.11"/>
<PackageVersion Include="Pomelo.EntityFrameworkCore.MySql" Version="9.0.0"/>
<PackageVersion Include="Pomelo.EntityFrameworkCore.MySql.Json.Microsoft" Version="9.0.0"/>
<!-- Unique to Marechai.MobyGames -->
<PackageVersion Include="HtmlAgilityPack" Version="1.12.1"/>
<PackageVersion Include="MySqlConnector" Version="2.4.0"/>
<!-- Build infrastructure -->
<PackageVersion Include="Packaging.Targets" Version="0.1.232"/>
<!-- Unique to Marechai.Server.csproj -->

View File

@@ -795,7 +795,11 @@ public enum ProductCodeIssuer : byte
Ubisoft = 5,
Bethesda = 6,
Sega = 7,
Amazon = 8
Amazon = 8,
[Display(Name = "PlayStation Network")]
PSN = 9,
eBay = 10,
Other = 255
}
public enum BarcodeType : byte
@@ -865,4 +869,31 @@ public enum UnM49Type : byte
SubRegion = 2,
[Display(Name = "Country")]
Country = 3
}
public enum SoftwareGenreType : byte
{
[Display(Name = "Genre")]
Genre = 0,
[Display(Name = "Perspective")]
Perspective = 1,
[Display(Name = "Gameplay")]
Gameplay = 2,
[Display(Name = "Setting")]
Setting = 3
}
public enum MobyGamesImportStatus : byte
{
Pending = 0,
Imported = 1,
Rejected = 2,
Failed = 3
}
public enum MobyGamesRejectionReview : byte
{
Pending = 0,
Confirmed = 1,
Overridden = 2
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,243 @@
using System;
using Microsoft.EntityFrameworkCore.Metadata;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Marechai.Database.Migrations
{
/// <inheritdoc />
public partial class AddMobyGamesImportModels : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "MobyGamesImportStates",
columns: table => new
{
Id = table.Column<long>(type: "bigint", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
MobyGameId = table.Column<int>(type: "int", nullable: false),
Status = table.Column<byte>(type: "tinyint unsigned", nullable: false),
ErrorMessage = table.Column<string>(type: "varchar(1024)", maxLength: 1024, nullable: true)
.Annotation("MySql:CharSet", "utf8mb4"),
ProcessedOn = table.Column<DateTime>(type: "datetime(6)", nullable: true),
BatchNumber = table.Column<int>(type: "int", nullable: false),
SoftwareId = table.Column<ulong>(type: "bigint unsigned", nullable: true),
CreatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
UpdatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.ComputedColumn)
},
constraints: table =>
{
table.PrimaryKey("PK_MobyGamesImportStates", x => x.Id);
})
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateTable(
name: "MobyGamesRejections",
columns: table => new
{
Id = table.Column<long>(type: "bigint", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
MobyGameId = table.Column<int>(type: "int", nullable: false),
GameName = table.Column<string>(type: "varchar(512)", maxLength: 512, nullable: false)
.Annotation("MySql:CharSet", "utf8mb4"),
Reason = table.Column<string>(type: "varchar(1024)", maxLength: 1024, nullable: false)
.Annotation("MySql:CharSet", "utf8mb4"),
RejectedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false),
ReviewedOn = table.Column<DateTime>(type: "datetime(6)", nullable: true),
ReviewAction = table.Column<byte>(type: "tinyint unsigned", nullable: false),
CreatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
UpdatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.ComputedColumn)
},
constraints: table =>
{
table.PrimaryKey("PK_MobyGamesRejections", x => x.Id);
})
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateTable(
name: "PeopleBySoftware",
columns: table => new
{
Id = table.Column<long>(type: "bigint", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
SoftwareId = table.Column<ulong>(type: "bigint unsigned", nullable: false),
PersonId = table.Column<int>(type: "int", nullable: false),
Role = table.Column<string>(type: "varchar(128)", maxLength: 128, nullable: false)
.Annotation("MySql:CharSet", "utf8mb4"),
CreatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
UpdatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.ComputedColumn)
},
constraints: table =>
{
table.PrimaryKey("PK_PeopleBySoftware", x => x.Id);
table.ForeignKey(
name: "FK_PeopleBySoftware_People_PersonId",
column: x => x.PersonId,
principalTable: "People",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "FK_PeopleBySoftware_Softwares_SoftwareId",
column: x => x.SoftwareId,
principalTable: "Softwares",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
})
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateTable(
name: "SoftwareAttributes",
columns: table => new
{
Id = table.Column<long>(type: "bigint", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
SoftwareReleaseId = table.Column<ulong>(type: "bigint unsigned", nullable: false),
Category = table.Column<string>(type: "varchar(64)", maxLength: 64, nullable: false)
.Annotation("MySql:CharSet", "utf8mb4"),
Key = table.Column<string>(type: "varchar(128)", maxLength: 128, nullable: false)
.Annotation("MySql:CharSet", "utf8mb4"),
Value = table.Column<string>(type: "varchar(512)", maxLength: 512, nullable: false)
.Annotation("MySql:CharSet", "utf8mb4"),
CreatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
UpdatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.ComputedColumn)
},
constraints: table =>
{
table.PrimaryKey("PK_SoftwareAttributes", x => x.Id);
table.ForeignKey(
name: "FK_SoftwareAttributes_SoftwareReleases_SoftwareReleaseId",
column: x => x.SoftwareReleaseId,
principalTable: "SoftwareReleases",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
})
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateTable(
name: "SoftwareGenres",
columns: table => new
{
Id = table.Column<int>(type: "int", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
Name = table.Column<string>(type: "varchar(128)", maxLength: 128, nullable: false)
.Annotation("MySql:CharSet", "utf8mb4"),
Type = table.Column<byte>(type: "tinyint unsigned", nullable: false),
CreatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.IdentityColumn),
UpdatedOn = table.Column<DateTime>(type: "datetime(6)", nullable: false)
.Annotation("MySql:ValueGenerationStrategy", MySqlValueGenerationStrategy.ComputedColumn)
},
constraints: table =>
{
table.PrimaryKey("PK_SoftwareGenres", x => x.Id);
})
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateTable(
name: "GenresBySoftware",
columns: table => new
{
SoftwareId = table.Column<ulong>(type: "bigint unsigned", nullable: false),
GenreId = table.Column<int>(type: "int", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_GenresBySoftware", x => new { x.SoftwareId, x.GenreId });
table.ForeignKey(
name: "FK_GenresBySoftware_SoftwareGenres_GenreId",
column: x => x.GenreId,
principalTable: "SoftwareGenres",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "FK_GenresBySoftware_Softwares_SoftwareId",
column: x => x.SoftwareId,
principalTable: "Softwares",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
})
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.CreateIndex(
name: "IX_GenresBySoftware_GenreId",
table: "GenresBySoftware",
column: "GenreId");
migrationBuilder.CreateIndex(
name: "IX_MobyGamesImportStates_MobyGameId",
table: "MobyGamesImportStates",
column: "MobyGameId",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_MobyGamesImportStates_Status",
table: "MobyGamesImportStates",
column: "Status");
migrationBuilder.CreateIndex(
name: "IX_MobyGamesRejections_MobyGameId",
table: "MobyGamesRejections",
column: "MobyGameId");
migrationBuilder.CreateIndex(
name: "IX_MobyGamesRejections_ReviewAction",
table: "MobyGamesRejections",
column: "ReviewAction");
migrationBuilder.CreateIndex(
name: "IX_PeopleBySoftware_PersonId",
table: "PeopleBySoftware",
column: "PersonId");
migrationBuilder.CreateIndex(
name: "IX_PeopleBySoftware_SoftwareId_PersonId_Role",
table: "PeopleBySoftware",
columns: new[] { "SoftwareId", "PersonId", "Role" },
unique: true);
migrationBuilder.CreateIndex(
name: "IX_SoftwareAttributes_SoftwareReleaseId_Category_Key",
table: "SoftwareAttributes",
columns: new[] { "SoftwareReleaseId", "Category", "Key" });
migrationBuilder.CreateIndex(
name: "IX_SoftwareGenres_Name_Type",
table: "SoftwareGenres",
columns: new[] { "Name", "Type" },
unique: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "GenresBySoftware");
migrationBuilder.DropTable(
name: "MobyGamesImportStates");
migrationBuilder.DropTable(
name: "MobyGamesRejections");
migrationBuilder.DropTable(
name: "PeopleBySoftware");
migrationBuilder.DropTable(
name: "SoftwareAttributes");
migrationBuilder.DropTable(
name: "SoftwareGenres");
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,58 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Marechai.Database.Migrations
{
/// <inheritdoc />
public partial class AlterMobyGameIdToString : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AlterColumn<string>(
name: "MobyGameId",
table: "MobyGamesRejections",
type: "varchar(64)",
maxLength: 64,
nullable: false,
oldClrType: typeof(int),
oldType: "int")
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<string>(
name: "MobyGameId",
table: "MobyGamesImportStates",
type: "varchar(64)",
maxLength: 64,
nullable: false,
oldClrType: typeof(int),
oldType: "int")
.Annotation("MySql:CharSet", "utf8mb4");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AlterColumn<int>(
name: "MobyGameId",
table: "MobyGamesRejections",
type: "int",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(64)",
oldMaxLength: 64)
.OldAnnotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AlterColumn<int>(
name: "MobyGameId",
table: "MobyGamesImportStates",
type: "int",
nullable: false,
oldClrType: typeof(string),
oldType: "varchar(64)",
oldMaxLength: 64)
.OldAnnotation("MySql:CharSet", "utf8mb4");
}
}
}

View File

@@ -2182,6 +2182,21 @@ namespace Marechai.Database.Migrations
b.ToTable("forbidden", (string)null);
});
modelBuilder.Entity("Marechai.Database.Models.GenreBySoftware", b =>
{
b.Property<ulong>("SoftwareId")
.HasColumnType("bigint unsigned");
b.Property<int>("GenreId")
.HasColumnType("int");
b.HasKey("SoftwareId", "GenreId");
b.HasIndex("GenreId");
b.ToTable("GenresBySoftware");
});
modelBuilder.Entity("Marechai.Database.Models.Gpu", b =>
{
b.Property<int>("Id")
@@ -4211,6 +4226,110 @@ namespace Marechai.Database.Migrations
b.ToTable("MinimumGpuBySoftwareRelease");
});
modelBuilder.Entity("Marechai.Database.Models.MobyGamesImportState", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<long>("Id"));
b.Property<int>("BatchNumber")
.HasColumnType("int");
b.Property<DateTime>("CreatedOn")
.ValueGeneratedOnAdd()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
b.Property<string>("ErrorMessage")
.HasMaxLength(1024)
.HasColumnType("varchar(1024)");
b.Property<string>("MobyGameId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<DateTime?>("ProcessedOn")
.HasColumnType("datetime(6)");
b.Property<ulong?>("SoftwareId")
.HasColumnType("bigint unsigned");
b.Property<byte>("Status")
.HasColumnType("tinyint unsigned");
b.Property<DateTime>("UpdatedOn")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlComputedColumn(b.Property<DateTime>("UpdatedOn"));
b.HasKey("Id");
b.HasIndex("MobyGameId")
.IsUnique();
b.HasIndex("Status");
b.ToTable("MobyGamesImportStates");
});
modelBuilder.Entity("Marechai.Database.Models.MobyGamesRejection", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<long>("Id"));
b.Property<DateTime>("CreatedOn")
.ValueGeneratedOnAdd()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
b.Property<string>("GameName")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("varchar(512)");
b.Property<string>("MobyGameId")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<string>("Reason")
.IsRequired()
.HasMaxLength(1024)
.HasColumnType("varchar(1024)");
b.Property<DateTime>("RejectedOn")
.HasColumnType("datetime(6)");
b.Property<byte>("ReviewAction")
.HasColumnType("tinyint unsigned");
b.Property<DateTime?>("ReviewedOn")
.HasColumnType("datetime(6)");
b.Property<DateTime>("UpdatedOn")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlComputedColumn(b.Property<DateTime>("UpdatedOn"));
b.HasKey("Id");
b.HasIndex("MobyGameId");
b.HasIndex("ReviewAction");
b.ToTable("MobyGamesRejections");
});
modelBuilder.Entity("Marechai.Database.Models.MoneyDonation", b =>
{
b.Property<int>("Id")
@@ -4584,6 +4703,47 @@ namespace Marechai.Database.Migrations
b.ToTable("PeopleByMagazines");
});
modelBuilder.Entity("Marechai.Database.Models.PeopleBySoftware", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<long>("Id"));
b.Property<DateTime>("CreatedOn")
.ValueGeneratedOnAdd()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
b.Property<int>("PersonId")
.HasColumnType("int");
b.Property<string>("Role")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("varchar(128)");
b.Property<ulong>("SoftwareId")
.HasColumnType("bigint unsigned");
b.Property<DateTime>("UpdatedOn")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlComputedColumn(b.Property<DateTime>("UpdatedOn"));
b.HasKey("Id");
b.HasIndex("PersonId");
b.HasIndex("SoftwareId", "PersonId", "Role")
.IsUnique();
b.ToTable("PeopleBySoftware");
});
modelBuilder.Entity("Marechai.Database.Models.Person", b =>
{
b.Property<int>("Id")
@@ -5227,6 +5387,51 @@ namespace Marechai.Database.Migrations
b.ToTable("Softwares");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwareAttribute", b =>
{
b.Property<long>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("bigint");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<long>("Id"));
b.Property<string>("Category")
.IsRequired()
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<DateTime>("CreatedOn")
.ValueGeneratedOnAdd()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
b.Property<string>("Key")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("varchar(128)");
b.Property<ulong>("SoftwareReleaseId")
.HasColumnType("bigint unsigned");
b.Property<DateTime>("UpdatedOn")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlComputedColumn(b.Property<DateTime>("UpdatedOn"));
b.Property<string>("Value")
.IsRequired()
.HasMaxLength(512)
.HasColumnType("varchar(512)");
b.HasKey("Id");
b.HasIndex("SoftwareReleaseId", "Category", "Key");
b.ToTable("SoftwareAttributes");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwareBarcode", b =>
{
b.Property<ulong>("Id")
@@ -5398,6 +5603,42 @@ namespace Marechai.Database.Migrations
b.ToTable("SoftwareFamilies");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwareGenre", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("int");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<int>("Id"));
b.Property<DateTime>("CreatedOn")
.ValueGeneratedOnAdd()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
b.Property<string>("Name")
.IsRequired()
.HasMaxLength(128)
.HasColumnType("varchar(128)");
b.Property<byte>("Type")
.HasColumnType("tinyint unsigned");
b.Property<DateTime>("UpdatedOn")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlComputedColumn(b.Property<DateTime>("UpdatedOn"));
b.HasKey("Id");
b.HasIndex("Name", "Type")
.IsUnique();
b.ToTable("SoftwareGenres");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwareOSCompatibility", b =>
{
b.Property<ulong>("SoftwareVersionId")
@@ -6785,6 +7026,25 @@ namespace Marechai.Database.Migrations
b.Navigation("MediaDumpFileImage");
});
modelBuilder.Entity("Marechai.Database.Models.GenreBySoftware", b =>
{
b.HasOne("Marechai.Database.Models.SoftwareGenre", "Genre")
.WithMany("Softwares")
.HasForeignKey("GenreId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Marechai.Database.Models.Software", "Software")
.WithMany("Genres")
.HasForeignKey("SoftwareId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Genre");
b.Navigation("Software");
});
modelBuilder.Entity("Marechai.Database.Models.Gpu", b =>
{
b.HasOne("Marechai.Database.Models.Company", "Company")
@@ -7274,6 +7534,25 @@ namespace Marechai.Database.Migrations
b.Navigation("Role");
});
modelBuilder.Entity("Marechai.Database.Models.PeopleBySoftware", b =>
{
b.HasOne("Marechai.Database.Models.Person", "Person")
.WithMany()
.HasForeignKey("PersonId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("Marechai.Database.Models.Software", "Software")
.WithMany("Credits")
.HasForeignKey("SoftwareId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Person");
b.Navigation("Software");
});
modelBuilder.Entity("Marechai.Database.Models.Person", b =>
{
b.HasOne("Marechai.Database.Models.Iso31661Numeric", "CountryOfBirth")
@@ -7420,6 +7699,17 @@ namespace Marechai.Database.Migrations
b.Navigation("Family");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwareAttribute", b =>
{
b.HasOne("Marechai.Database.Models.SoftwareRelease", "SoftwareRelease")
.WithMany("Attributes")
.HasForeignKey("SoftwareReleaseId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("SoftwareRelease");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwareBarcode", b =>
{
b.HasOne("Marechai.Database.Models.SoftwareRelease", "Release")
@@ -8098,10 +8388,14 @@ namespace Marechai.Database.Migrations
b.Navigation("CompilationReleases");
b.Navigation("Credits");
b.Navigation("Descriptions");
b.Navigation("DirectReleases");
b.Navigation("Genres");
b.Navigation("Screenshots");
b.Navigation("Versions");
@@ -8116,6 +8410,11 @@ namespace Marechai.Database.Migrations
b.Navigation("Softwares");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwareGenre", b =>
{
b.Navigation("Softwares");
});
modelBuilder.Entity("Marechai.Database.Models.SoftwarePlatform", b =>
{
b.Navigation("Screenshots");
@@ -8125,6 +8424,8 @@ namespace Marechai.Database.Migrations
modelBuilder.Entity("Marechai.Database.Models.SoftwareRelease", b =>
{
b.Navigation("Attributes");
b.Navigation("Barcodes");
b.Navigation("CollectedBy");

View File

@@ -0,0 +1,9 @@
namespace Marechai.Database.Models;
public class GenreBySoftware
{
public ulong SoftwareId { get; set; }
public virtual Software Software { get; set; }
public int GenreId { get; set; }
public virtual SoftwareGenre Genre { get; set; }
}

View File

@@ -149,6 +149,12 @@ public class MarechaiContext : IdentityDbContext<ApplicationUser, ApplicationRol
public virtual DbSet<CollectedBook> CollectedBooks { get; set; }
public virtual DbSet<CollectedDocument> CollectedDocuments { get; set; }
public virtual DbSet<CollectedSoftwareRelease> CollectedSoftwareReleases { get; set; }
public virtual DbSet<SoftwareGenre> SoftwareGenres { get; set; }
public virtual DbSet<GenreBySoftware> GenresBySoftware { get; set; }
public virtual DbSet<PeopleBySoftware> PeopleBySoftware { get; set; }
public virtual DbSet<SoftwareAttribute> SoftwareAttributes { get; set; }
public virtual DbSet<MobyGamesImportState> MobyGamesImportStates { get; set; }
public virtual DbSet<MobyGamesRejection> MobyGamesRejections { get; set; }
protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
{
@@ -2386,5 +2392,62 @@ public class MarechaiContext : IdentityDbContext<ApplicationUser, ApplicationRol
.HasForeignKey(e => e.SoftwareReleaseId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<SoftwareGenre>(entity =>
{
entity.HasIndex(e => new { e.Name, e.Type }).IsUnique();
});
modelBuilder.Entity<GenreBySoftware>(entity =>
{
entity.HasKey(e => new { e.SoftwareId, e.GenreId });
entity.HasOne(e => e.Software)
.WithMany(p => p.Genres)
.HasForeignKey(e => e.SoftwareId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasOne(e => e.Genre)
.WithMany(p => p.Softwares)
.HasForeignKey(e => e.GenreId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<PeopleBySoftware>(entity =>
{
entity.HasIndex(e => new { e.SoftwareId, e.PersonId, e.Role }).IsUnique();
entity.HasOne(e => e.Software)
.WithMany(p => p.Credits)
.HasForeignKey(e => e.SoftwareId)
.OnDelete(DeleteBehavior.Cascade);
entity.HasOne(e => e.Person)
.WithMany()
.HasForeignKey(e => e.PersonId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<SoftwareAttribute>(entity =>
{
entity.HasIndex(e => new { e.SoftwareReleaseId, e.Category, e.Key });
entity.HasOne(e => e.SoftwareRelease)
.WithMany(p => p.Attributes)
.HasForeignKey(e => e.SoftwareReleaseId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<MobyGamesImportState>(entity =>
{
entity.HasIndex(e => e.MobyGameId).IsUnique();
entity.HasIndex(e => e.Status);
});
modelBuilder.Entity<MobyGamesRejection>(entity =>
{
entity.HasIndex(e => e.MobyGameId);
entity.HasIndex(e => e.ReviewAction);
});
}
}

View File

@@ -0,0 +1,24 @@
using System;
using System.ComponentModel.DataAnnotations;
using Marechai.Data;
namespace Marechai.Database.Models;
public class MobyGamesImportState : BaseModel<long>
{
[Required]
[StringLength(64)]
public string MobyGameId { get; set; }
[Required]
public MobyGamesImportStatus Status { get; set; }
[StringLength(1024)]
public string ErrorMessage { get; set; }
public DateTime? ProcessedOn { get; set; }
public int BatchNumber { get; set; }
public ulong? SoftwareId { get; set; }
}

View File

@@ -0,0 +1,27 @@
using System;
using System.ComponentModel.DataAnnotations;
using Marechai.Data;
namespace Marechai.Database.Models;
public class MobyGamesRejection : BaseModel<long>
{
[Required]
[StringLength(64)]
public string MobyGameId { get; set; }
[Required]
[StringLength(512)]
public string GameName { get; set; }
[Required]
[StringLength(1024)]
public string Reason { get; set; }
public DateTime RejectedOn { get; set; }
public DateTime? ReviewedOn { get; set; }
[Required]
public MobyGamesRejectionReview ReviewAction { get; set; }
}

View File

@@ -0,0 +1,15 @@
using System.ComponentModel.DataAnnotations;
namespace Marechai.Database.Models;
public class PeopleBySoftware : BaseModel<long>
{
public ulong SoftwareId { get; set; }
public virtual Software Software { get; set; }
public int PersonId { get; set; }
public virtual Person Person { get; set; }
[Required]
[StringLength(128)]
public string Role { get; set; }
}

View File

@@ -14,7 +14,9 @@ public class Software : BaseModel<ulong>
public virtual ICollection<SoftwareVersion> Versions { get; set; }
public virtual ICollection<SoftwareCompanyRole> CompanyRoles { get; set; }
public virtual ICollection<SoftwareScreenshot> Screenshots { get; set; }
public virtual ICollection<SoftwareRelease> DirectReleases { get; set; }
public virtual ICollection<SoftwareRelease> DirectReleases { get; set; }
public virtual ICollection<SoftwareBySoftwareRelease> CompilationReleases { get; set; }
public virtual ICollection<SoftwareDescription> Descriptions { get; set; }
public virtual ICollection<GenreBySoftware> Genres { get; set; }
public virtual ICollection<PeopleBySoftware> Credits { get; set; }
}

View File

@@ -0,0 +1,21 @@
using System.ComponentModel.DataAnnotations;
namespace Marechai.Database.Models;
public class SoftwareAttribute : BaseModel<long>
{
public ulong SoftwareReleaseId { get; set; }
public virtual SoftwareRelease SoftwareRelease { get; set; }
[Required]
[StringLength(64)]
public string Category { get; set; }
[Required]
[StringLength(128)]
public string Key { get; set; }
[Required]
[StringLength(512)]
public string Value { get; set; }
}

View File

@@ -0,0 +1,17 @@
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using Marechai.Data;
namespace Marechai.Database.Models;
public class SoftwareGenre : BaseModel<int>
{
[Required]
[StringLength(128)]
public string Name { get; set; }
[Required]
public SoftwareGenreType Type { get; set; }
public virtual ICollection<GenreBySoftware> Softwares { get; set; }
}

View File

@@ -40,4 +40,5 @@ public class SoftwareRelease : BaseModel<ulong>
public virtual ICollection<SoftwareVersionBySoftwareRelease> IncludedVersions { get; set; }
public virtual ICollection<SoftwareBySoftwareRelease> IncludedSoftware { get; set; }
public virtual ICollection<CollectedSoftwareRelease> CollectedBy { get; set; }
public virtual ICollection<SoftwareAttribute> Attributes { get; set; }
}

View File

@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<RootNamespace>Marechai.MobyGames</RootNamespace>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="HtmlAgilityPack"/>
<PackageReference Include="MySqlConnector"/>
<PackageReference Include="Microsoft.EntityFrameworkCore.Proxies"/>
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql"/>
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql.Json.Microsoft"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Marechai.Database\Marechai.Database.csproj"/>
</ItemGroup>
<ItemGroup>
<Content Include="appsettings.json" CopyToOutputDirectory="PreserveNewest"/>
</ItemGroup>
</Project>

View File

@@ -0,0 +1,18 @@
using System.Threading;
using System.Threading.Tasks;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames;
public class MarechaiContextFactory : IDbContextFactory<MarechaiContext>
{
readonly DbContextOptions<MarechaiContext> _options;
public MarechaiContextFactory(DbContextOptions<MarechaiContext> options) => _options = options;
public MarechaiContext CreateDbContext() => new(_options);
public Task<MarechaiContext> CreateDbContextAsync(CancellationToken cancellationToken = default) =>
Task.FromResult(CreateDbContext());
}

View File

@@ -0,0 +1,8 @@
namespace Marechai.MobyGames.Models;
public class MobyGamesRawRow
{
public string Id { get; set; }
public int Chunk { get; set; }
public string Body { get; set; }
}

View File

@@ -0,0 +1,92 @@
using System;
using System.Collections.Generic;
namespace Marechai.MobyGames.Models;
public class ParsedGame
{
public string MobyGameId { get; set; }
public string Name { get; set; }
// From Main tab
public List<string> Publishers { get; set; } = [];
public List<string> Developers { get; set; } = [];
public string ReleaseDate { get; set; }
public List<string> Platforms { get; set; } = [];
public List<ParsedGenre> Genres { get; set; } = [];
public string Description { get; set; }
public List<string> Groups { get; set; } = [];
// From Credits tab
public List<ParsedCredit> Credits { get; set; } = [];
// From Releases tab
public List<ParsedRelease> Releases { get; set; } = [];
// From Specs tab
public List<ParsedSpec> Specs { get; set; } = [];
// From Rating Systems tab
public List<ParsedRating> Ratings { get; set; } = [];
// Tab availability flags
public bool HasMainTab { get; set; }
public bool HasCreditsTab { get; set; }
public bool HasReleasesTab { get; set; }
public bool HasSpecsTab { get; set; }
public bool HasRatingsTab { get; set; }
}
public class ParsedGenre
{
public string Type { get; set; } // Genre, Perspective, Gameplay, Setting
public string Name { get; set; }
}
public class ParsedCredit
{
public string Role { get; set; }
public string PersonName { get; set; }
}
public class ParsedRelease
{
public string Platform { get; set; }
public string Publisher { get; set; }
public string Developer { get; set; }
public string Distributor { get; set; }
public string Localizer { get; set; }
public Dictionary<string, string> CompanyRoles { get; set; } = new();
public List<string> Countries { get; set; } = [];
public string ReleaseDate { get; set; }
public string Comments { get; set; }
public List<ParsedBarcode> Barcodes { get; set; } = [];
public List<ParsedProductCode> ProductCodes { get; set; } = [];
}
public class ParsedBarcode
{
public string Type { get; set; } // UPC-A, EAN-13
public string Code { get; set; }
}
public class ParsedProductCode
{
public string Type { get; set; } // Sony PN, PSN/SEN Code
public string Code { get; set; }
}
public class ParsedSpec
{
public string Platform { get; set; }
public string Key { get; set; }
public string Value { get; set; }
}
public class ParsedRating
{
public string Platform { get; set; }
public string System { get; set; } // ESRB, PEGI, etc.
public string Rating { get; set; }
public string Descriptors { get; set; }
}

View File

@@ -0,0 +1,70 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using HtmlAgilityPack;
using Marechai.MobyGames.Models;
namespace Marechai.MobyGames.Parsers;
public static class CreditsTabParser
{
public static void Parse(HtmlDocument doc, ParsedGame game)
{
game.HasCreditsTab = true;
// Credits are in a table with rows having class "crln"
var creditRows = doc.DocumentNode.SelectNodes("//table[@summary='List of Credits']//tr[@class='crln']");
if(creditRows is null) return;
foreach(var row in creditRows)
{
var cells = row.SelectNodes("td");
if(cells is null || cells.Count < 2) continue;
string role = WebUtility.HtmlDecode(cells[0].InnerText).Trim();
// Skip copyright notices
if(role.StartsWith("(C)") || role.StartsWith("©")) continue;
// Second cell may contain multiple people separated by commas
var personLinks = cells[1].SelectNodes(".//a[contains(@href,'/developer/')]");
if(personLinks != null)
{
foreach(var link in personLinks)
{
string personName = WebUtility.HtmlDecode(link.InnerText).Trim();
if(!string.IsNullOrWhiteSpace(personName))
{
game.Credits.Add(new ParsedCredit
{
Role = role,
PersonName = personName
});
}
}
}
else
{
// No links — try plain text (rare but possible)
string text = WebUtility.HtmlDecode(cells[1].InnerText).Trim();
if(!string.IsNullOrWhiteSpace(text) && text != "All rights reserved")
{
// Split by comma for multiple names
foreach(string name in text.Split(',').Select(s => s.Trim()).Where(s => !string.IsNullOrWhiteSpace(s)))
{
game.Credits.Add(new ParsedCredit
{
Role = role,
PersonName = name
});
}
}
}
}
}
}

View File

@@ -0,0 +1,208 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using HtmlAgilityPack;
using Marechai.MobyGames.Models;
namespace Marechai.MobyGames.Parsers;
public static class MainTabParser
{
public static void Parse(HtmlDocument doc, ParsedGame game)
{
game.HasMainTab = true;
ParseGameName(doc, game);
ParseCoreInfo(doc, game);
ParseGenres(doc, game);
ParseDescription(doc, game);
ParseGroups(doc, game);
}
static void ParseGameName(HtmlDocument doc, ParsedGame game)
{
var h1 = doc.DocumentNode.SelectSingleNode("//h1[contains(@class,'niceHeaderTitle')]");
if(h1 is null) return;
var a = h1.SelectSingleNode("a");
game.Name = a != null ? WebUtility.HtmlDecode(a.InnerText).Trim() : WebUtility.HtmlDecode(h1.GetDirectInnerText()).Trim();
}
static void ParseCoreInfo(HtmlDocument doc, ParsedGame game)
{
var releaseDiv = doc.DocumentNode.SelectSingleNode("//*[@id='coreGameRelease']");
if(releaseDiv is null) return;
var boldDivs = releaseDiv.SelectNodes("div[contains(@style,'font-weight: bold')]");
if(boldDivs is null) return;
foreach(var boldDiv in boldDivs)
{
string label = WebUtility.HtmlDecode(boldDiv.InnerText).Trim();
var valueDiv = boldDiv.NextSibling;
while(valueDiv != null && valueDiv.NodeType != HtmlNodeType.Element)
valueDiv = valueDiv.NextSibling;
if(valueDiv is null) continue;
string value = WebUtility.HtmlDecode(valueDiv.InnerText).Trim();
switch(label)
{
case "Published by":
var pubLinks = valueDiv.SelectNodes(".//a");
if(pubLinks != null)
game.Publishers = pubLinks
.Select(a => WebUtility.HtmlDecode(a.InnerText).Trim())
.Where(s => !string.IsNullOrWhiteSpace(s))
.ToList();
else if(!string.IsNullOrWhiteSpace(value))
game.Publishers = [value];
break;
case "Developed by":
var devLinks = valueDiv.SelectNodes(".//a");
if(devLinks != null)
game.Developers = devLinks
.Select(a => WebUtility.HtmlDecode(a.InnerText).Trim())
.Where(s => !string.IsNullOrWhiteSpace(s))
.ToList();
else if(!string.IsNullOrWhiteSpace(value))
game.Developers = [value];
break;
case "Released":
game.ReleaseDate = value;
break;
case "Platform":
case "Platforms":
var platformLinks = valueDiv.SelectNodes(".//a");
if(platformLinks != null)
game.Platforms = platformLinks
.Select(a => WebUtility.HtmlDecode(a.InnerText).Trim())
.Where(s => !string.IsNullOrWhiteSpace(s))
.Distinct()
.ToList();
else
game.Platforms = [value];
break;
}
}
}
static void ParseGenres(HtmlDocument doc, ParsedGame game)
{
var genreDiv = doc.DocumentNode.SelectSingleNode("//*[@id='coreGameGenre']");
if(genreDiv is null) return;
var boldDivs = genreDiv.SelectNodes(".//div[contains(@style,'font-weight: bold')]");
if(boldDivs is null) return;
foreach(var boldDiv in boldDivs)
{
string genreType = WebUtility.HtmlDecode(boldDiv.InnerText).Trim();
var valueDiv = boldDiv.NextSibling;
while(valueDiv != null && valueDiv.NodeType != HtmlNodeType.Element)
valueDiv = valueDiv.NextSibling;
if(valueDiv is null) continue;
var links = valueDiv.SelectNodes(".//a");
if(links is null) continue;
foreach(var link in links)
{
string name = WebUtility.HtmlDecode(link.InnerText).Trim();
if(!string.IsNullOrWhiteSpace(name))
{
game.Genres.Add(new ParsedGenre
{
Type = genreType,
Name = name
});
}
}
}
}
static void ParseDescription(HtmlDocument doc, ParsedGame game)
{
var descH2 = doc.DocumentNode.SelectSingleNode("//h2[text()='Description']");
if(descH2 is null) return;
// Collect text nodes and br tags between Description h2 and the next h2 or sideBarLinks div
var sibling = descH2.NextSibling;
var parts = new List<string>();
while(sibling != null)
{
// Stop at next structural element
if(sibling.NodeType == HtmlNodeType.Element)
{
string tag = sibling.Name.ToLowerInvariant();
if(tag is "h2") break;
if(sibling.GetAttributeValue("class", "").Contains("sideBarLinks")) break;
if(tag == "br")
{
parts.Add("\n");
}
else if(tag == "i" || tag == "b" || tag == "em" || tag == "strong" || tag == "a")
{
parts.Add(WebUtility.HtmlDecode(sibling.InnerText));
}
else
{
parts.Add(WebUtility.HtmlDecode(sibling.InnerText));
}
}
else if(sibling.NodeType == HtmlNodeType.Text)
{
parts.Add(WebUtility.HtmlDecode(sibling.InnerText));
}
sibling = sibling.NextSibling;
}
game.Description = string.Join("", parts).Trim();
}
static void ParseGroups(HtmlDocument doc, ParsedGame game)
{
var groupsH2 = doc.DocumentNode.SelectSingleNode("//h2[text()='Part of the Following Groups']");
if(groupsH2 is null) return;
var ul = groupsH2.NextSibling;
while(ul != null && ul.Name != "ul")
ul = ul.NextSibling;
if(ul is null) return;
var links = ul.SelectNodes(".//a");
if(links is null) return;
foreach(var link in links)
{
string name = WebUtility.HtmlDecode(link.InnerText).Trim();
if(!string.IsNullOrWhiteSpace(name))
game.Groups.Add(name);
}
}
}

View File

@@ -0,0 +1,88 @@
using System.Net;
using HtmlAgilityPack;
using Marechai.MobyGames.Models;
namespace Marechai.MobyGames.Parsers;
public static class RatingsTabParser
{
public static void Parse(HtmlDocument doc, ParsedGame game)
{
game.HasRatingsTab = true;
var contentDiv = doc.DocumentNode.SelectSingleNode("//div[contains(@class,'col-md-8')]");
if(contentDiv is null) return;
string currentPlatform = null;
foreach(var child in contentDiv.ChildNodes)
{
if(child.NodeType != HtmlNodeType.Element) continue;
if(child.Name == "h2")
{
currentPlatform = WebUtility.HtmlDecode(child.InnerText).Trim();
continue;
}
if(child.Name == "table" &&
child.GetAttributeValue("summary", "").Contains("Rating"))
{
var rows = child.SelectNodes("tr");
if(rows is null) continue;
foreach(var row in rows)
{
var cells = row.SelectNodes("td");
if(cells is null || cells.Count < 3) continue;
string system = WebUtility.HtmlDecode(cells[0].InnerText).Trim();
string ratingCell = cells[2].InnerText.Trim();
// Skip unknown ratings
if(ratingCell == "unknown") continue;
// Extract rating value — could be in link text
var ratingLink = cells[2].SelectSingleNode(".//a");
string rating = ratingLink != null
? WebUtility.HtmlDecode(ratingLink.InnerText).Trim()
: WebUtility.HtmlDecode(ratingCell).Trim();
// Extract descriptors if present (text after parentheses)
string descriptors = null;
string fullText = WebUtility.HtmlDecode(cells[2].InnerText);
int parenStart = fullText.IndexOf('(');
if(parenStart >= 0)
{
int parenEnd = fullText.LastIndexOf(')');
if(parenEnd > parenStart)
{
string inside = fullText.Substring(parenStart + 1, parenEnd - parenStart - 1);
// Remove label prefixes like "Descriptors: " or "Content Indicator: "
int colonPos = inside.IndexOf(':');
descriptors = colonPos >= 0
? inside[(colonPos + 1)..].Trim()
: inside.Trim();
}
}
game.Ratings.Add(new ParsedRating
{
Platform = currentPlatform,
System = system,
Rating = rating,
Descriptors = descriptors
});
}
}
}
}
}

View File

@@ -0,0 +1,179 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Marechai.MobyGames.Models;
namespace Marechai.MobyGames.Parsers;
public static partial class ReleasesTabParser
{
public static void Parse(HtmlDocument doc, ParsedGame game)
{
game.HasReleasesTab = true;
var contentDiv = doc.DocumentNode.SelectSingleNode("//div[contains(@class,'col-md-8')]");
if(contentDiv is null) return;
string currentPlatform = null;
string currentPublisher = null;
string currentDeveloper = null;
string currentDistributor = null;
string currentLocalizer = null;
var currentExtraRoles = new Dictionary<string, string>();
foreach(var child in contentDiv.ChildNodes)
{
if(child.NodeType != HtmlNodeType.Element) continue;
// Platform header
if(child.Name == "h2")
{
currentPlatform = WebUtility.HtmlDecode(child.InnerText).Trim();
currentPublisher = null;
currentDeveloper = null;
currentDistributor = null;
currentLocalizer = null;
currentExtraRoles.Clear();
continue;
}
// Company role line: <div class="floatholder"><div class="fl">Published by</div><a>Company</a></div>
if(child.Name == "div" &&
child.GetAttributeValue("class", "").Contains("floatholder") &&
!child.GetAttributeValue("class", "").Contains("relInfo"))
{
var roleDiv = child.SelectSingleNode("div[contains(@class,'fl')]");
if(roleDiv != null)
{
string roleLabel = WebUtility.HtmlDecode(roleDiv.InnerText).Trim();
var companyLink = child.SelectSingleNode(".//a");
string companyName = companyLink != null
? WebUtility.HtmlDecode(companyLink.InnerText).Trim()
: null;
switch(roleLabel)
{
case "Published by":
currentPublisher = companyName;
break;
case "Developed by":
currentDeveloper = companyName;
break;
case "Distributed by":
currentDistributor = companyName;
break;
case "Localized by":
currentLocalizer = companyName;
break;
default:
if(companyName != null)
currentExtraRoles[roleLabel] = companyName;
break;
}
continue;
}
}
// Release block: <div style="margin: 0.25em ..."> containing relInfo divs
if(child.Name == "div" && child.GetAttributeValue("style", "").Contains("margin:"))
{
var relInfoDivs = child.SelectNodes(".//div[contains(@class,'relInfo')]");
if(relInfoDivs is null) continue;
var release = new ParsedRelease
{
Platform = currentPlatform,
Publisher = currentPublisher,
Developer = currentDeveloper,
Distributor = currentDistributor,
Localizer = currentLocalizer,
CompanyRoles = new Dictionary<string, string>(currentExtraRoles)
};
foreach(var relInfo in relInfoDivs)
{
var titleDiv = relInfo.SelectSingleNode("div[contains(@class,'relInfoTitle')]");
var detailsDiv = relInfo.SelectSingleNode("div[contains(@class,'relInfoDetails')]");
if(titleDiv is null || detailsDiv is null) continue;
string title = WebUtility.HtmlDecode(titleDiv.InnerText).Trim();
string details = WebUtility.HtmlDecode(detailsDiv.InnerText).Trim();
switch(title)
{
case "Country":
case "Countries":
// Extract country names from spans (strip flag imgs)
var spans = detailsDiv.SelectNodes(".//span");
if(spans != null)
{
foreach(var span in spans)
{
// Get text, remove img alt text influence
string countryText = WebUtility.HtmlDecode(span.InnerText).Trim()
.TrimEnd(',').Trim();
if(!string.IsNullOrWhiteSpace(countryText))
release.Countries.Add(countryText);
}
}
else
{
release.Countries.Add(details);
}
break;
case "Release Date":
release.ReleaseDate = details;
break;
case "Comments":
release.Comments = details;
break;
case "UPC-A":
release.Barcodes.Add(new ParsedBarcode { Type = "UPC-A", Code = CleanBarcode(details) });
break;
case "EAN-13":
release.Barcodes.Add(new ParsedBarcode { Type = "EAN-13", Code = CleanBarcode(details) });
break;
case "Sony PN":
release.ProductCodes.Add(new ParsedProductCode { Type = "Sony PN", Code = details });
break;
case "PSN/SEN Code":
release.ProductCodes.Add(new ParsedProductCode { Type = "PSN/SEN Code", Code = details });
break;
default:
// Other product codes (Nintendo PN, Microsoft PN, etc.)
if(title.EndsWith("PN") || title.Contains("Code"))
release.ProductCodes.Add(new ParsedProductCode { Type = title, Code = details });
break;
}
}
game.Releases.Add(release);
}
}
}
static string CleanBarcode(string barcode) =>
WhitespaceRegex().Replace(barcode, "").Replace("\u00a0", "");
[GeneratedRegex(@"\s+")]
private static partial Regex WhitespaceRegex();
}

View File

@@ -0,0 +1,50 @@
using System.Net;
using HtmlAgilityPack;
using Marechai.MobyGames.Models;
namespace Marechai.MobyGames.Parsers;
public static class SpecsTabParser
{
public static void Parse(HtmlDocument doc, ParsedGame game)
{
game.HasSpecsTab = true;
var tables = doc.DocumentNode.SelectNodes("//table[contains(@class,'techInfo')]");
if(tables is null) return;
foreach(var table in tables)
{
// Platform name from thead
var thead = table.SelectSingleNode(".//thead//td");
string platform = thead != null ? WebUtility.HtmlDecode(thead.InnerText).Trim() : "Unknown";
var rows = table.SelectNodes(".//tr[@valign='middle' or @valign='top']");
if(rows is null) continue;
foreach(var row in rows)
{
// Skip thead rows
if(row.SelectSingleNode("th") != null) continue;
var cells = row.SelectNodes("td");
if(cells is null || cells.Count < 2) continue;
string key = WebUtility.HtmlDecode(cells[0].InnerText).Trim();
string value = WebUtility.HtmlDecode(cells[1].InnerText).Trim();
if(string.IsNullOrWhiteSpace(key) || string.IsNullOrWhiteSpace(value)) continue;
game.Specs.Add(new ParsedSpec
{
Platform = platform,
Key = key,
Value = value
});
}
}
}
}

View File

@@ -0,0 +1,78 @@
using HtmlAgilityPack;
namespace Marechai.MobyGames.Parsers;
public enum MobyTab
{
Main,
Credits,
CoverArt,
Releases,
Specs,
RatingSystems,
Reviews,
Screenshots,
PromoArt,
Trivia,
AdBlurb,
BuyTrade,
Unknown
}
public static class TabDetector
{
public static MobyTab Detect(string html)
{
var doc = new HtmlDocument();
doc.LoadHtml(html);
// Try active tab detection from <li class="active"> or <li class="disabled"> with matching text
var activeLi = doc.DocumentNode.SelectSingleNode("//ul[contains(@class,'nav-tabs')]//li[contains(@class,'active')]");
if(activeLi != null)
{
string tabText = activeLi.InnerText.Trim();
return tabText switch
{
"Main" => MobyTab.Main,
"Credits" => MobyTab.Credits,
"Cover Art" => MobyTab.CoverArt,
"Releases" => MobyTab.Releases,
"Specs" => MobyTab.Specs,
"Rating Systems" => MobyTab.RatingSystems,
"Reviews" => MobyTab.Reviews,
"Screenshots" => MobyTab.Screenshots,
"Promo Art" => MobyTab.PromoArt,
"Trivia" => MobyTab.Trivia,
"Ad Blurb" => MobyTab.AdBlurb,
"Buy/Trade" => MobyTab.BuyTrade,
_ => MobyTab.Unknown
};
}
// Fallback: detect from h1 title suffix
var h1 = doc.DocumentNode.SelectSingleNode("//h1[contains(@class,'niceHeaderTitle')]");
if(h1 != null)
{
string text = h1.GetDirectInnerText().Trim();
if(text.EndsWith("Credits")) return MobyTab.Credits;
if(text.EndsWith("Covers")) return MobyTab.CoverArt;
if(text.EndsWith("Releases")) return MobyTab.Releases;
if(text.EndsWith("Specs")) return MobyTab.Specs;
if(text.EndsWith("Rating Systems")) return MobyTab.RatingSystems;
if(text.EndsWith("Reviews")) return MobyTab.Reviews;
if(text.EndsWith("Screenshots")) return MobyTab.Screenshots;
}
// If no active tab and no suffix, it's likely the Main tab
// (Main tab has no suffix — just the game name)
var descH2 = doc.DocumentNode.SelectSingleNode("//h2[text()='Description']");
if(descH2 != null) return MobyTab.Main;
return MobyTab.Unknown;
}
}

View File

@@ -0,0 +1,108 @@
using System;
using System.Threading.Tasks;
using Marechai.Database.Models;
using Marechai.MobyGames.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
namespace Marechai.MobyGames;
class Program
{
static async Task<int> Main(string[] args)
{
Console.WriteLine("\e[32;1mMarechai MobyGames HTML Import Tool\e[0m\n");
var config = new ConfigurationBuilder()
.AddJsonFile("appsettings.json")
.Build();
string marechaiConn = config.GetConnectionString("DefaultConnection");
string mobyConn = config.GetConnectionString("MobyGamesSource");
if(string.IsNullOrEmpty(marechaiConn) || string.IsNullOrEmpty(mobyConn))
{
Console.WriteLine("\e[31;1mMissing connection strings in appsettings.json\e[0m");
return 1;
}
// Setup EF context factory
var optionsBuilder = new DbContextOptionsBuilder<MarechaiContext>();
optionsBuilder.UseLazyLoadingProxies()
.UseMySql(marechaiConn,
new MariaDbServerVersion(new Version(10, 5, 0)),
b => b.UseMicrosoftJson().EnableStringComparisonTranslations());
var factory = new MarechaiContextFactory(optionsBuilder.Options);
var sourceDb = new SourceDatabaseService(mobyConn);
var companyMatcher = new CompanyMatcher(factory);
var personMatcher = new PersonMatcher(factory);
var platformMatcher = new PlatformMatcher(factory);
var countryMatcher = new CountryMatcher(factory);
var stateService = new StateService(factory);
var importService = new ImportService(factory, sourceDb, companyMatcher,
personMatcher, platformMatcher,
countryMatcher, stateService);
string command = args.Length > 0 ? args[0].ToLowerInvariant() : "import";
switch(command)
{
case "import":
int batchSize = config.GetValue("Import:BatchSize", 500);
for(int i = 0; i < args.Length - 1; i++)
{
if(args[i] == "--batch-size" && int.TryParse(args[i + 1], out int bs))
batchSize = bs;
}
// Determine batch number
var processed = await stateService.GetProcessedGameIdsAsync();
int batchNumber = (processed.Count / batchSize) + 1;
Console.WriteLine($" Batch size: {batchSize}, Batch number: {batchNumber}");
int totalGames = await sourceDb.GetTotalGameCountAsync();
Console.WriteLine($" Total games in source: {totalGames}");
await importService.RunBatchAsync(batchSize, batchNumber);
break;
case "status":
await stateService.PrintStatusAsync();
break;
case "reset":
string resetId = null;
for(int i = 0; i < args.Length - 1; i++)
{
if(args[i] == "--game")
resetId = args[i + 1];
}
if(!string.IsNullOrEmpty(resetId))
await stateService.ResetGameAsync(resetId);
else
Console.WriteLine(" Usage: reset --game <mobyGameId>");
break;
default:
Console.WriteLine(" Usage:");
Console.WriteLine(" import [--batch-size N] Import next batch of games");
Console.WriteLine(" status Show import status counts");
Console.WriteLine(" reset --game <id> Reset a game to unprocessed");
break;
}
return 0;
}
}

View File

@@ -0,0 +1,207 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames.Services;
public partial class CompanyMatcher
{
readonly IDbContextFactory<MarechaiContext> _contextFactory;
List<Company> _companies;
Dictionary<string, List<Company>> _soundexIndex;
Dictionary<string, List<Company>> _strippedSoundexIndex;
readonly Dictionary<string, Company> _cache = new(StringComparer.OrdinalIgnoreCase);
// Common company suffixes to strip for matching
static readonly string[] CompanySuffixes =
[
", Inc.", ", Inc", " Inc.", " Inc",
", Ltd.", ", Ltd", " Ltd.", " Ltd",
", LLC", " LLC",
", S.A.", " S.A.", ", SA", " SA",
", S.L.", " S.L.",
", GmbH", " GmbH",
", AG", " AG",
", Co.", " Co.",
", Corp.", " Corp.", " Corp",
" Corporation",
", Limited", " Limited",
" Interactive",
" Entertainment",
" Software",
" Games",
" Studios",
" Studio",
" Productions",
" Publishing",
" Digital",
" Media",
" Group",
" Company"
];
public CompanyMatcher(IDbContextFactory<MarechaiContext> contextFactory) => _contextFactory = contextFactory;
public async Task LoadAsync()
{
await using var context = await _contextFactory.CreateDbContextAsync();
_companies = await context.Companies
.Select(c => new Company { Id = c.Id, Name = c.Name })
.ToListAsync();
_soundexIndex = SoundexHelper.BuildSoundexIndex(_companies, c => c.Name);
_strippedSoundexIndex = SoundexHelper.BuildSoundexIndex(_companies, c => StripSuffix(c.Name));
}
public async Task<(Company company, string matchType)> MatchOrCreateAsync(string name)
{
if(string.IsNullOrWhiteSpace(name))
return (null, "empty");
string normalizedName = name.Replace("\u00a0", " ").Trim();
// Check cache first
if(_cache.TryGetValue(normalizedName, out var cached))
return (cached, "cached");
// Exact match
var exact = _companies.FirstOrDefault(c =>
string.Equals(c.Name, normalizedName, StringComparison.OrdinalIgnoreCase));
if(exact != null)
{
_cache[normalizedName] = exact;
return (exact, "exact");
}
// Exact match on stripped names (e.g., "Apple Inc." matches "Apple")
string strippedInput = StripSuffix(normalizedName);
var strippedExact = _companies.FirstOrDefault(c =>
string.Equals(StripSuffix(c.Name), strippedInput, StringComparison.OrdinalIgnoreCase));
if(strippedExact != null)
{
_cache[normalizedName] = strippedExact;
return (strippedExact, "exact-stripped");
}
// Soundex match on full name
string soundex = SoundexHelper.Generate(normalizedName);
if(_soundexIndex.TryGetValue(soundex, out var candidates) && candidates.Count > 0)
{
if(candidates.Count == 1)
{
_cache[normalizedName] = candidates[0];
return (candidates[0], "soundex");
}
var prompted = PromptMultiple(normalizedName, candidates);
if(prompted != null)
{
_cache[normalizedName] = prompted;
return (prompted, "soundex-selected");
}
}
// Soundex match on stripped name
string strippedSoundex = SoundexHelper.Generate(strippedInput);
if(_strippedSoundexIndex.TryGetValue(strippedSoundex, out var strippedCandidates) &&
strippedCandidates.Count > 0)
{
if(strippedCandidates.Count == 1)
{
_cache[normalizedName] = strippedCandidates[0];
return (strippedCandidates[0], "soundex-stripped");
}
var prompted = PromptMultiple(normalizedName, strippedCandidates);
if(prompted != null)
{
_cache[normalizedName] = prompted;
return (prompted, "soundex-stripped-selected");
}
}
// Create new company
await using var context = await _contextFactory.CreateDbContextAsync();
var newCompany = new Company
{
Name = normalizedName,
Status = Data.CompanyStatus.Unknown
};
context.Companies.Add(newCompany);
await context.SaveChangesAsync();
_companies.Add(newCompany);
// Update Soundex index
if(!_soundexIndex.TryGetValue(soundex, out var list))
{
list = [];
_soundexIndex[soundex] = list;
}
list.Add(newCompany);
_cache[normalizedName] = newCompany;
Console.WriteLine($" Created new company: \"{normalizedName}\" (ID: {newCompany.Id})");
return (newCompany, "created");
}
static Company PromptMultiple(string normalizedName, List<Company> candidates)
{
Console.WriteLine($"\n Multiple Soundex matches for \"{normalizedName}\":");
for(int i = 0; i < candidates.Count; i++)
Console.WriteLine($" [{i + 1}] {candidates[i].Name} (ID: {candidates[i].Id})");
Console.WriteLine($" [0] Create new company");
Console.Write(" Select: ");
string input = Console.ReadLine()?.Trim();
if(int.TryParse(input, out int choice) && choice >= 1 && choice <= candidates.Count)
return candidates[choice - 1];
return null; // User chose 0 or invalid — fall through to create
}
static string StripSuffix(string name)
{
if(string.IsNullOrWhiteSpace(name))
return name;
string result = name;
foreach(string suffix in CompanySuffixes)
{
if(result.EndsWith(suffix, StringComparison.OrdinalIgnoreCase))
{
result = result[..^suffix.Length].TrimEnd();
break; // Only strip the first matching suffix
}
}
return result;
}
}

View File

@@ -0,0 +1,96 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames.Services;
public class CountryMatcher
{
readonly IDbContextFactory<MarechaiContext> _contextFactory;
List<UnM49> _countries;
readonly Dictionary<string, UnM49> _cache = new(StringComparer.OrdinalIgnoreCase);
static readonly Dictionary<string, string> Aliases = new(StringComparer.OrdinalIgnoreCase)
{
["United States"] = "United States of America",
["Russia"] = "Russian Federation",
["South Korea"] = "Republic of Korea",
["North Korea"] = "Democratic People's Republic of Korea",
["The Netherlands"] = "Netherlands",
["Czechia"] = "Czechia",
["Taiwan"] = "Taiwan",
["Hong Kong"] = "China, Hong Kong Special Administrative Region",
["Macau"] = "China, Macao Special Administrative Region",
["Vietnam"] = "Viet Nam",
["Brunei"] = "Brunei Darussalam",
["Bolivia"] = "Bolivia (Plurinational State of)",
["Iran"] = "Iran (Islamic Republic of)",
["Syria"] = "Syrian Arab Republic",
["Venezuela"] = "Venezuela (Bolivarian Republic of)",
["Tanzania"] = "United Republic of Tanzania",
["Moldova"] = "Republic of Moldova",
["Ivory Coast"] = "Côte d'Ivoire",
["Laos"] = "Lao People's Democratic Republic",
["North Macedonia"] = "North Macedonia"
};
public CountryMatcher(IDbContextFactory<MarechaiContext> contextFactory) => _contextFactory = contextFactory;
public async Task LoadAsync()
{
await using var context = await _contextFactory.CreateDbContextAsync();
_countries = await context.UnM49
.Where(u => u.Type == Data.UnM49Type.Country)
.Select(u => new UnM49 { Id = u.Id, Name = u.Name })
.ToListAsync();
}
public UnM49 Match(string countryName)
{
if(string.IsNullOrWhiteSpace(countryName))
return null;
string normalized = countryName.Replace("\u00a0", " ").Trim();
// Strip continent prefix like "Africa » South Africa"
int arrowIndex = normalized.IndexOf('»');
if(arrowIndex >= 0)
normalized = normalized[(arrowIndex + 1)..].Trim();
if(_cache.TryGetValue(normalized, out var cached))
return cached;
// Try alias
if(Aliases.TryGetValue(normalized, out string aliased))
normalized = aliased;
// Exact match
var exact = _countries.FirstOrDefault(c =>
string.Equals(c.Name, normalized, StringComparison.OrdinalIgnoreCase));
if(exact != null)
{
_cache[countryName] = exact;
return exact;
}
// Soundex match
string soundex = SoundexHelper.Generate(normalized);
var matches = _countries.Where(c => SoundexHelper.Generate(c.Name) == soundex).ToList();
if(matches.Count == 1)
{
_cache[countryName] = matches[0];
return matches[0];
}
return null;
}
}

View File

@@ -0,0 +1,56 @@
using System.Collections.Generic;
using HtmlAgilityPack;
using Marechai.MobyGames.Models;
using Marechai.MobyGames.Parsers;
namespace Marechai.MobyGames.Services;
public static class GameAssembler
{
public static ParsedGame Assemble(string gameId, List<MobyGamesRawRow> rows)
{
var game = new ParsedGame { MobyGameId = gameId };
foreach(var row in rows)
{
var doc = new HtmlDocument();
doc.LoadHtml(row.Body);
MobyTab tab = TabDetector.Detect(row.Body);
switch(tab)
{
case MobyTab.Main:
MainTabParser.Parse(doc, game);
break;
case MobyTab.Credits:
CreditsTabParser.Parse(doc, game);
break;
case MobyTab.Releases:
ReleasesTabParser.Parse(doc, game);
break;
case MobyTab.Specs:
SpecsTabParser.Parse(doc, game);
break;
case MobyTab.RatingSystems:
RatingsTabParser.Parse(doc, game);
break;
// Skip tabs we don't import: Screenshots, CoverArt, PromoArt, Reviews, Trivia, AdBlurb, BuyTrade
}
}
// If game name not set from Main tab, try extracting from any tab's h1
if(string.IsNullOrWhiteSpace(game.Name) && rows.Count > 0)
{
var doc = new HtmlDocument();
doc.LoadHtml(rows[0].Body);
var h1 = doc.DocumentNode.SelectSingleNode("//h1[contains(@class,'niceHeaderTitle')]//a");
if(h1 != null)
game.Name = System.Net.WebUtility.HtmlDecode(h1.InnerText).Trim();
}
return game;
}
}

View File

@@ -0,0 +1,654 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
using Marechai.Data;
using Marechai.Database.Models;
using Marechai.MobyGames.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames.Services;
public class ImportService
{
readonly IDbContextFactory<MarechaiContext> _contextFactory;
readonly SourceDatabaseService _sourceDb;
readonly CompanyMatcher _companyMatcher;
readonly PersonMatcher _personMatcher;
readonly PlatformMatcher _platformMatcher;
readonly CountryMatcher _countryMatcher;
readonly StateService _stateService;
public ImportService(
IDbContextFactory<MarechaiContext> contextFactory,
SourceDatabaseService sourceDb,
CompanyMatcher companyMatcher,
PersonMatcher personMatcher,
PlatformMatcher platformMatcher,
CountryMatcher countryMatcher,
StateService stateService)
{
_contextFactory = contextFactory;
_sourceDb = sourceDb;
_companyMatcher = companyMatcher;
_personMatcher = personMatcher;
_platformMatcher = platformMatcher;
_countryMatcher = countryMatcher;
_stateService = stateService;
}
public async Task RunBatchAsync(int batchSize, int batchNumber)
{
Console.WriteLine("\n Loading reference data...");
await _companyMatcher.LoadAsync();
await _personMatcher.LoadAsync();
await _platformMatcher.LoadAsync();
await _countryMatcher.LoadAsync();
var processedIds = await _stateService.GetProcessedGameIdsAsync();
Console.WriteLine($" Already processed: {processedIds.Count} games");
var gameIds = await _sourceDb.GetUnprocessedGameIdsAsync(batchSize, processedIds);
Console.WriteLine($" Found {gameIds.Count} unprocessed games for this batch\n");
if(gameIds.Count == 0)
{
Console.WriteLine(" No more games to process.");
return;
}
bool acceptAll = false;
int imported = 0, rejected = 0, failed = 0;
for(int i = 0; i < gameIds.Count; i++)
{
string gameId = gameIds[i];
try
{
var rows = await _sourceDb.GetRowsForGameAsync(gameId);
var game = GameAssembler.Assemble(gameId, rows);
Console.WriteLine($"\n [{i + 1}/{gameIds.Count}] Game ID: {gameId}");
Console.WriteLine($" Name: {game.Name ?? "(unknown)"}");
Console.WriteLine($" Publisher: {(game.Publishers.Count > 0 ? string.Join("; ", game.Publishers) : "(none)")}");
Console.WriteLine($" Developer: {(game.Developers.Count > 0 ? string.Join("; ", game.Developers) : "(none)")}");
Console.WriteLine($" Date: {game.ReleaseDate ?? "(none)"}");
Console.WriteLine($" Platforms: {string.Join(", ", game.Platforms)}");
Console.WriteLine($" Genres: {string.Join(", ", game.Genres.Select(g => $"{g.Type}:{g.Name}"))}");
Console.WriteLine($" Credits: {game.Credits.Count} entries");
Console.WriteLine($" Releases: {game.Releases.Count} entries");
Console.WriteLine($" Specs: {game.Specs.Count} entries");
Console.WriteLine($" Ratings: {game.Ratings.Count} entries");
if(string.IsNullOrWhiteSpace(game.Name))
{
Console.WriteLine(" SKIPPING: No game name found");
await _stateService.MarkFailedAsync(gameId, "No game name", batchNumber);
failed++;
continue;
}
if(!acceptAll)
{
Console.Write("\n [A]ccept / [R]eject / [S]kip / Accept A[l]l / [Q]uit: ");
string input = Console.ReadLine()?.Trim().ToUpperInvariant();
switch(input)
{
case "Q":
Console.WriteLine($"\n Quitting. {imported} imported, {rejected} rejected, {failed} failed so far.");
return;
case "R":
Console.Write(" Rejection reason: ");
string reason = Console.ReadLine()?.Trim() ?? "User rejected";
await _stateService.MarkRejectedAsync(gameId, game.Name, reason, batchNumber);
rejected++;
continue;
case "S":
continue;
case "L":
acceptAll = true;
break;
case "A":
break;
default:
continue;
}
}
await ImportGameAsync(game, batchNumber);
imported++;
}
catch(Exception ex)
{
Console.WriteLine($" ERROR: {ex.Message}");
await _stateService.MarkFailedAsync(gameId, ex.Message, batchNumber);
failed++;
}
}
Console.WriteLine($"\n Batch complete: {imported} imported, {rejected} rejected, {failed} failed");
}
async Task ImportGameAsync(ParsedGame game, int batchNumber)
{
await using var context = await _contextFactory.CreateDbContextAsync();
// 1. Check for existing Software with same name
var existingByName = await context.Softwares
.Where(s => s.Name == game.Name)
.Select(s => new { s.Id, s.Name, s.IsGame })
.ToListAsync();
Software software;
if(existingByName.Count > 0)
{
// Parse the incoming game's year for comparison
string incomingYear = ExtractYear(game.ReleaseDate) ?? "(no date)";
// Also check releases tab for earliest date
if(incomingYear == "(no date)" && game.Releases.Count > 0)
{
var firstRelDate = game.Releases
.Where(r => !string.IsNullOrWhiteSpace(r.ReleaseDate))
.Select(r => r.ReleaseDate)
.FirstOrDefault();
if(firstRelDate != null)
incomingYear = ExtractYear(firstRelDate) ?? "(no date)";
}
Console.WriteLine($"\n WARNING: {existingByName.Count} existing entries named \"{game.Name}\"");
Console.WriteLine($" Incoming game year: {incomingYear}, platforms: {string.Join(", ", game.Platforms)}");
// Fetch release info for each existing match
foreach(var existing in existingByName)
{
var releases = await context.SoftwareReleases
.Where(r => r.SoftwareId == (ulong)existing.Id)
.Select(r => new
{
r.ReleaseDate,
Platform = r.Platform != null ? r.Platform.Name : null
})
.ToListAsync();
var years = releases.Where(r => r.ReleaseDate.HasValue)
.Select(r => r.ReleaseDate.Value.Year.ToString())
.Distinct();
var platforms = releases.Where(r => r.Platform != null)
.Select(r => r.Platform)
.Distinct();
string yearStr = years.Any() ? string.Join("/", years) : "no releases";
string platformStr = platforms.Any() ? string.Join(", ", platforms) : "no platforms";
Console.WriteLine($" [{existingByName.IndexOf(existing) + 1}] ID: {existing.Id}, " +
$"Year(s): {yearStr}, Platforms: {platformStr}");
}
Console.Write(" [N]ew entry / [1-N] Link to existing / [S]kip: ");
string input = Console.ReadLine()?.Trim().ToUpperInvariant();
if(input == "S")
{
Console.WriteLine(" Skipped.");
return;
}
if(int.TryParse(input, out int choice) && choice >= 1 && choice <= existingByName.Count)
{
software = await context.Softwares.FindAsync(existingByName[choice - 1].Id);
Console.WriteLine($" Linked to existing Software ID: {software.Id}");
}
else
{
software = new Software { Name = game.Name, IsGame = true };
context.Softwares.Add(software);
await context.SaveChangesAsync();
Console.WriteLine($" Created new Software ID: {software.Id}");
}
}
else
{
software = new Software { Name = game.Name, IsGame = true };
context.Softwares.Add(software);
await context.SaveChangesAsync();
}
// 2. Description
if(!string.IsNullOrWhiteSpace(game.Description))
{
context.SoftwareDescriptions.Add(new SoftwareDescription
{
SoftwareId = software.Id,
LanguageCode = "eng",
Text = game.Description
});
}
// 3. Genres
foreach(var genre in game.Genres)
{
var genreType = genre.Type switch
{
"Genre" => SoftwareGenreType.Genre,
"Perspective" => SoftwareGenreType.Perspective,
"Gameplay" => SoftwareGenreType.Gameplay,
"Setting" => SoftwareGenreType.Setting,
_ => SoftwareGenreType.Genre
};
var dbGenre = await context.SoftwareGenres
.FirstOrDefaultAsync(g => g.Name == genre.Name && g.Type == genreType);
if(dbGenre is null)
{
dbGenre = new SoftwareGenre { Name = genre.Name, Type = genreType };
context.SoftwareGenres.Add(dbGenre);
await context.SaveChangesAsync();
}
context.GenresBySoftware.Add(new GenreBySoftware
{
SoftwareId = software.Id,
GenreId = dbGenre.Id
});
}
// 4. Company roles from Main tab (developers)
foreach(string devName in game.Developers)
{
if(string.IsNullOrWhiteSpace(devName)) continue;
var (devCompany, _) = await _companyMatcher.MatchOrCreateAsync(devName);
if(devCompany != null)
{
bool exists = await context.SoftwareCompanyRoles
.AnyAsync(r => r.SoftwareId == software.Id &&
r.CompanyId == devCompany.Id &&
r.RoleId == "dev");
if(!exists)
{
context.SoftwareCompanyRoles.Add(new SoftwareCompanyRole
{
SoftwareId = software.Id,
CompanyId = devCompany.Id,
RoleId = "dev"
});
}
}
}
// 5. Process releases from Releases tab (or create basic release from Main tab)
if(game.Releases.Count > 0)
await ImportReleasesAsync(context, software, game);
else
await ImportBasicReleaseAsync(context, software, game);
// 6. Credits
foreach(var credit in game.Credits)
{
var person = await _personMatcher.MatchOrCreateAsync(credit.PersonName);
if(person != null)
{
// Check for duplicates
bool exists = await context.PeopleBySoftware
.AnyAsync(p => p.SoftwareId == software.Id &&
p.PersonId == person.Id &&
p.Role == credit.Role);
if(!exists)
{
context.PeopleBySoftware.Add(new PeopleBySoftware
{
SoftwareId = software.Id,
PersonId = person.Id,
Role = credit.Role
});
}
}
}
// 7. Specs and Ratings stored as SoftwareAttributes (need release IDs — done inside ImportReleasesAsync)
await context.SaveChangesAsync();
await _stateService.MarkImportedAsync(game.MobyGameId, batchNumber, software.Id);
Console.WriteLine($" Imported as Software ID: {software.Id}");
}
async Task ImportReleasesAsync(MarechaiContext context, Software software, ParsedGame game)
{
// Group releases by platform
var platformGroups = game.Releases.GroupBy(r => r.Platform ?? "Unknown");
foreach(var platformGroup in platformGroups)
{
var platform = await _platformMatcher.MatchOrCreateAsync(platformGroup.Key);
foreach(var release in platformGroup)
{
var (publisher, _) = await _companyMatcher.MatchOrCreateAsync(
release.Publisher ?? game.Publishers.FirstOrDefault());
if(publisher is null) continue;
var (releaseDate, precision) = ParseDate(release.ReleaseDate);
var dbRelease = new SoftwareRelease
{
SoftwareId = software.Id,
PlatformId = platform?.Id,
PublisherId = publisher.Id,
ReleaseDate = releaseDate,
ReleaseDatePrecision = precision,
IsCompilation = false,
Title = release.Comments
};
context.SoftwareReleases.Add(dbRelease);
await context.SaveChangesAsync();
// Barcodes
foreach(var barcode in release.Barcodes)
{
var barcodeType = barcode.Type switch
{
"UPC-A" => BarcodeType.UPC_A,
"EAN-13" => BarcodeType.EAN_13,
_ => BarcodeType.Unknown
};
// Check for existing barcode
bool exists = await context.SoftwareBarcodes
.AnyAsync(b => b.Code == barcode.Code);
if(!exists)
{
context.SoftwareBarcodes.Add(new SoftwareBarcode
{
ReleaseId = dbRelease.Id,
Code = barcode.Code,
Type = barcodeType
});
}
}
// Product codes
foreach(var productCode in release.ProductCodes)
{
var issuer = productCode.Type switch
{
"Sony PN" => ProductCodeIssuer.Sony,
"PSN/SEN Code" => ProductCodeIssuer.PSN,
"Microsoft PN" => ProductCodeIssuer.Microsoft,
"Nintendo PN" => ProductCodeIssuer.Nintendo,
"Sega PN" => ProductCodeIssuer.Sega,
"Activision PN" => ProductCodeIssuer.Activision,
"Amazon ASIN" => ProductCodeIssuer.Amazon,
"eBay Item No." => ProductCodeIssuer.eBay,
_ => ProductCodeIssuer.Other
};
context.SoftwareProductCodes.Add(new SoftwareProductCode
{
ReleaseId = dbRelease.Id,
Issuer = issuer,
Code = productCode.Code
});
}
// Countries → UnM49 regions
foreach(string country in release.Countries)
{
var region = _countryMatcher.Match(country);
if(region != null)
{
bool exists = await context.UnM49BySoftwareRelease
.AnyAsync(u => u.SoftwareReleaseId == dbRelease.Id &&
u.UnM49Id == region.Id);
if(!exists)
{
context.UnM49BySoftwareRelease.Add(new UnM49BySoftwareRelease
{
SoftwareReleaseId = dbRelease.Id,
UnM49Id = region.Id
});
}
}
}
// Distributor/Localizer company roles on the software
if(!string.IsNullOrWhiteSpace(release.Distributor))
{
var (dist, _) = await _companyMatcher.MatchOrCreateAsync(release.Distributor);
if(dist != null)
{
bool exists = await context.SoftwareCompanyRoles
.AnyAsync(r => r.SoftwareId == software.Id &&
r.CompanyId == dist.Id &&
r.RoleId == "dis");
if(!exists)
{
context.SoftwareCompanyRoles.Add(new SoftwareCompanyRole
{
SoftwareId = software.Id,
CompanyId = dist.Id,
RoleId = "dis"
});
}
}
}
if(!string.IsNullOrWhiteSpace(release.Localizer))
{
var (loc, _) = await _companyMatcher.MatchOrCreateAsync(release.Localizer);
if(loc != null)
{
bool exists = await context.SoftwareCompanyRoles
.AnyAsync(r => r.SoftwareId == software.Id &&
r.CompanyId == loc.Id &&
r.RoleId == "loc");
if(!exists)
{
context.SoftwareCompanyRoles.Add(new SoftwareCompanyRole
{
SoftwareId = software.Id,
CompanyId = loc.Id,
RoleId = "loc"
});
}
}
}
// Extra company roles from releases tab (e.g., "Ported by", etc.)
foreach(var (roleLabel, companyName) in release.CompanyRoles)
{
string roleId = MapRoleLabel(roleLabel);
if(roleId is null)
{
Console.WriteLine($" WARNING: Unrecognized company role \"{roleLabel}\" " +
$"for company \"{companyName}\" — skipping");
continue;
}
var (roleCompany, _) = await _companyMatcher.MatchOrCreateAsync(companyName);
if(roleCompany != null)
{
bool exists = await context.SoftwareCompanyRoles
.AnyAsync(r => r.SoftwareId == software.Id &&
r.CompanyId == roleCompany.Id &&
r.RoleId == roleId);
if(!exists)
{
context.SoftwareCompanyRoles.Add(new SoftwareCompanyRole
{
SoftwareId = software.Id,
CompanyId = roleCompany.Id,
RoleId = roleId
});
}
}
}
// Specs for this platform
foreach(var spec in game.Specs.Where(s => s.Platform == platformGroup.Key))
{
context.SoftwareAttributes.Add(new SoftwareAttribute
{
SoftwareReleaseId = dbRelease.Id,
Category = "Spec",
Key = spec.Key,
Value = spec.Value
});
}
// Ratings for this platform
foreach(var rating in game.Ratings.Where(r => r.Platform == platformGroup.Key))
{
string value = rating.Rating;
if(!string.IsNullOrWhiteSpace(rating.Descriptors))
value += $" ({rating.Descriptors})";
context.SoftwareAttributes.Add(new SoftwareAttribute
{
SoftwareReleaseId = dbRelease.Id,
Category = "Rating",
Key = rating.System,
Value = value
});
}
}
}
}
async Task ImportBasicReleaseAsync(MarechaiContext context, Software software, ParsedGame game)
{
// Create one release per platform from Main tab data
var (publisher, _) = await _companyMatcher.MatchOrCreateAsync(game.Publishers.FirstOrDefault());
if(publisher is null) return;
var (releaseDate, precision) = ParseDate(game.ReleaseDate);
foreach(string platformName in game.Platforms.Count > 0 ? game.Platforms : ["Unknown"])
{
var platform = await _platformMatcher.MatchOrCreateAsync(platformName);
var dbRelease = new SoftwareRelease
{
SoftwareId = software.Id,
PlatformId = platform?.Id,
PublisherId = publisher.Id,
ReleaseDate = releaseDate,
ReleaseDatePrecision = precision,
IsCompilation = false
};
context.SoftwareReleases.Add(dbRelease);
}
}
static (DateTime? date, DatePrecision precision) ParseDate(string dateStr)
{
if(string.IsNullOrWhiteSpace(dateStr))
return (null, DatePrecision.Full);
dateStr = dateStr.Trim();
// Full date: "Oct 07, 2010" or "Nov 14, 2007"
if(DateTime.TryParseExact(dateStr, "MMM dd, yyyy",
CultureInfo.InvariantCulture, DateTimeStyles.None, out var fullDate))
return (fullDate, DatePrecision.Full);
// Month+Year: "Oct, 2010" or "Dec 2007"
if(DateTime.TryParseExact(dateStr, "MMM, yyyy",
CultureInfo.InvariantCulture, DateTimeStyles.None, out var monthYear))
return (monthYear, DatePrecision.MonthYear);
if(DateTime.TryParseExact(dateStr, "MMM yyyy",
CultureInfo.InvariantCulture, DateTimeStyles.None, out monthYear))
return (monthYear, DatePrecision.MonthYear);
// Year only: "2008"
if(int.TryParse(dateStr, out int year) && year is > 1900 and < 2100)
return (new DateTime(year, 1, 1), DatePrecision.YearOnly);
return (null, DatePrecision.Full);
}
static string ExtractYear(string dateStr)
{
if(string.IsNullOrWhiteSpace(dateStr))
return null;
dateStr = dateStr.Trim();
// Full date: "Oct 07, 2010"
if(DateTime.TryParseExact(dateStr, "MMM dd, yyyy",
CultureInfo.InvariantCulture, DateTimeStyles.None, out var full))
return full.Year.ToString();
// Month+Year: "Oct, 2010" or "Dec 2007"
if(DateTime.TryParseExact(dateStr, "MMM, yyyy",
CultureInfo.InvariantCulture, DateTimeStyles.None, out var my))
return my.Year.ToString();
if(DateTime.TryParseExact(dateStr, "MMM yyyy",
CultureInfo.InvariantCulture, DateTimeStyles.None, out my))
return my.Year.ToString();
// Year only: "2008"
if(int.TryParse(dateStr, out int year) && year is > 1900 and < 2100)
return year.ToString();
return null;
}
/// <summary>
/// Maps a MobyGames company role label to a Marechai SoftwareRole code.
/// Returns null for unrecognized roles (which will be logged as warnings).
/// </summary>
static string MapRoleLabel(string roleLabel) => roleLabel switch
{
"Published by" => "pub",
"Developed by" => "dev",
"Distributed by" => "dis",
"Localized by" => "loc",
"Ported by" => "por",
"Manufactured by" => "mfg",
_ => null
};
}

View File

@@ -0,0 +1,109 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames.Services;
public class PersonMatcher
{
readonly IDbContextFactory<MarechaiContext> _contextFactory;
List<Person> _people;
Dictionary<string, List<Person>> _soundexIndex;
readonly Dictionary<string, Person> _cache = new(StringComparer.OrdinalIgnoreCase);
public PersonMatcher(IDbContextFactory<MarechaiContext> contextFactory) => _contextFactory = contextFactory;
public async Task LoadAsync()
{
await using var context = await _contextFactory.CreateDbContextAsync();
_people = await context.People
.Select(p => new Person
{
Id = p.Id,
Name = p.Name,
Surname = p.Surname,
DisplayName = p.DisplayName,
Alias = p.Alias
})
.ToListAsync();
_soundexIndex = SoundexHelper.BuildSoundexIndex(_people, p => p.FullName);
}
public async Task<Person> MatchOrCreateAsync(string fullName)
{
if(string.IsNullOrWhiteSpace(fullName))
return null;
string normalized = fullName.Replace("\u00a0", " ").Trim();
if(_cache.TryGetValue(normalized, out var cached))
return cached;
// Exact match on FullName, DisplayName, or Alias
var exact = _people.FirstOrDefault(p =>
string.Equals(p.FullName, normalized, StringComparison.OrdinalIgnoreCase) ||
string.Equals(p.DisplayName, normalized, StringComparison.OrdinalIgnoreCase) ||
string.Equals(p.Alias, normalized, StringComparison.OrdinalIgnoreCase));
if(exact != null)
{
_cache[normalized] = exact;
return exact;
}
// Soundex match
string soundex = SoundexHelper.Generate(normalized);
if(_soundexIndex.TryGetValue(soundex, out var candidates) && candidates.Count == 1)
{
_cache[normalized] = candidates[0];
return candidates[0];
}
// Split into name/surname for creation
string name, surname;
int lastSpace = normalized.LastIndexOf(' ');
if(lastSpace > 0)
{
name = normalized[..lastSpace].Trim();
surname = normalized[(lastSpace + 1)..].Trim();
}
else
{
name = normalized;
surname = "";
}
await using var context = await _contextFactory.CreateDbContextAsync();
var newPerson = new Person
{
Name = name,
Surname = surname
};
context.People.Add(newPerson);
await context.SaveChangesAsync();
_people.Add(newPerson);
if(!_soundexIndex.TryGetValue(soundex, out var list))
{
list = [];
_soundexIndex[soundex] = list;
}
list.Add(newPerson);
_cache[normalized] = newPerson;
return newPerson;
}
}

View File

@@ -0,0 +1,61 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames.Services;
public class PlatformMatcher
{
readonly IDbContextFactory<MarechaiContext> _contextFactory;
List<SoftwarePlatform> _platforms;
readonly Dictionary<string, SoftwarePlatform> _cache = new(StringComparer.OrdinalIgnoreCase);
public PlatformMatcher(IDbContextFactory<MarechaiContext> contextFactory) => _contextFactory = contextFactory;
public async Task LoadAsync()
{
await using var context = await _contextFactory.CreateDbContextAsync();
_platforms = await context.SoftwarePlatforms
.Select(p => new SoftwarePlatform { Id = p.Id, Name = p.Name })
.ToListAsync();
}
public async Task<SoftwarePlatform> MatchOrCreateAsync(string name)
{
if(string.IsNullOrWhiteSpace(name))
return null;
string normalized = name.Replace("\u00a0", " ").Trim();
if(_cache.TryGetValue(normalized, out var cached))
return cached;
var exact = _platforms.FirstOrDefault(p =>
string.Equals(p.Name, normalized, StringComparison.OrdinalIgnoreCase));
if(exact != null)
{
_cache[normalized] = exact;
return exact;
}
await using var context = await _contextFactory.CreateDbContextAsync();
var newPlatform = new SoftwarePlatform { Name = normalized };
context.SoftwarePlatforms.Add(newPlatform);
await context.SaveChangesAsync();
_platforms.Add(newPlatform);
_cache[normalized] = newPlatform;
Console.WriteLine($" Created new platform: \"{normalized}\" (ID: {newPlatform.Id})");
return newPlatform;
}
}

View File

@@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace Marechai.MobyGames.Services;
public static class SoundexHelper
{
static readonly char[] Map =
[
'0', '1', '2', '3', '0', '1', '2', '0', '0', '2', '2', '4', '5',
'5', '0', '1', '2', '6', '2', '3', '0', '1', '0', '2', '0', '2'
];
public static string Generate(string input)
{
if(string.IsNullOrWhiteSpace(input))
return "0000";
Span<char> result = stackalloc char[4];
result[0] = char.ToUpperInvariant(input[0]);
int resultIndex = 1;
char lastCode = GetCode(result[0]);
for(int i = 1; i < input.Length && resultIndex < 4; i++)
{
char c = input[i];
if(!char.IsLetter(c)) continue;
char code = GetCode(char.ToUpperInvariant(c));
if(code == '0' || code == lastCode)
{
lastCode = code;
continue;
}
result[resultIndex++] = code;
lastCode = code;
}
while(resultIndex < 4)
result[resultIndex++] = '0';
return new string(result);
}
static char GetCode(char c)
{
int index = char.ToUpperInvariant(c) - 'A';
return index is >= 0 and < 26 ? Map[index] : '0';
}
public static Dictionary<string, List<T>> BuildSoundexIndex<T>(
IEnumerable<T> items, Func<T, string> nameSelector)
{
var dict = new Dictionary<string, List<T>>();
foreach(var item in items)
{
string name = nameSelector(item);
if(string.IsNullOrWhiteSpace(name)) continue;
string code = Generate(name);
if(!dict.TryGetValue(code, out var list))
{
list = [];
dict[code] = list;
}
list.Add(item);
}
return dict;
}
}

View File

@@ -0,0 +1,95 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using Marechai.MobyGames.Models;
using MySqlConnector;
namespace Marechai.MobyGames.Services;
public class SourceDatabaseService
{
readonly string _connectionString;
public SourceDatabaseService(string connectionString) => _connectionString = connectionString;
public async Task<List<string>> GetDistinctGameIdsAsync(int limit, int offset = 0)
{
var ids = new List<string>();
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
await using var cmd = new MySqlCommand(
"SELECT DISTINCT id FROM mobygames_raw ORDER BY id LIMIT @limit OFFSET @offset", connection);
cmd.Parameters.AddWithValue("@limit", limit);
cmd.Parameters.AddWithValue("@offset", offset);
await using var reader = await cmd.ExecuteReaderAsync();
while(await reader.ReadAsync())
ids.Add(reader.GetString(0));
return ids;
}
public async Task<List<string>> GetUnprocessedGameIdsAsync(int batchSize, HashSet<string> processedIds)
{
var ids = new List<string>();
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
await using var cmd = new MySqlCommand(
"SELECT DISTINCT id FROM mobygames_raw ORDER BY id", connection);
await using var reader = await cmd.ExecuteReaderAsync();
while(await reader.ReadAsync() && ids.Count < batchSize)
{
string id = reader.GetString(0);
if(!processedIds.Contains(id))
ids.Add(id);
}
return ids;
}
public async Task<List<MobyGamesRawRow>> GetRowsForGameAsync(string gameId)
{
var rows = new List<MobyGamesRawRow>();
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
await using var cmd = new MySqlCommand(
"SELECT id, chunk, body FROM mobygames_raw WHERE id = @id ORDER BY chunk", connection);
cmd.Parameters.AddWithValue("@id", gameId);
await using var reader = await cmd.ExecuteReaderAsync();
while(await reader.ReadAsync())
{
rows.Add(new MobyGamesRawRow
{
Id = reader.GetString(0),
Chunk = reader.GetInt32(1),
Body = reader.GetString(2)
});
}
return rows;
}
public async Task<int> GetTotalGameCountAsync()
{
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
await using var cmd = new MySqlCommand("SELECT COUNT(DISTINCT id) FROM mobygames_raw", connection);
var result = await cmd.ExecuteScalarAsync();
return System.Convert.ToInt32(result);
}
}

View File

@@ -0,0 +1,158 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Marechai.Data;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames.Services;
public class StateService
{
readonly IDbContextFactory<MarechaiContext> _contextFactory;
public StateService(IDbContextFactory<MarechaiContext> contextFactory) => _contextFactory = contextFactory;
public async Task<HashSet<string>> GetProcessedGameIdsAsync()
{
await using var context = await _contextFactory.CreateDbContextAsync();
var ids = await context.MobyGamesImportStates
.Select(s => s.MobyGameId)
.ToListAsync();
return [..ids];
}
public async Task MarkImportedAsync(string mobyGameId, int batchNumber, ulong softwareId)
{
await using var context = await _contextFactory.CreateDbContextAsync();
var existing = await context.MobyGamesImportStates
.FirstOrDefaultAsync(s => s.MobyGameId == mobyGameId);
if(existing != null)
{
existing.Status = MobyGamesImportStatus.Imported;
existing.ProcessedOn = DateTime.UtcNow;
existing.BatchNumber = batchNumber;
existing.SoftwareId = softwareId;
}
else
{
context.MobyGamesImportStates.Add(new MobyGamesImportState
{
MobyGameId = mobyGameId,
Status = MobyGamesImportStatus.Imported,
ProcessedOn = DateTime.UtcNow,
BatchNumber = batchNumber,
SoftwareId = softwareId
});
}
await context.SaveChangesAsync();
}
public async Task MarkRejectedAsync(string mobyGameId, string gameName, string reason, int batchNumber)
{
await using var context = await _contextFactory.CreateDbContextAsync();
var existing = await context.MobyGamesImportStates
.FirstOrDefaultAsync(s => s.MobyGameId == mobyGameId);
if(existing != null)
{
existing.Status = MobyGamesImportStatus.Rejected;
existing.ProcessedOn = DateTime.UtcNow;
existing.BatchNumber = batchNumber;
}
else
{
context.MobyGamesImportStates.Add(new MobyGamesImportState
{
MobyGameId = mobyGameId,
Status = MobyGamesImportStatus.Rejected,
ProcessedOn = DateTime.UtcNow,
BatchNumber = batchNumber
});
}
context.MobyGamesRejections.Add(new MobyGamesRejection
{
MobyGameId = mobyGameId,
GameName = gameName ?? "Unknown",
Reason = reason,
RejectedOn = DateTime.UtcNow,
ReviewAction = MobyGamesRejectionReview.Pending
});
await context.SaveChangesAsync();
}
public async Task MarkFailedAsync(string mobyGameId, string error, int batchNumber)
{
await using var context = await _contextFactory.CreateDbContextAsync();
var existing = await context.MobyGamesImportStates
.FirstOrDefaultAsync(s => s.MobyGameId == mobyGameId);
if(existing != null)
{
existing.Status = MobyGamesImportStatus.Failed;
existing.ErrorMessage = error?[..Math.Min(error.Length, 1024)];
existing.ProcessedOn = DateTime.UtcNow;
existing.BatchNumber = batchNumber;
}
else
{
context.MobyGamesImportStates.Add(new MobyGamesImportState
{
MobyGameId = mobyGameId,
Status = MobyGamesImportStatus.Failed,
ErrorMessage = error?[..Math.Min(error?.Length ?? 0, 1024)],
ProcessedOn = DateTime.UtcNow,
BatchNumber = batchNumber
});
}
await context.SaveChangesAsync();
}
public async Task PrintStatusAsync()
{
await using var context = await _contextFactory.CreateDbContextAsync();
int pending = await context.MobyGamesImportStates.CountAsync(s => s.Status == MobyGamesImportStatus.Pending);
int imported = await context.MobyGamesImportStates.CountAsync(s => s.Status == MobyGamesImportStatus.Imported);
int rejected = await context.MobyGamesImportStates.CountAsync(s => s.Status == MobyGamesImportStatus.Rejected);
int failed = await context.MobyGamesImportStates.CountAsync(s => s.Status == MobyGamesImportStatus.Failed);
int total = pending + imported + rejected + failed;
Console.WriteLine($"\n Import Status:");
Console.WriteLine($" Imported: {imported}");
Console.WriteLine($" Rejected: {rejected}");
Console.WriteLine($" Failed: {failed}");
Console.WriteLine($" Pending: {pending}");
Console.WriteLine($" Total: {total}");
}
public async Task ResetGameAsync(string mobyGameId)
{
await using var context = await _contextFactory.CreateDbContextAsync();
var state = await context.MobyGamesImportStates
.FirstOrDefaultAsync(s => s.MobyGameId == mobyGameId);
if(state != null)
{
context.MobyGamesImportStates.Remove(state);
await context.SaveChangesAsync();
Console.WriteLine($" Reset game {mobyGameId} to unprocessed");
}
else
{
Console.WriteLine($" Game {mobyGameId} not found in import state");
}
}
}

View File

@@ -11,5 +11,6 @@
<Project Path="Marechai.Data/Marechai.Data.csproj" />
<Project Path="Marechai.Database/Marechai.Database.csproj" />
<Project Path="Marechai.Server/Marechai.Server.csproj" />
<Project Path="Marechai.MobyGames/Marechai.MobyGames.csproj" />
<Project Path="Marechai/Marechai.csproj" />
</Solution>