mirror of
https://github.com/claunia/marechai.git
synced 2026-07-08 17:57:08 +00:00
feat: Add IGDB game enrichment phase
Pull age ratings, franchise, involved companies, screenshots, summary, videos, releases and similar games into matched Software entries via a new enrich-games command, following the existing Mirror/Match/Enrich pipeline conventions.
This commit is contained in:
13354
Marechai.Database/Migrations/20260626184925_AddIgdbGameEnrichmentTracking.Designer.cs
generated
Normal file
13354
Marechai.Database/Migrations/20260626184925_AddIgdbGameEnrichmentTracking.Designer.cs
generated
Normal file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,51 @@
|
||||
using System;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace Marechai.Database.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddIgdbGameEnrichmentTracking : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.AddColumn<DateTime>(
|
||||
name: "EnrichedOn",
|
||||
table: "IgdbGames",
|
||||
type: "datetime(6)",
|
||||
nullable: true);
|
||||
|
||||
migrationBuilder.AddColumn<bool>(
|
||||
name: "EnrichmentApplied",
|
||||
table: "IgdbGames",
|
||||
type: "bit(1)",
|
||||
nullable: false,
|
||||
defaultValue: false);
|
||||
|
||||
migrationBuilder.AddColumn<string>(
|
||||
name: "ScreenshotImageIdsJson",
|
||||
table: "IgdbGames",
|
||||
type: "text",
|
||||
nullable: true)
|
||||
.Annotation("MySql:CharSet", "utf8mb4");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropColumn(
|
||||
name: "EnrichedOn",
|
||||
table: "IgdbGames");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "EnrichmentApplied",
|
||||
table: "IgdbGames");
|
||||
|
||||
migrationBuilder.DropColumn(
|
||||
name: "ScreenshotImageIdsJson",
|
||||
table: "IgdbGames");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2753,6 +2753,12 @@ namespace Marechai.Database.Migrations
|
||||
|
||||
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
|
||||
|
||||
b.Property<DateTime?>("EnrichedOn")
|
||||
.HasColumnType("datetime(6)");
|
||||
|
||||
b.Property<bool>("EnrichmentApplied")
|
||||
.HasColumnType("bit(1)");
|
||||
|
||||
b.Property<string>("ErrorMessage")
|
||||
.HasMaxLength(1024)
|
||||
.HasColumnType("varchar(1024)");
|
||||
@@ -2791,6 +2797,9 @@ namespace Marechai.Database.Migrations
|
||||
b.Property<double?>("PlatformOverlapScore")
|
||||
.HasColumnType("double");
|
||||
|
||||
b.Property<string>("ScreenshotImageIdsJson")
|
||||
.HasColumnType("text");
|
||||
|
||||
b.Property<string>("Slug")
|
||||
.HasMaxLength(255)
|
||||
.HasColumnType("varchar(255)");
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
using System.ComponentModel.DataAnnotations.Schema;
|
||||
using Marechai.Data;
|
||||
|
||||
namespace Marechai.Database.Models;
|
||||
@@ -55,4 +56,13 @@ public class IgdbGame : BaseModel<long>
|
||||
public int BatchNumber { get; set; }
|
||||
|
||||
public ulong? SoftwareId { get; set; }
|
||||
|
||||
public bool EnrichmentApplied { get; set; }
|
||||
|
||||
public DateTime? EnrichedOn { get; set; }
|
||||
|
||||
/// <summary>Raw JSON array of IGDB screenshot <c>image_id</c> strings, cached here for a later
|
||||
/// screenshot-download phase that has not been implemented yet.</summary>
|
||||
[Column(TypeName = "text")]
|
||||
public string ScreenshotImageIdsJson { get; set; }
|
||||
}
|
||||
|
||||
@@ -225,6 +225,43 @@ class Program
|
||||
break;
|
||||
}
|
||||
|
||||
case "enrich-games":
|
||||
{
|
||||
IgdbHttpClient client = RequireClient();
|
||||
|
||||
if(client == null)
|
||||
return 1;
|
||||
|
||||
string assetRootPath = config.GetValue<string>("AssetRootPath");
|
||||
|
||||
if(string.IsNullOrEmpty(assetRootPath))
|
||||
{
|
||||
Console.WriteLine("\e[31;1mMissing AssetRootPath in appsettings.json\e[0m");
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
var enricher = new GameEnricherService(factory, client, assetRootPath);
|
||||
GameEnricherService.Stats stats = await enricher.RunAsync(dryRun);
|
||||
|
||||
Console.WriteLine($"""
|
||||
|
||||
Done. Processed {stats.GamesProcessed} games ({stats.GamesFailed} failed).
|
||||
Families created: {stats.FamiliesCreated}
|
||||
Families linked: {stats.FamiliesLinked}
|
||||
Company roles added: {stats.CompanyRolesAdded}
|
||||
Descriptions added: {stats.DescriptionsAdded}
|
||||
Videos added: {stats.VideosAdded}
|
||||
Releases added: {stats.ReleasesAdded}
|
||||
Platforms auto-created: {stats.PlatformsAutoCreated}
|
||||
Ratings added: {stats.RatingsAdded}
|
||||
Screenshot ids cached: {stats.ScreenshotsCached}
|
||||
Similar-software links: {stats.SimilarLinksAdded}
|
||||
""");
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case "stats":
|
||||
{
|
||||
var stats = new MatchStatsService(factory);
|
||||
@@ -258,7 +295,7 @@ class Program
|
||||
|
||||
if(igdbId == null || type == null)
|
||||
{
|
||||
Console.WriteLine(" --igdb-id <id> and --type companies|companies-enrichment|games|platforms|platforms-enrichment are required.");
|
||||
Console.WriteLine(" --igdb-id <id> and --type companies|companies-enrichment|games|games-enrichment|platforms|platforms-enrichment are required.");
|
||||
|
||||
return 1;
|
||||
}
|
||||
@@ -354,6 +391,20 @@ class Program
|
||||
break;
|
||||
}
|
||||
|
||||
case "games-enrichment":
|
||||
{
|
||||
IgdbGame entity = await context.IgdbGames.FirstOrDefaultAsync(g => g.IgdbId == igdbId);
|
||||
|
||||
if(entity != null)
|
||||
{
|
||||
entity.EnrichmentApplied = false;
|
||||
entity.EnrichedOn = null;
|
||||
entity.ScreenshotImageIdsJson = null;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
case "games":
|
||||
{
|
||||
IgdbGame entity = await context.IgdbGames.FirstOrDefaultAsync(g => g.IgdbId == igdbId);
|
||||
@@ -417,9 +468,11 @@ class Program
|
||||
match-games Match mirrored games against Software
|
||||
enrich-companies Fill empty Company fields from matched IGDB data (resumable)
|
||||
enrich-platforms Download missing SoftwarePlatform logos from matched IGDB data (resumable)
|
||||
enrich-games Fill age ratings, franchise, companies, description, videos, releases,
|
||||
similar games from matched IGDB data (resumable)
|
||||
stats Print match status counts
|
||||
review-ambiguous --type <companies|games|platforms> [--limit N]
|
||||
reset --igdb-id <id> --type <companies|companies-enrichment|games|platforms|platforms-enrichment>
|
||||
reset --igdb-id <id> --type <companies|companies-enrichment|games|games-enrichment|platforms|platforms-enrichment>
|
||||
|
||||
Options:
|
||||
--batch-size N Mirror one batch of up to N rows (default Import:BatchSize).
|
||||
|
||||
774
Marechai.Igdb/Services/GameEnricherService.cs
Normal file
774
Marechai.Igdb/Services/GameEnricherService.cs
Normal file
@@ -0,0 +1,774 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Tasks;
|
||||
using Markdig;
|
||||
using Marechai.Data;
|
||||
using Marechai.Database.Models;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Marechai.Igdb.Services;
|
||||
|
||||
/// <summary>Fills in the richer per-game fields IGDB exposes (age ratings, franchise, involved companies,
|
||||
/// screenshots, slug, summary, videos, release dates, similar games) for <see cref="Software" /> entries already
|
||||
/// matched to an <see cref="IgdbGame" />. Resumable and idempotent like <see cref="CompanyEnricherService" /> /
|
||||
/// <see cref="PlatformEnricherService" />: each game is marked <see cref="IgdbGame.EnrichmentApplied" /> once
|
||||
/// processed and skipped on subsequent runs.</summary>
|
||||
public class GameEnricherService
|
||||
{
|
||||
const int GameBatchSize = 50;
|
||||
const int IgdbMaxPageSize = 500;
|
||||
|
||||
static readonly Dictionary<string, string> InvolvedCompanyRoleMap = new()
|
||||
{
|
||||
["developer"] = "dev",
|
||||
["publisher"] = "pub",
|
||||
["porting"] = "por",
|
||||
["supporting"] = "ctb"
|
||||
};
|
||||
|
||||
readonly IDbContextFactory<MarechaiContext> _contextFactory;
|
||||
readonly IgdbHttpClient _client;
|
||||
readonly HttpClient _httpClient;
|
||||
readonly string _assetRootPath;
|
||||
|
||||
public GameEnricherService(IDbContextFactory<MarechaiContext> contextFactory, IgdbHttpClient client,
|
||||
string assetRootPath)
|
||||
{
|
||||
_contextFactory = contextFactory;
|
||||
_client = client;
|
||||
_assetRootPath = assetRootPath;
|
||||
_httpClient = new HttpClient();
|
||||
}
|
||||
|
||||
public class Stats
|
||||
{
|
||||
public int GamesProcessed;
|
||||
public int GamesFailed;
|
||||
public int FamiliesCreated;
|
||||
public int FamiliesLinked;
|
||||
public int CompanyRolesAdded;
|
||||
public int DescriptionsAdded;
|
||||
public int VideosAdded;
|
||||
public int ReleasesAdded;
|
||||
public int PlatformsAutoCreated;
|
||||
public int RatingsAdded;
|
||||
public int ScreenshotsCached;
|
||||
public int SimilarLinksAdded;
|
||||
}
|
||||
|
||||
public async Task<Stats> RunAsync(bool dryRun)
|
||||
{
|
||||
var stats = new Stats();
|
||||
|
||||
List<long> pendingIgdbIds;
|
||||
|
||||
await using(var probe = await _contextFactory.CreateDbContextAsync())
|
||||
{
|
||||
pendingIgdbIds = await probe.IgdbGames
|
||||
.Where(g => g.MatchStatus == IgdbMatchStatus.Matched &&
|
||||
g.SoftwareId != null && !g.EnrichmentApplied)
|
||||
.OrderBy(g => g.IgdbId)
|
||||
.Select(g => g.IgdbId)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
if(pendingIgdbIds.Count == 0)
|
||||
{
|
||||
Console.WriteLine(" No matched games pending enrichment.");
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
Dictionary<int, string> orgNames = await FetchLookupAsync("age_rating_organizations", "name");
|
||||
Dictionary<int, string> ratingTexts = await FetchLookupAsync("age_rating_categories", "rating");
|
||||
|
||||
for(int offset = 0; offset < pendingIgdbIds.Count; offset += GameBatchSize)
|
||||
{
|
||||
List<long> batchIds = pendingIgdbIds.Skip(offset).Take(GameBatchSize).ToList();
|
||||
await ProcessBatchAsync(batchIds, orgNames, ratingTexts, dryRun, stats);
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
async Task ProcessBatchAsync(List<long> batchIds, Dictionary<int, string> orgNames,
|
||||
Dictionary<int, string> ratingTexts, bool dryRun, Stats stats)
|
||||
{
|
||||
List<JsonElement> games = await FetchByFieldAsync("games",
|
||||
"id,slug,summary,age_ratings,franchises,screenshots,videos,release_dates,similar_games", "id",
|
||||
batchIds);
|
||||
|
||||
var gamesById = games.ToDictionary(g => g.GetProperty("id").GetInt64());
|
||||
|
||||
var allScreenshotIds = new List<long>();
|
||||
var allVideoIds = new List<long>();
|
||||
var allReleaseIds = new List<long>();
|
||||
var allAgeRatingIds = new List<long>();
|
||||
var allFranchiseIds = new List<long>();
|
||||
|
||||
foreach(JsonElement g in games)
|
||||
{
|
||||
allScreenshotIds.AddRange(GetLongArray(g, "screenshots"));
|
||||
allVideoIds.AddRange(GetLongArray(g, "videos"));
|
||||
allReleaseIds.AddRange(GetLongArray(g, "release_dates"));
|
||||
allAgeRatingIds.AddRange(GetLongArray(g, "age_ratings"));
|
||||
allFranchiseIds.AddRange(GetLongArray(g, "franchises"));
|
||||
}
|
||||
|
||||
List<JsonElement> involvedCompanies = await FetchByFieldAsync("involved_companies",
|
||||
"id,game,company,developer,publisher,porting,supporting", "game", batchIds);
|
||||
|
||||
List<JsonElement> screenshots = await FetchByFieldAsync("screenshots", "id,image_id", "id",
|
||||
allScreenshotIds);
|
||||
|
||||
List<JsonElement> videos = await FetchByFieldAsync("game_videos", "id,video_id,name", "id", allVideoIds);
|
||||
|
||||
List<JsonElement> releaseDates = await FetchByFieldAsync("release_dates", "id,date,platform,date_format",
|
||||
"id", allReleaseIds);
|
||||
|
||||
List<JsonElement> ageRatings = await FetchByFieldAsync("age_ratings", "id,organization,rating_category",
|
||||
"id", allAgeRatingIds);
|
||||
|
||||
List<JsonElement> franchises = await FetchByFieldAsync("franchises", "id,name", "id", allFranchiseIds);
|
||||
|
||||
var involvedByGame = involvedCompanies.GroupBy(e => e.GetProperty("game").GetInt64())
|
||||
.ToDictionary(gr => gr.Key, gr => gr.ToList());
|
||||
var screenshotsById = screenshots.ToDictionary(e => e.GetProperty("id").GetInt64());
|
||||
var videosById = videos.ToDictionary(e => e.GetProperty("id").GetInt64());
|
||||
var releasesById = releaseDates.ToDictionary(e => e.GetProperty("id").GetInt64());
|
||||
var ageRatingsById = ageRatings.ToDictionary(e => e.GetProperty("id").GetInt64());
|
||||
var franchisesById = franchises.ToDictionary(e => e.GetProperty("id").GetInt64());
|
||||
|
||||
foreach(long igdbId in batchIds)
|
||||
{
|
||||
if(!gamesById.TryGetValue(igdbId, out JsonElement game))
|
||||
continue;
|
||||
|
||||
try
|
||||
{
|
||||
await ProcessGameAsync(igdbId, game, involvedByGame, screenshotsById, videosById, releasesById,
|
||||
ageRatingsById, franchisesById, orgNames, ratingTexts, dryRun, stats);
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
stats.GamesFailed++;
|
||||
Console.WriteLine($" [{igdbId}] ERROR: {ex.Message}");
|
||||
|
||||
if(!dryRun)
|
||||
{
|
||||
await using var errContext = await _contextFactory.CreateDbContextAsync();
|
||||
IgdbGame errGame = await errContext.IgdbGames.FirstOrDefaultAsync(x => x.IgdbId == igdbId);
|
||||
|
||||
if(errGame != null)
|
||||
{
|
||||
errGame.ErrorMessage = ex.Message.Length > 1024 ? ex.Message[..1024] : ex.Message;
|
||||
await errContext.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async Task ProcessGameAsync(long igdbId, JsonElement game, Dictionary<long, List<JsonElement>> involvedByGame,
|
||||
Dictionary<long, JsonElement> screenshotsById,
|
||||
Dictionary<long, JsonElement> videosById, Dictionary<long, JsonElement> releasesById,
|
||||
Dictionary<long, JsonElement> ageRatingsById,
|
||||
Dictionary<long, JsonElement> franchisesById, Dictionary<int, string> orgNames,
|
||||
Dictionary<int, string> ratingTexts, bool dryRun, Stats stats)
|
||||
{
|
||||
await using var context = await _contextFactory.CreateDbContextAsync();
|
||||
|
||||
IgdbGame igdbGame = await context.IgdbGames.FirstOrDefaultAsync(g => g.IgdbId == igdbId);
|
||||
|
||||
if(igdbGame?.SoftwareId == null)
|
||||
return;
|
||||
|
||||
Software software = await context.Softwares.Include(s => s.Descriptions).Include(s => s.Videos)
|
||||
.Include(s => s.CompanyRoles)
|
||||
.Include(s => s.DirectReleases).ThenInclude(r => r.Attributes)
|
||||
.FirstOrDefaultAsync(s => s.Id == igdbGame.SoftwareId);
|
||||
|
||||
if(software == null)
|
||||
{
|
||||
Console.WriteLine($" [{igdbId}] No local software {igdbGame.SoftwareId} found, skipping.");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
stats.GamesProcessed++;
|
||||
|
||||
// -- Franchise -> SoftwareFamily --------------------------------------------------------
|
||||
List<long> franchiseIds = GetLongArray(game, "franchises");
|
||||
|
||||
if(franchiseIds.Count > 0 && software.FamilyId == null)
|
||||
{
|
||||
if(franchiseIds.Count > 1)
|
||||
Console.WriteLine(
|
||||
$" [{igdbId}] Game has {franchiseIds.Count} franchises, using only the first.");
|
||||
|
||||
if(franchisesById.TryGetValue(franchiseIds[0], out JsonElement franchise) &&
|
||||
franchise.TryGetProperty("name", out JsonElement franchiseNameEl) &&
|
||||
franchiseNameEl.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
string franchiseName = franchiseNameEl.GetString();
|
||||
|
||||
SoftwareFamily family = await context.SoftwareFamilies
|
||||
.FirstOrDefaultAsync(f => f.Name == franchiseName);
|
||||
|
||||
if(family == null)
|
||||
{
|
||||
family = new SoftwareFamily { Name = franchiseName };
|
||||
context.SoftwareFamilies.Add(family);
|
||||
|
||||
if(!dryRun)
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
stats.FamiliesCreated++;
|
||||
Console.WriteLine($" [{igdbId}] Created software family \"{franchiseName}\".");
|
||||
}
|
||||
|
||||
software.FamilyId = family.Id;
|
||||
stats.FamiliesLinked++;
|
||||
Console.WriteLine($" [{igdbId}] Linked software family \"{franchiseName}\".");
|
||||
}
|
||||
}
|
||||
|
||||
// -- Involved companies -> SoftwareCompanyRole -------------------------------------------
|
||||
var existingRoles = new HashSet<(int CompanyId, string RoleId)>(software.CompanyRoles
|
||||
.Select(r => (r.CompanyId, r.RoleId)));
|
||||
|
||||
if(involvedByGame.TryGetValue(igdbId, out List<JsonElement> involved))
|
||||
{
|
||||
foreach(JsonElement ic in involved)
|
||||
{
|
||||
long companyIgdbId = ic.GetProperty("company").GetInt64();
|
||||
|
||||
IgdbCompany igdbCompany = await context.IgdbCompanies
|
||||
.FirstOrDefaultAsync(c => c.IgdbId == companyIgdbId &&
|
||||
c.MatchStatus == IgdbMatchStatus.Matched &&
|
||||
c.CompanyId != null);
|
||||
|
||||
if(igdbCompany == null)
|
||||
{
|
||||
IgdbCompany unmatched =
|
||||
await context.IgdbCompanies.FirstOrDefaultAsync(c => c.IgdbId == companyIgdbId);
|
||||
|
||||
Console.WriteLine(
|
||||
$" [{igdbId}] Involved company \"{unmatched?.Name ?? "unknown"}\" (igdb id {companyIgdbId}) has no local match, skipping.");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
int companyId = igdbCompany.CompanyId.Value;
|
||||
|
||||
bool anyRoleToAdd = InvolvedCompanyRoleMap.Any(kv =>
|
||||
ic.TryGetProperty(kv.Key, out JsonElement f) && f.ValueKind == JsonValueKind.True &&
|
||||
!existingRoles.Contains((companyId, kv.Value)));
|
||||
|
||||
string companyName = anyRoleToAdd
|
||||
? (await context.Companies.FirstOrDefaultAsync(c => c.Id == companyId))
|
||||
?.Name ?? companyId.ToString()
|
||||
: null;
|
||||
|
||||
foreach((string flag, string roleId) in InvolvedCompanyRoleMap)
|
||||
{
|
||||
if(!ic.TryGetProperty(flag, out JsonElement flagEl) || flagEl.ValueKind != JsonValueKind.True)
|
||||
continue;
|
||||
|
||||
if(existingRoles.Contains((companyId, roleId)))
|
||||
continue;
|
||||
|
||||
context.SoftwareCompanyRoles.Add(new SoftwareCompanyRole
|
||||
{
|
||||
SoftwareId = software.Id, CompanyId = companyId, RoleId = roleId
|
||||
});
|
||||
|
||||
existingRoles.Add((companyId, roleId));
|
||||
stats.CompanyRolesAdded++;
|
||||
Console.WriteLine($" [{igdbId}] Added company role {roleId} for company \"{companyName}\" (id {companyId}).");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// -- Screenshots (ids cached for a later download phase) --------------------------------
|
||||
List<long> screenshotIds = GetLongArray(game, "screenshots");
|
||||
|
||||
if(screenshotIds.Count > 0 && string.IsNullOrWhiteSpace(igdbGame.ScreenshotImageIdsJson))
|
||||
{
|
||||
List<string> imageIds = screenshotIds.Select(id => screenshotsById.TryGetValue(id, out JsonElement s) &&
|
||||
s.TryGetProperty("image_id", out JsonElement iid) &&
|
||||
iid.ValueKind == JsonValueKind.String
|
||||
? iid.GetString()
|
||||
: null).Where(s => s != null).ToList();
|
||||
|
||||
if(imageIds.Count > 0)
|
||||
{
|
||||
igdbGame.ScreenshotImageIdsJson = JsonSerializer.Serialize(imageIds);
|
||||
stats.ScreenshotsCached += imageIds.Count;
|
||||
Console.WriteLine($" [{igdbId}] Cached {imageIds.Count} screenshot image ids.");
|
||||
}
|
||||
}
|
||||
|
||||
// -- Description from summary ------------------------------------------------------------
|
||||
if(game.TryGetProperty("summary", out JsonElement summaryEl) &&
|
||||
summaryEl.ValueKind == JsonValueKind.String &&
|
||||
!string.IsNullOrWhiteSpace(summaryEl.GetString()) &&
|
||||
!software.Descriptions.Any(d => d.LanguageCode == "eng"))
|
||||
{
|
||||
string markdown = summaryEl.GetString().Replace("\r\n", "\n").Replace("\n", "\n\n");
|
||||
|
||||
software.Descriptions.Add(new SoftwareDescription
|
||||
{
|
||||
LanguageCode = "eng",
|
||||
Text = markdown,
|
||||
Html = Markdown.ToHtml(markdown, new MarkdownPipelineBuilder().UseAdvancedExtensions().Build())
|
||||
});
|
||||
|
||||
stats.DescriptionsAdded++;
|
||||
Console.WriteLine($" [{igdbId}] Description added.");
|
||||
}
|
||||
|
||||
// -- Videos --------------------------------------------------------------------------------
|
||||
var existingVideos = new HashSet<string>(software.Videos.Where(v => v.Provider == "YouTube")
|
||||
.Select(v => v.VideoId));
|
||||
|
||||
foreach(long videoId in GetLongArray(game, "videos"))
|
||||
{
|
||||
if(!videosById.TryGetValue(videoId, out JsonElement v))
|
||||
continue;
|
||||
|
||||
if(!v.TryGetProperty("video_id", out JsonElement vidEl) || vidEl.ValueKind != JsonValueKind.String)
|
||||
continue;
|
||||
|
||||
string ytId = vidEl.GetString();
|
||||
|
||||
if(existingVideos.Contains(ytId))
|
||||
continue;
|
||||
|
||||
string title = v.TryGetProperty("name", out JsonElement nameEl) && nameEl.ValueKind == JsonValueKind.String
|
||||
? nameEl.GetString()
|
||||
: null;
|
||||
|
||||
software.Videos.Add(new SoftwareVideo { Provider = "YouTube", VideoId = ytId, Title = title });
|
||||
existingVideos.Add(ytId);
|
||||
stats.VideosAdded++;
|
||||
Console.WriteLine($" [{igdbId}] Video added: {ytId}.");
|
||||
}
|
||||
|
||||
// -- Releases from release_dates -----------------------------------------------------------
|
||||
int? firstPublisherCompanyId = null;
|
||||
|
||||
if(involvedByGame.TryGetValue(igdbId, out List<JsonElement> involvedForPublisher))
|
||||
foreach(JsonElement ic in involvedForPublisher)
|
||||
if(ic.TryGetProperty("publisher", out JsonElement pubEl) && pubEl.ValueKind == JsonValueKind.True)
|
||||
{
|
||||
long companyIgdbId = ic.GetProperty("company").GetInt64();
|
||||
|
||||
IgdbCompany pubCompany = await context.IgdbCompanies
|
||||
.FirstOrDefaultAsync(c => c.IgdbId == companyIgdbId &&
|
||||
c.MatchStatus == IgdbMatchStatus.Matched &&
|
||||
c.CompanyId != null);
|
||||
|
||||
if(pubCompany != null)
|
||||
{
|
||||
firstPublisherCompanyId = pubCompany.CompanyId;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
var existingReleases = new HashSet<(ulong? PlatformId, DateTime Date)>(software.DirectReleases
|
||||
.Where(r => r.ReleaseDate != null).Select(r => (r.PlatformId, r.ReleaseDate.Value.Date)));
|
||||
|
||||
foreach(long releaseDateId in GetLongArray(game, "release_dates"))
|
||||
{
|
||||
if(!releasesById.TryGetValue(releaseDateId, out JsonElement rd))
|
||||
continue;
|
||||
|
||||
if(!rd.TryGetProperty("date", out JsonElement dateEl) || dateEl.ValueKind != JsonValueKind.Number)
|
||||
continue;
|
||||
|
||||
if(!rd.TryGetProperty("platform", out JsonElement platformEl) ||
|
||||
platformEl.ValueKind != JsonValueKind.Number)
|
||||
continue;
|
||||
|
||||
long platformIgdbId = platformEl.GetInt64();
|
||||
|
||||
IgdbPlatform igdbPlatform =
|
||||
await context.IgdbPlatforms.FirstOrDefaultAsync(p => p.Id == (int) platformIgdbId);
|
||||
|
||||
if(igdbPlatform == null)
|
||||
{
|
||||
igdbPlatform = await AutoCreatePlatformAsync(context, platformIgdbId, dryRun, stats, igdbId);
|
||||
|
||||
if(igdbPlatform == null)
|
||||
continue;
|
||||
}
|
||||
else if(igdbPlatform.SoftwarePlatformId == null)
|
||||
{
|
||||
Console.WriteLine(
|
||||
$" [{igdbId}] Platform {platformIgdbId} ({igdbPlatform.Name}) is mirrored but not matched, skipping release.");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if(firstPublisherCompanyId == null)
|
||||
{
|
||||
Console.WriteLine(
|
||||
$" [{igdbId}] No resolved publisher for release on platform \"{igdbPlatform.Name}\" (id {platformIgdbId}), skipping.");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
DateTime releaseDate = DateTimeOffset.FromUnixTimeSeconds(dateEl.GetInt64()).UtcDateTime;
|
||||
|
||||
int? dateFormat = rd.TryGetProperty("date_format", out JsonElement dfEl) &&
|
||||
dfEl.ValueKind == JsonValueKind.Number
|
||||
? dfEl.GetInt32()
|
||||
: null;
|
||||
|
||||
DatePrecision precision = ToDatePrecision(dateFormat);
|
||||
|
||||
ulong? localPlatformId = igdbPlatform.SoftwarePlatformId;
|
||||
|
||||
if(existingReleases.Contains((localPlatformId, releaseDate.Date)))
|
||||
continue;
|
||||
|
||||
var newRelease = new SoftwareRelease
|
||||
{
|
||||
SoftwareId = software.Id,
|
||||
PlatformId = localPlatformId,
|
||||
PublisherId = firstPublisherCompanyId.Value,
|
||||
ReleaseDate = releaseDate,
|
||||
ReleaseDatePrecision = precision,
|
||||
Attributes = []
|
||||
};
|
||||
|
||||
software.DirectReleases.Add(newRelease);
|
||||
existingReleases.Add((localPlatformId, releaseDate.Date));
|
||||
stats.ReleasesAdded++;
|
||||
Console.WriteLine(
|
||||
$" [{igdbId}] Release added: platform \"{igdbPlatform.Name}\" (id {localPlatformId}), date {releaseDate:yyyy-MM-dd}.");
|
||||
}
|
||||
|
||||
// -- Age ratings (after releases exist) ----------------------------------------------------
|
||||
var ratingsToApply = new List<(string Key, string Value)>();
|
||||
|
||||
foreach(long ageRatingId in GetLongArray(game, "age_ratings"))
|
||||
{
|
||||
if(!ageRatingsById.TryGetValue(ageRatingId, out JsonElement ar))
|
||||
continue;
|
||||
|
||||
if(!ar.TryGetProperty("organization", out JsonElement orgEl) || orgEl.ValueKind != JsonValueKind.Number)
|
||||
continue;
|
||||
|
||||
if(!ar.TryGetProperty("rating_category", out JsonElement catEl) ||
|
||||
catEl.ValueKind != JsonValueKind.Number)
|
||||
continue;
|
||||
|
||||
if(!orgNames.TryGetValue(orgEl.GetInt32(), out string orgName))
|
||||
{
|
||||
Console.WriteLine($" [{igdbId}] Unknown age rating organization {orgEl.GetInt32()}, skipping.");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!ratingTexts.TryGetValue(catEl.GetInt32(), out string ratingText))
|
||||
{
|
||||
Console.WriteLine($" [{igdbId}] Unknown age rating category {catEl.GetInt32()}, skipping.");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
ratingsToApply.Add((orgName, ratingText));
|
||||
}
|
||||
|
||||
if(ratingsToApply.Count > 0)
|
||||
foreach(SoftwareRelease release in software.DirectReleases)
|
||||
{
|
||||
var existingAttrs = new HashSet<string>(release.Attributes.Where(a => a.Category == "rating")
|
||||
.Select(a => a.Key));
|
||||
|
||||
foreach((string key, string value) in ratingsToApply)
|
||||
{
|
||||
if(existingAttrs.Contains(key))
|
||||
continue;
|
||||
|
||||
release.Attributes.Add(new SoftwareAttribute { Category = "rating", Key = key, Value = value });
|
||||
existingAttrs.Add(key);
|
||||
stats.RatingsAdded++;
|
||||
|
||||
string releaseLabel = release.Id == 0
|
||||
? $"new release on platform {release.PlatformId}, {release.ReleaseDate:yyyy-MM-dd}"
|
||||
: $"release {release.Id}";
|
||||
|
||||
Console.WriteLine($" [{igdbId}] Rating added to {releaseLabel}: {key} = {value}.");
|
||||
}
|
||||
}
|
||||
|
||||
// -- Similar games -> SoftwareSimilarTo ----------------------------------------------------
|
||||
List<long> similarGameIds = GetLongArray(game, "similar_games");
|
||||
|
||||
if(similarGameIds.Count > 0)
|
||||
{
|
||||
List<ulong> similarSoftwareIds = await context.IgdbGames
|
||||
.Where(g => similarGameIds.Contains(g.IgdbId) &&
|
||||
g.MatchStatus == IgdbMatchStatus.Matched &&
|
||||
g.SoftwareId != null)
|
||||
.Select(g => g.SoftwareId.Value).ToListAsync();
|
||||
|
||||
var existingPairs = new HashSet<(ulong, ulong)>();
|
||||
|
||||
List<SoftwareSimilarTo> existingLinks = await context.SoftwareSimilarTo
|
||||
.Where(s => s.SoftwareId == software.Id ||
|
||||
s.SimilarSoftwareId == software.Id)
|
||||
.ToListAsync();
|
||||
|
||||
foreach(SoftwareSimilarTo link in existingLinks)
|
||||
existingPairs.Add((link.SoftwareId, link.SimilarSoftwareId));
|
||||
|
||||
foreach(ulong otherId in similarSoftwareIds.Distinct())
|
||||
{
|
||||
if(otherId == software.Id)
|
||||
continue;
|
||||
|
||||
ulong lo = Math.Min(software.Id, otherId);
|
||||
ulong hi = Math.Max(software.Id, otherId);
|
||||
|
||||
if(existingPairs.Contains((lo, hi)))
|
||||
continue;
|
||||
|
||||
context.SoftwareSimilarTo.Add(new SoftwareSimilarTo { SoftwareId = lo, SimilarSoftwareId = hi });
|
||||
existingPairs.Add((lo, hi));
|
||||
stats.SimilarLinksAdded++;
|
||||
Console.WriteLine($" [{igdbId}] Similar software link added: {lo} <-> {hi}.");
|
||||
}
|
||||
}
|
||||
|
||||
if(!dryRun)
|
||||
{
|
||||
igdbGame.EnrichmentApplied = true;
|
||||
igdbGame.EnrichedOn = DateTime.UtcNow;
|
||||
await context.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
|
||||
async Task<IgdbPlatform> AutoCreatePlatformAsync(MarechaiContext context, long platformIgdbId, bool dryRun,
|
||||
Stats stats, long gameIgdbId)
|
||||
{
|
||||
string query = new ApicalypseQueryBuilder().Fields("id,name,platform_logo.image_id")
|
||||
.Where($"id = {platformIgdbId}").Build();
|
||||
|
||||
using JsonDocument doc = await _client.QueryAsync("platforms", query);
|
||||
|
||||
if(doc.RootElement.GetArrayLength() == 0)
|
||||
{
|
||||
Console.WriteLine($" [{gameIgdbId}] IGDB platform {platformIgdbId} could not be fetched, skipping release.");
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
JsonElement element = doc.RootElement[0];
|
||||
string name = element.GetProperty("name").GetString();
|
||||
|
||||
string logoImageId = element.TryGetProperty("platform_logo", out JsonElement logo) &&
|
||||
logo.TryGetProperty("image_id", out JsonElement imageId)
|
||||
? imageId.GetString()
|
||||
: null;
|
||||
|
||||
var softwarePlatform = new SoftwarePlatform { Name = name };
|
||||
context.SoftwarePlatforms.Add(softwarePlatform);
|
||||
|
||||
if(!dryRun)
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
var igdbPlatform = new IgdbPlatform
|
||||
{
|
||||
Id = (int) platformIgdbId,
|
||||
Name = name,
|
||||
MatchStatus = IgdbMatchStatus.Matched,
|
||||
MatchType = "igdb-game-enrich-autocreate",
|
||||
MatchedOn = DateTime.UtcNow,
|
||||
SoftwarePlatformId = softwarePlatform.Id,
|
||||
LogoImageId = logoImageId
|
||||
};
|
||||
|
||||
context.IgdbPlatforms.Add(igdbPlatform);
|
||||
stats.PlatformsAutoCreated++;
|
||||
Console.WriteLine($" [{gameIgdbId}] Auto-created platform \"{name}\" (igdb id {platformIgdbId}).");
|
||||
|
||||
if(!string.IsNullOrWhiteSpace(logoImageId) && !dryRun)
|
||||
{
|
||||
await DownloadAndApplyLogoAsync(platformIgdbId, softwarePlatform, logoImageId);
|
||||
igdbPlatform.EnrichmentApplied = true;
|
||||
igdbPlatform.EnrichedOn = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
if(!dryRun)
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
return igdbPlatform;
|
||||
}
|
||||
|
||||
async Task DownloadAndApplyLogoAsync(long igdbId, SoftwarePlatform platform, string imageId)
|
||||
{
|
||||
byte[] bytes = await IgdbImageDownloader.DownloadBestAsync(_httpClient, imageId,
|
||||
IgdbImageDownloader.LogoSizeCandidates);
|
||||
|
||||
if(bytes == null)
|
||||
{
|
||||
Console.WriteLine($" [{igdbId}] Failed to download logo: no candidate size succeeded.");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
var guid = Guid.NewGuid();
|
||||
|
||||
string itemPhotosRoot = Path.Combine(_assetRootPath, "photos", "platform-logos");
|
||||
string itemThumbsRoot = Path.Combine(itemPhotosRoot, "thumbs");
|
||||
string itemOriginalPhotosRoot = Path.Combine(itemPhotosRoot, "originals");
|
||||
|
||||
foreach(string dir in new[]
|
||||
{
|
||||
itemPhotosRoot, itemThumbsRoot, itemOriginalPhotosRoot,
|
||||
Path.Combine(itemThumbsRoot, "jpeg", "4k"),
|
||||
Path.Combine(itemPhotosRoot, "jpeg", "4k"),
|
||||
Path.Combine(itemThumbsRoot, "webp", "4k"),
|
||||
Path.Combine(itemPhotosRoot, "webp", "4k"),
|
||||
Path.Combine(itemThumbsRoot, "avif", "4k"),
|
||||
Path.Combine(itemPhotosRoot, "avif", "4k")
|
||||
})
|
||||
Directory.CreateDirectory(dir);
|
||||
|
||||
string originalPath = Path.Combine(itemOriginalPhotosRoot, $"{guid}.jpg");
|
||||
await File.WriteAllBytesAsync(originalPath, bytes);
|
||||
|
||||
foreach((string format, string outExt) in new[]
|
||||
{
|
||||
("jpeg", "jpg"), ("webp", "webp"), ("avif", "avif")
|
||||
})
|
||||
{
|
||||
string fullPath = Path.Combine(itemPhotosRoot, format, "4k", $"{guid}.{outExt}");
|
||||
string thumbPath = Path.Combine(itemThumbsRoot, format, "4k", $"{guid}.{outExt}");
|
||||
|
||||
ConvertUsingImageMagick(originalPath, fullPath, 256, 256);
|
||||
ConvertUsingImageMagick(originalPath, thumbPath, 64, 64);
|
||||
}
|
||||
|
||||
platform.LogoId = guid;
|
||||
platform.LogoExtension = "jpg";
|
||||
|
||||
Console.WriteLine($" [{igdbId}] Logo downloaded and converted for auto-created platform {platform.Id}.");
|
||||
}
|
||||
|
||||
// Mirrors PlatformEnricherService.ConvertUsingImageMagick.
|
||||
static void ConvertUsingImageMagick(string originalPath, string outputPath, int width, int height)
|
||||
{
|
||||
try
|
||||
{
|
||||
var p = new Process
|
||||
{
|
||||
StartInfo =
|
||||
{
|
||||
FileName = "magick",
|
||||
CreateNoWindow = true,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true
|
||||
}
|
||||
};
|
||||
|
||||
p.StartInfo.ArgumentList.Add("-limit");
|
||||
p.StartInfo.ArgumentList.Add("thread");
|
||||
p.StartInfo.ArgumentList.Add("1");
|
||||
p.StartInfo.ArgumentList.Add($"{originalPath}[0]");
|
||||
p.StartInfo.ArgumentList.Add("-resize");
|
||||
p.StartInfo.ArgumentList.Add($"{width}x{height}>");
|
||||
p.StartInfo.ArgumentList.Add("-strip");
|
||||
p.StartInfo.ArgumentList.Add("-quality");
|
||||
p.StartInfo.ArgumentList.Add("75");
|
||||
p.StartInfo.ArgumentList.Add(outputPath);
|
||||
p.StartInfo.Environment["MAGICK_THREAD_LIMIT"] = "1";
|
||||
|
||||
p.Start();
|
||||
|
||||
Task<string> stderrTask = p.StandardError.ReadToEndAsync();
|
||||
Task<string> stdoutTask = p.StandardOutput.ReadToEndAsync();
|
||||
p.WaitForExit();
|
||||
_ = stderrTask.GetAwaiter().GetResult();
|
||||
_ = stdoutTask.GetAwaiter().GetResult();
|
||||
|
||||
if(p.ExitCode != 0)
|
||||
Console.Error.WriteLine($"convert failed for {outputPath}: exit {p.ExitCode}");
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"convert spawn failed for {outputPath}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
async Task<Dictionary<int, string>> FetchLookupAsync(string endpoint, string field)
|
||||
{
|
||||
var result = new Dictionary<int, string>();
|
||||
|
||||
string query = new ApicalypseQueryBuilder().Fields($"id,{field}").Limit(IgdbMaxPageSize).Build();
|
||||
|
||||
using JsonDocument doc = await _client.QueryAsync(endpoint, query);
|
||||
|
||||
foreach(JsonElement el in doc.RootElement.EnumerateArray())
|
||||
{
|
||||
int id = el.GetProperty("id").GetInt32();
|
||||
|
||||
if(el.TryGetProperty(field, out JsonElement v) && v.ValueKind == JsonValueKind.String)
|
||||
result[id] = v.GetString();
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async Task<List<JsonElement>> FetchByFieldAsync(string endpoint, string fields, string filterField,
|
||||
IEnumerable<long> ids)
|
||||
{
|
||||
List<long> distinctIds = ids.Distinct().ToList();
|
||||
var result = new List<JsonElement>();
|
||||
|
||||
if(distinctIds.Count == 0)
|
||||
return result;
|
||||
|
||||
foreach(List<long> chunk in Chunk(distinctIds, IgdbMaxPageSize))
|
||||
{
|
||||
string query = new ApicalypseQueryBuilder().Fields(fields)
|
||||
.Where($"{filterField} = ({string.Join(",", chunk)})")
|
||||
.Limit(IgdbMaxPageSize).Build();
|
||||
|
||||
using JsonDocument doc = await _client.QueryAsync(endpoint, query);
|
||||
|
||||
foreach(JsonElement el in doc.RootElement.EnumerateArray())
|
||||
result.Add(el.Clone());
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
static IEnumerable<List<long>> Chunk(List<long> source, int size)
|
||||
{
|
||||
for(int i = 0; i < source.Count; i += size)
|
||||
yield return source.GetRange(i, Math.Min(size, source.Count - i));
|
||||
}
|
||||
|
||||
static List<long> GetLongArray(JsonElement element, string property) =>
|
||||
element.TryGetProperty(property, out JsonElement arr) && arr.ValueKind == JsonValueKind.Array
|
||||
? arr.EnumerateArray().Select(x => x.GetInt64()).ToList()
|
||||
: [];
|
||||
|
||||
static DatePrecision ToDatePrecision(int? igdbDateFormat) => igdbDateFormat switch
|
||||
{
|
||||
0 => DatePrecision.Full,
|
||||
1 => DatePrecision.MonthYear,
|
||||
_ => DatePrecision.YearOnly
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user