feat: Download missing SoftwarePlatform logos from matched IGDB platforms

Adds a platform logo enrichment stage to the IGDB pipeline (Marechai.Igdb),
mirroring the existing company enrichment flow but for SoftwarePlatform's
raster logo (LogoId/LogoExtension), which previously had to be uploaded
manually.

- IgdbPlatform gains LogoImageId, EnrichmentApplied, and EnrichedOn columns
  (migration AddIgdbPlatformLogoEnrichment), mirroring IgdbCompany's
  enrichment-tracking pattern.
- PlatformMirrorService now requests platform_logo.image_id from IGDB and
  stores it on insert and on re-mirror.
- New PlatformEnricherService: for platforms matched to an IGDB platform
  with no local logo yet, downloads the best available logo image and
  reproduces the exact originals/jpeg/webp/avif + 4k/thumbs layout and
  ImageMagick conversion parameters used by the manual logo upload
  endpoint (SoftwarePlatformsController.UploadLogoAsync), so enriched and
  manually-uploaded logos are indistinguishable on disk.
- New IgdbImageDownloader helper, shared between company and platform
  enrichment: IGDB's documented image sizes top out at logo_med for
  logo-type images, with no documented "original" size, and any size
  accepts an undocumented-but-functional "_2x" retina suffix. Rather than
  trusting any single size identifier, it downloads each plausible
  candidate (logo_med_2x, logo_med, original), decodes the actual pixel
  dimensions of each with `magick identify`, and keeps whichever is truly
  largest. CompanyEnricherService.CacheLogoAsync is updated to use this
  helper too, replacing its previous unconditional reliance on the
  undocumented t_original template.
- Program.cs: new `enrich-platforms` command (wired through a new
  AssetRootPath config key, since output goes straight into the real
  asset tree rather than a review cache), usage text, and a
  `platforms-enrichment` reset case to clear EnrichmentApplied/EnrichedOn
  for re-runs.

Marechai.Igdb intentionally does not take a project reference on
Marechai.Server (which owns the Photos helper and ASP.NET Core
dependencies) to perform these image conversions; the ImageMagick
subprocess invocation is duplicated in PlatformEnricherService to keep
the console tool's dependency footprint small.
This commit is contained in:
2026-06-26 11:49:16 +01:00
parent d2ab7fc931
commit d76dd99090
10 changed files with 13613 additions and 16 deletions

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,52 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Marechai.Database.Migrations
{
/// <inheritdoc />
public partial class AddIgdbPlatformLogoEnrichment : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<DateTime>(
name: "EnrichedOn",
table: "IgdbPlatforms",
type: "datetime(6)",
nullable: true);
migrationBuilder.AddColumn<bool>(
name: "EnrichmentApplied",
table: "IgdbPlatforms",
type: "bit(1)",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<string>(
name: "LogoImageId",
table: "IgdbPlatforms",
type: "varchar(64)",
maxLength: 64,
nullable: true)
.Annotation("MySql:CharSet", "utf8mb4");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "EnrichedOn",
table: "IgdbPlatforms");
migrationBuilder.DropColumn(
name: "EnrichmentApplied",
table: "IgdbPlatforms");
migrationBuilder.DropColumn(
name: "LogoImageId",
table: "IgdbPlatforms");
}
}
}

View File

@@ -2870,6 +2870,16 @@ namespace Marechai.Database.Migrations
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
b.Property<DateTime?>("EnrichedOn")
.HasColumnType("datetime(6)");
b.Property<bool>("EnrichmentApplied")
.HasColumnType("bit(1)");
b.Property<string>("LogoImageId")
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<byte>("MatchStatus")
.HasColumnType("tinyint unsigned");

View File

@@ -19,4 +19,11 @@ public class IgdbPlatform : BaseModel<int>
public DateTime? MatchedOn { get; set; }
public ulong? SoftwarePlatformId { get; set; }
[StringLength(64)]
public string LogoImageId { get; set; }
public bool EnrichmentApplied { get; set; }
public DateTime? EnrichedOn { get; set; }
}

View File

@@ -187,6 +187,30 @@ class Program
break;
}
case "enrich-platforms":
{
string assetRootPath = config.GetValue<string>("AssetRootPath");
if(string.IsNullOrEmpty(assetRootPath))
{
Console.WriteLine("\e[31;1mMissing AssetRootPath in appsettings.json\e[0m");
return 1;
}
var enricher = new PlatformEnricherService(factory, assetRootPath);
PlatformEnricherService.Stats stats = await enricher.RunAsync(dryRun);
Console.WriteLine($"""
Done. Processed {stats.PlatformsProcessed} platforms ({stats.PlatformsSkippedNoLocalMatch} skipped, no local match).
Logos downloaded: {stats.LogosDownloaded}
Logos already present: {stats.LogosSkippedAlreadyPresent}
""");
break;
}
case "stats":
{
var stats = new MatchStatsService(factory);
@@ -220,7 +244,7 @@ class Program
if(igdbId == null || type == null)
{
Console.WriteLine(" --igdb-id <id> and --type companies|companies-enrichment|games|platforms are required.");
Console.WriteLine(" --igdb-id <id> and --type companies|companies-enrichment|games|platforms|platforms-enrichment are required.");
return 1;
}
@@ -303,6 +327,19 @@ class Program
break;
}
case "platforms-enrichment":
{
IgdbPlatform entity = await context.IgdbPlatforms.FirstOrDefaultAsync(p => p.Id == igdbId);
if(entity != null)
{
entity.EnrichmentApplied = false;
entity.EnrichedOn = null;
}
break;
}
case "games":
{
IgdbGame entity = await context.IgdbGames.FirstOrDefaultAsync(g => g.IgdbId == igdbId);
@@ -364,9 +401,10 @@ class Program
match-companies Match mirrored companies against Company
match-games Match mirrored games against Software
enrich-companies Fill empty Company fields from matched IGDB data (resumable)
enrich-platforms Download missing SoftwarePlatform logos from matched IGDB data (resumable)
stats Print match status counts
review-ambiguous --type <companies|games|platforms> [--limit N]
reset --igdb-id <id> --type <companies|companies-enrichment|games|platforms>
reset --igdb-id <id> --type <companies|companies-enrichment|games|platforms|platforms-enrichment>
Options:
--batch-size N Mirror one batch of up to N rows (default Import:BatchSize).

View File

@@ -13,8 +13,6 @@ namespace Marechai.Igdb.Services;
public class CompanyEnricherService
{
const string LogoCdnUrlTemplate = "https://images.igdb.com/igdb/image/upload/t_original/{0}.jpg";
readonly IDbContextFactory<MarechaiContext> _contextFactory;
readonly string _logoCachePath;
readonly HttpClient _httpClient;
@@ -171,16 +169,18 @@ public class CompanyEnricherService
string destination = Path.Combine(_logoCachePath, $"{companyId}.jpg");
try
byte[] bytes = await IgdbImageDownloader.DownloadBestAsync(_httpClient, imageId,
IgdbImageDownloader.LogoSizeCandidates);
if(bytes == null)
{
byte[] bytes = await _httpClient.GetByteArrayAsync(string.Format(LogoCdnUrlTemplate, imageId));
await File.WriteAllBytesAsync(destination, bytes);
Console.WriteLine($" [{igdbId}] Logo cached to {destination}.");
}
catch(HttpRequestException e)
{
Console.WriteLine($" [{igdbId}] Failed to download logo: {e.Message}.");
Console.WriteLine($" [{igdbId}] Failed to download logo: no candidate size succeeded.");
return;
}
await File.WriteAllBytesAsync(destination, bytes);
Console.WriteLine($" [{igdbId}] Logo cached to {destination}.");
}
static string FirstWebsite(string websitesJson)

View File

@@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Net.Http;
using System.Threading.Tasks;
namespace Marechai.Igdb.Services;
/// <summary>
/// Downloads IGDB CDN images, trying several size candidates and keeping whichever decodes
/// to the largest actual pixel area. IGDB's documented size identifiers
/// (https://api-docs.igdb.com/#images) top out at <c>1080p</c>, with no documented
/// <c>original</c> size — but <c>t_original</c> works in practice for some image types, and
/// any size accepts a <c>_2x</c> retina suffix that Cloudinary won't upscale past the source's
/// native resolution. None of this is fully documented, so candidates are never trusted blindly:
/// each is downloaded and measured, and the actually-largest one wins.
/// </summary>
internal static class IgdbImageDownloader
{
public static readonly string[] LogoSizeCandidates = { "logo_med_2x", "logo_med", "original" };
public static async Task<byte[]> DownloadBestAsync(HttpClient httpClient, string imageId,
IReadOnlyList<string> sizeCandidates)
{
byte[] best = null;
long bestArea = -1;
foreach(string size in sizeCandidates)
{
byte[] candidate;
try
{
candidate = await httpClient.GetByteArrayAsync(
$"https://images.igdb.com/igdb/image/upload/t_{size}/{imageId}.jpg");
}
catch(HttpRequestException)
{
continue;
}
(int width, int height) = IdentifyDimensions(candidate);
long area = (long)width * height;
if(area > bestArea)
{
best = candidate;
bestArea = area;
}
}
return best;
}
static (int width, int height) IdentifyDimensions(byte[] bytes)
{
string tempPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.jpg");
try
{
File.WriteAllBytes(tempPath, bytes);
var p = new Process
{
StartInfo =
{
FileName = "magick",
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true
}
};
p.StartInfo.ArgumentList.Add("identify");
p.StartInfo.ArgumentList.Add("-format");
p.StartInfo.ArgumentList.Add("%w %h\n");
p.StartInfo.ArgumentList.Add($"{tempPath}[0]");
p.StartInfo.Environment["MAGICK_THREAD_LIMIT"] = "1";
p.Start();
Task<string> stderrTask = p.StandardError.ReadToEndAsync();
Task<string> stdoutTask = p.StandardOutput.ReadToEndAsync();
p.WaitForExit();
string stdout = stdoutTask.GetAwaiter().GetResult();
_ = stderrTask.GetAwaiter().GetResult();
if(p.ExitCode != 0) return (0, 0);
string firstLine = stdout.Split('\n', 2)[0].Trim();
string[] parts = firstLine.Split(' ', StringSplitOptions.RemoveEmptyEntries);
if(parts.Length < 2) return (0, 0);
if(!int.TryParse(parts[0], out int w) || !int.TryParse(parts[1], out int h)) return (0, 0);
return (w, h);
}
catch(Exception)
{
return (0, 0);
}
finally
{
try
{
File.Delete(tempPath);
}
catch(IOException) { }
}
}
}

View File

@@ -0,0 +1,179 @@
using System;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Threading.Tasks;
using Marechai.Data;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.Igdb.Services;
public class PlatformEnricherService
{
readonly IDbContextFactory<MarechaiContext> _contextFactory;
readonly string _assetRootPath;
readonly HttpClient _httpClient;
public PlatformEnricherService(IDbContextFactory<MarechaiContext> contextFactory, string assetRootPath)
{
_contextFactory = contextFactory;
_assetRootPath = assetRootPath;
_httpClient = new HttpClient();
}
public class Stats
{
public int PlatformsProcessed;
public int PlatformsSkippedNoLocalMatch;
public int LogosDownloaded;
public int LogosSkippedAlreadyPresent;
}
public async Task<Stats> RunAsync(bool dryRun)
{
var stats = new Stats();
await using var context = await _contextFactory.CreateDbContextAsync();
var pending = await context.IgdbPlatforms
.Where(p => p.MatchStatus == IgdbMatchStatus.Matched &&
p.SoftwarePlatformId != null && !p.EnrichmentApplied)
.ToListAsync();
foreach(IgdbPlatform igdbPlatform in pending)
{
SoftwarePlatform platform =
await context.SoftwarePlatforms.FirstOrDefaultAsync(p => p.Id == igdbPlatform.SoftwarePlatformId);
if(platform == null)
{
stats.PlatformsSkippedNoLocalMatch++;
Console.WriteLine(
$" [{igdbPlatform.Id}] No local platform {igdbPlatform.SoftwarePlatformId} found, skipping.");
continue;
}
stats.PlatformsProcessed++;
if(platform.LogoId.HasValue)
stats.LogosSkippedAlreadyPresent++;
else if(!string.IsNullOrWhiteSpace(igdbPlatform.LogoImageId))
{
if(!dryRun)
await DownloadAndApplyLogoAsync(igdbPlatform.Id, platform, igdbPlatform.LogoImageId);
stats.LogosDownloaded++;
}
if(!dryRun)
{
igdbPlatform.EnrichmentApplied = true;
igdbPlatform.EnrichedOn = DateTime.UtcNow;
}
}
if(!dryRun)
await context.SaveChangesAsync();
return stats;
}
async Task DownloadAndApplyLogoAsync(int igdbId, SoftwarePlatform platform, string imageId)
{
byte[] bytes = await IgdbImageDownloader.DownloadBestAsync(_httpClient, imageId,
IgdbImageDownloader.LogoSizeCandidates);
if(bytes == null)
{
Console.WriteLine($" [{igdbId}] Failed to download logo: no candidate size succeeded.");
return;
}
var guid = Guid.NewGuid();
string itemPhotosRoot = Path.Combine(_assetRootPath, "photos", "platform-logos");
string itemThumbsRoot = Path.Combine(itemPhotosRoot, "thumbs");
string itemOriginalPhotosRoot = Path.Combine(itemPhotosRoot, "originals");
foreach(string dir in new[]
{
itemPhotosRoot, itemThumbsRoot, itemOriginalPhotosRoot,
Path.Combine(itemThumbsRoot, "jpeg", "4k"), Path.Combine(itemPhotosRoot, "jpeg", "4k"),
Path.Combine(itemThumbsRoot, "webp", "4k"), Path.Combine(itemPhotosRoot, "webp", "4k"),
Path.Combine(itemThumbsRoot, "avif", "4k"), Path.Combine(itemPhotosRoot, "avif", "4k")
})
Directory.CreateDirectory(dir);
string originalPath = Path.Combine(itemOriginalPhotosRoot, $"{guid}.jpg");
await File.WriteAllBytesAsync(originalPath, bytes);
foreach((string format, string outExt) in new[]
{
("jpeg", "jpg"), ("webp", "webp"), ("avif", "avif")
})
{
string fullPath = Path.Combine(itemPhotosRoot, format, "4k", $"{guid}.{outExt}");
string thumbPath = Path.Combine(itemThumbsRoot, format, "4k", $"{guid}.{outExt}");
ConvertUsingImageMagick(originalPath, fullPath, 256, 256);
ConvertUsingImageMagick(originalPath, thumbPath, 64, 64);
}
platform.LogoId = guid;
platform.LogoExtension = "jpg";
Console.WriteLine($" [{igdbId}] Logo downloaded and converted for platform {platform.Id}.");
}
// Mirrors Photos.ConvertUsingImageMagick (Marechai.Server/Helpers/Photos.cs) without taking a
// dependency on Marechai.Server, since Marechai.Igdb is a standalone console tool that doesn't
// otherwise need ASP.NET Core.
static void ConvertUsingImageMagick(string originalPath, string outputPath, int width, int height)
{
try
{
var p = new Process
{
StartInfo =
{
FileName = "magick",
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true
}
};
p.StartInfo.ArgumentList.Add("-limit");
p.StartInfo.ArgumentList.Add("thread");
p.StartInfo.ArgumentList.Add("1");
p.StartInfo.ArgumentList.Add($"{originalPath}[0]");
p.StartInfo.ArgumentList.Add("-resize");
p.StartInfo.ArgumentList.Add($"{width}x{height}>");
p.StartInfo.ArgumentList.Add("-strip");
p.StartInfo.ArgumentList.Add("-quality");
p.StartInfo.ArgumentList.Add("75");
p.StartInfo.ArgumentList.Add(outputPath);
p.StartInfo.Environment["MAGICK_THREAD_LIMIT"] = "1";
p.Start();
Task<string> stderrTask = p.StandardError.ReadToEndAsync();
Task<string> stdoutTask = p.StandardOutput.ReadToEndAsync();
p.WaitForExit();
_ = stderrTask.GetAwaiter().GetResult();
_ = stdoutTask.GetAwaiter().GetResult();
if(p.ExitCode != 0)
Console.Error.WriteLine($"convert failed for {outputPath}: exit {p.ExitCode}");
}
catch(Exception ex)
{
Console.Error.WriteLine($"convert spawn failed for {outputPath}: {ex.Message}");
}
}
}

View File

@@ -50,7 +50,7 @@ public class PlatformMirrorService
async Task<int> FetchAndUpsertPageAsync(int offset, int pageSize, bool dryRun)
{
string query = new ApicalypseQueryBuilder().Fields("id,name")
string query = new ApicalypseQueryBuilder().Fields("id,name,platform_logo.image_id")
.Sort("id asc")
.Limit(pageSize)
.Offset(offset)
@@ -76,6 +76,11 @@ public class PlatformMirrorService
int igdbId = element.GetProperty("id").GetInt32();
string name = element.GetProperty("name").GetString();
string logoImageId = element.TryGetProperty("platform_logo", out JsonElement logo) &&
logo.TryGetProperty("image_id", out JsonElement imageId)
? imageId.GetString()
: null;
var existing = await context.IgdbPlatforms.FirstOrDefaultAsync(p => p.Id == igdbId);
if(existing == null)
@@ -84,12 +89,14 @@ public class PlatformMirrorService
{
Id = igdbId,
Name = name,
MatchStatus = IgdbMatchStatus.Pending
MatchStatus = IgdbMatchStatus.Pending,
LogoImageId = logoImageId
});
}
else
{
existing.Name = name;
existing.Name = name;
existing.LogoImageId = logoImageId;
}
}

View File

@@ -2,5 +2,6 @@
"Import": {
"BatchSize": 500
},
"LogoCachePath": "state/igdb-logo-cache"
"LogoCachePath": "state/igdb-logo-cache",
"AssetRootPath": "/var/marechai/assets"
}