claunia/marechai
1
0
Fork 0
mirror of https://github.com/claunia/marechai.git synced 2026-07-08 17:57:08 +00:00
Code Issues 15 Packages Projects Releases Wiki Activity

feat: Enrich matched companies with IGDB data

Browse Source 
Adds an enrich-companies pipeline stage to Marechai.Igdb that fills empty
Company fields (country, website, founding/sold dates, status, description)
from already-matched IGDB companies, and caches raster logos for manual
SVG upload, without ever overwriting existing local data.
This commit is contained in:
Natalia Portillo
2026-06-25 23:53:40 +01:00
parent 3080d20cbf
commit 976d924bad
9 changed files with 13377 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12855
Marechai.Database/Migrations/20260625214910_AddIgdbCompanyEnrichmentFields.Designer.cs generated Normal file
View File

File diff suppressed because it is too large Load Diff

155
Marechai.Database/Migrations/20260625214910_AddIgdbCompanyEnrichmentFields.cs Normal file
View File

@@ -0,0 +1,155 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Marechai.Database.Migrations
{
/// <inheritdoc />
public partial class AddIgdbCompanyEnrichmentFields : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<long>(
name: "ChangeDate",
table: "IgdbCompanies",
type: "bigint",
nullable: true);
migrationBuilder.AddColumn<int>(
name: "ChangeDateFormat",
table: "IgdbCompanies",
type: "int",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "CompanyTypeHistoryJson",
table: "IgdbCompanies",
type: "text",
nullable: true)
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AddColumn<short>(
name: "Country",
table: "IgdbCompanies",
type: "smallint",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "DescriptionRaw",
table: "IgdbCompanies",
type: "text",
nullable: true)
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AddColumn<DateTime>(
name: "EnrichedOn",
table: "IgdbCompanies",
type: "datetime(6)",
nullable: true);
migrationBuilder.AddColumn<bool>(
name: "EnrichmentApplied",
table: "IgdbCompanies",
type: "bit(1)",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<string>(
name: "LogoImageId",
table: "IgdbCompanies",
type: "varchar(64)",
maxLength: 64,
nullable: true)
.Annotation("MySql:CharSet", "utf8mb4");
migrationBuilder.AddColumn<long>(
name: "ParentIgdbId",
table: "IgdbCompanies",
type: "bigint",
nullable: true);
migrationBuilder.AddColumn<long>(
name: "StartDate",
table: "IgdbCompanies",
type: "bigint",
nullable: true);
migrationBuilder.AddColumn<int>(
name: "StartDateFormat",
table: "IgdbCompanies",
type: "int",
nullable: true);
migrationBuilder.AddColumn<int>(
name: "Status",
table: "IgdbCompanies",
type: "int",
nullable: true);
migrationBuilder.AddColumn<string>(
name: "WebsitesJson",
table: "IgdbCompanies",
type: "text",
nullable: true)
.Annotation("MySql:CharSet", "utf8mb4");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ChangeDate",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "ChangeDateFormat",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "CompanyTypeHistoryJson",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "Country",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "DescriptionRaw",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "EnrichedOn",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "EnrichmentApplied",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "LogoImageId",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "ParentIgdbId",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "StartDate",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "StartDateFormat",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "Status",
table: "IgdbCompanies");
migrationBuilder.DropColumn(
name: "WebsitesJson",
table: "IgdbCompanies");
}
}
}

40
Marechai.Database/Migrations/MarechaiContextModelSnapshot.cs
View File

@@ -2555,15 +2555,36 @@ namespace Marechai.Database.Migrations
.HasMaxLength(4096)
.HasColumnType("varchar(4096)");
b.Property<long?>("ChangeDate")
.HasColumnType("bigint");
b.Property<int?>("ChangeDateFormat")
.HasColumnType("int");
b.Property<int?>("CompanyId")
.HasColumnType("int");
b.Property<string>("CompanyTypeHistoryJson")
.HasColumnType("text");
b.Property<short?>("Country")
.HasColumnType("smallint");
b.Property<DateTime>("CreatedOn")
.ValueGeneratedOnAdd()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlIdentityColumn(b.Property<DateTime>("CreatedOn"));
b.Property<string>("DescriptionRaw")
.HasColumnType("text");
b.Property<DateTime?>("EnrichedOn")
.HasColumnType("datetime(6)");
b.Property<bool>("EnrichmentApplied")
.HasColumnType("bit(1)");
b.Property<string>("ErrorMessage")
.HasMaxLength(1024)
.HasColumnType("varchar(1024)");
@@ -2571,6 +2592,10 @@ namespace Marechai.Database.Migrations
b.Property<long>("IgdbId")
.HasColumnType("bigint");
b.Property<string>("LogoImageId")
.HasMaxLength(64)
.HasColumnType("varchar(64)");
b.Property<double?>("MatchScore")
.HasColumnType("double");
@@ -2589,12 +2614,27 @@ namespace Marechai.Database.Migrations
.HasMaxLength(255)
.HasColumnType("varchar(255)");
b.Property<long?>("ParentIgdbId")
.HasColumnType("bigint");
b.Property<long?>("StartDate")
.HasColumnType("bigint");
b.Property<int?>("StartDateFormat")
.HasColumnType("int");
b.Property<int?>("Status")
.HasColumnType("int");
b.Property<DateTime>("UpdatedOn")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("datetime(6)");
MySqlPropertyBuilderExtensions.UseMySqlComputedColumn(b.Property<DateTime>("UpdatedOn"));
b.Property<string>("WebsitesJson")
.HasColumnType("text");
b.HasKey("Id");
b.HasIndex("CompanyId");

31
Marechai.Database/Models/IgdbCompany.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
using Marechai.Data;
namespace Marechai.Database.Models;
@@ -32,4 +33,34 @@ public class IgdbCompany : BaseModel<long>
public int BatchNumber { get; set; }
public int? CompanyId { get; set; }
[Column(TypeName = "text")]
public string WebsitesJson { get; set; }
public short? Country { get; set; }
[Column(TypeName = "text")]
public string DescriptionRaw { get; set; }
public long? StartDate { get; set; }
public int? StartDateFormat { get; set; }
public long? ChangeDate { get; set; }
public int? ChangeDateFormat { get; set; }
public int? Status { get; set; }
public long? ParentIgdbId { get; set; }
[StringLength(64)]
public string LogoImageId { get; set; }
[Column(TypeName = "text")]
public string CompanyTypeHistoryJson { get; set; }
public bool EnrichmentApplied { get; set; }
public DateTime? EnrichedOn { get; set; }
}

1
Marechai.Igdb/Marechai.Igdb.csproj
View File

@@ -8,6 +8,7 @@
<RuntimeIdentifier>linux-x64</RuntimeIdentifier>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Markdig"/>
<PackageReference Include="MySqlConnector"/>
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql"/>
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql.Json.Microsoft"/>

40
Marechai.Igdb/Program.cs
View File

@@ -151,6 +151,28 @@ class Program
break;
}
case "enrich-companies":
{
string logoCachePath = config.GetValue("LogoCachePath", "state/igdb-logo-cache");
var enricher = new CompanyEnricherService(factory, logoCachePath);
CompanyEnricherService.Stats stats = await enricher.RunAsync(dryRun);
Console.WriteLine($"""
Done. Processed {stats.CompaniesProcessed} companies ({stats.CompaniesSkippedNoLocalMatch} skipped, no local match).
Countries filled: {stats.CountriesFilled}
Websites filled: {stats.WebsitesFilled}
Founded dates filled: {stats.FoundedFilled}
Status updated: {stats.StatusFilled}
Sold dates filled: {stats.SoldFilled}
SoldTo resolved: {stats.SoldToFilled}
Descriptions added: {stats.DescriptionsAdded}
Logos cached: {stats.LogosCached}
""");
break;
}
case "stats":
{
var stats = new MatchStatsService(factory);
@@ -184,7 +206,7 @@ class Program
if(igdbId == null || type == null)
{
Console.WriteLine(" --igdb-id <id> and --type companies|games|platforms are required.");
Console.WriteLine(" --igdb-id <id> and --type companies|companies-enrichment|games|platforms are required.");
return 1;
}
@@ -254,6 +276,19 @@ class Program
break;
}
case "companies-enrichment":
{
IgdbCompany entity = await context.IgdbCompanies.FirstOrDefaultAsync(c => c.IgdbId == igdbId);
if(entity != null)
{
entity.EnrichmentApplied = false;
entity.EnrichedOn = null;
}
break;
}
case "games":
{
IgdbGame entity = await context.IgdbGames.FirstOrDefaultAsync(g => g.IgdbId == igdbId);
@@ -313,9 +348,10 @@ class Program
match-platforms Match mirrored platforms against SoftwarePlatform
match-companies Match mirrored companies against Company
match-games Match mirrored games against Software
enrich-companies Fill empty Company fields from matched IGDB data (resumable)
stats Print match status counts
review-ambiguous --type <companies|games|platforms> [--limit N]
reset --igdb-id <id> --type <companies|games|platforms>
reset --igdb-id <id> --type <companies|companies-enrichment|games|platforms>
Options:
--batch-size N Mirror one batch of up to N rows (default Import:BatchSize).

212
Marechai.Igdb/Services/CompanyEnricherService.cs Normal file
View File

@@ -0,0 +1,212 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading.Tasks;
using Markdig;
using Marechai.Data;
using Marechai.Database.Models;
using Microsoft.EntityFrameworkCore;
namespace Marechai.Igdb.Services;
public class CompanyEnricherService
{
const string LogoCdnUrlTemplate = "https://images.igdb.com/igdb/image/upload/t_original/{0}.jpg";
readonly IDbContextFactory<MarechaiContext> _contextFactory;
readonly string _logoCachePath;
readonly HttpClient _httpClient;
public CompanyEnricherService(IDbContextFactory<MarechaiContext> contextFactory, string logoCachePath)
{
_contextFactory = contextFactory;
_logoCachePath = logoCachePath;
_httpClient = new HttpClient();
}
public class Stats
{
public int CompaniesProcessed;
public int CompaniesSkippedNoLocalMatch;
public int CountriesFilled;
public int WebsitesFilled;
public int FoundedFilled;
public int StatusFilled;
public int SoldFilled;
public int SoldToFilled;
public int DescriptionsAdded;
public int LogosCached;
}
public async Task<Stats> RunAsync(bool dryRun)
{
var stats = new Stats();
await using var context = await _contextFactory.CreateDbContextAsync();
var pending = await context.IgdbCompanies
.Where(c => c.MatchStatus == IgdbMatchStatus.Matched && c.CompanyId != null &&
!c.EnrichmentApplied)
.ToListAsync();
foreach(IgdbCompany igdbCompany in pending)
{
Company company = await context.Companies.Include(c => c.Descriptions)
.FirstOrDefaultAsync(c => c.Id == igdbCompany.CompanyId);
if(company == null)
{
stats.CompaniesSkippedNoLocalMatch++;
Console.WriteLine($" [{igdbCompany.IgdbId}] No local company {igdbCompany.CompanyId} found, skipping.");
continue;
}
stats.CompaniesProcessed++;
if(company.CountryId == null && igdbCompany.Country != null)
{
company.CountryId = igdbCompany.Country;
stats.CountriesFilled++;
Console.WriteLine($" [{igdbCompany.IgdbId}] Country filled: {igdbCompany.Country}.");
}
if(string.IsNullOrWhiteSpace(company.Website) && !string.IsNullOrWhiteSpace(igdbCompany.WebsitesJson))
{
string website = FirstWebsite(igdbCompany.WebsitesJson);
if(!string.IsNullOrWhiteSpace(website) && website.Length <= 255)
{
company.Website = website;
stats.WebsitesFilled++;
Console.WriteLine($" [{igdbCompany.IgdbId}] Website filled: {website}.");
}
}
if(company.Founded == null && igdbCompany.StartDate != null && igdbCompany.StartDateFormat != 7)
{
company.Founded = DateTimeOffset.FromUnixTimeSeconds(igdbCompany.StartDate.Value).UtcDateTime;
company.FoundedPrecision = ToDatePrecision(igdbCompany.StartDateFormat);
stats.FoundedFilled++;
Console.WriteLine($" [{igdbCompany.IgdbId}] Founded date filled: {company.Founded:yyyy-MM-dd}.");
}
if(company.Status == CompanyStatus.Unknown && igdbCompany.Status != null)
{
CompanyStatus? status = ToCompanyStatus(igdbCompany.Status.Value);
if(status != null)
{
company.Status = status.Value;
stats.StatusFilled++;
Console.WriteLine($" [{igdbCompany.IgdbId}] Status filled: {status.Value}.");
}
}
if(company.Sold == null && igdbCompany.ChangeDate != null && igdbCompany.ChangeDateFormat != 7)
{
company.Sold = DateTimeOffset.FromUnixTimeSeconds(igdbCompany.ChangeDate.Value).UtcDateTime;
company.SoldPrecision = ToDatePrecision(igdbCompany.ChangeDateFormat);
stats.SoldFilled++;
Console.WriteLine($" [{igdbCompany.IgdbId}] Sold date filled: {company.Sold:yyyy-MM-dd}.");
}
if(company.SoldToId == null && igdbCompany.ParentIgdbId != null)
{
IgdbCompany parent = await context.IgdbCompanies
.FirstOrDefaultAsync(c => c.IgdbId == igdbCompany.ParentIgdbId &&
c.MatchStatus == IgdbMatchStatus.Matched);
if(parent?.CompanyId != null)
{
company.SoldToId = parent.CompanyId;
stats.SoldToFilled++;
Console.WriteLine($" [{igdbCompany.IgdbId}] SoldTo resolved: company {parent.CompanyId}.");
}
}
if(!string.IsNullOrWhiteSpace(igdbCompany.DescriptionRaw) &&
!company.Descriptions.Any(d => d.LanguageCode == "eng"))
{
string markdown = igdbCompany.DescriptionRaw.Replace("\r\n", "\n").Replace("\n", "\n\n");
company.Descriptions.Add(new CompanyDescription
{
LanguageCode = "eng",
Text = markdown,
Html = Markdown.ToHtml(markdown, new MarkdownPipelineBuilder().UseAdvancedExtensions()
.Build())
});
stats.DescriptionsAdded++;
Console.WriteLine($" [{igdbCompany.IgdbId}] Description added.");
}
if(!string.IsNullOrWhiteSpace(igdbCompany.LogoImageId) && !dryRun)
{
await CacheLogoAsync(igdbCompany.IgdbId, company.Id, igdbCompany.LogoImageId);
stats.LogosCached++;
}
else if(!string.IsNullOrWhiteSpace(igdbCompany.LogoImageId))
stats.LogosCached++;
if(!dryRun)
{
igdbCompany.EnrichmentApplied = true;
igdbCompany.EnrichedOn = DateTime.UtcNow;
}
}
if(!dryRun)
await context.SaveChangesAsync();
return stats;
}
async Task CacheLogoAsync(long igdbId, int companyId, string imageId)
{
Directory.CreateDirectory(_logoCachePath);
string destination = Path.Combine(_logoCachePath, $"{companyId}.jpg");
try
{
byte[] bytes = await _httpClient.GetByteArrayAsync(string.Format(LogoCdnUrlTemplate, imageId));
await File.WriteAllBytesAsync(destination, bytes);
Console.WriteLine($" [{igdbId}] Logo cached to {destination}.");
}
catch(HttpRequestException e)
{
Console.WriteLine($" [{igdbId}] Failed to download logo: {e.Message}.");
}
}
static string FirstWebsite(string websitesJson)
{
using JsonDocument doc = JsonDocument.Parse(websitesJson);
foreach(JsonElement element in doc.RootElement.EnumerateArray())
if(element.TryGetProperty("url", out JsonElement url))
return url.GetString();
return null;
}
static DatePrecision ToDatePrecision(int? igdbDateFormat) => igdbDateFormat switch
{
0 => DatePrecision.Full,
1 => DatePrecision.MonthYear,
_ => DatePrecision.YearOnly
};
static CompanyStatus? ToCompanyStatus(int igdbStatus) => igdbStatus switch
{
0 => CompanyStatus.Active,
1 => CompanyStatus.Defunct,
2 => CompanyStatus.Merged,
3 => CompanyStatus.Renamed,
_ => null
};
}

48
Marechai.Igdb/Services/CompanyMirrorService.cs
View File

@@ -58,7 +58,10 @@ public class CompanyMirrorService
async Task<(int count, long lastId)> FetchAndUpsertPageAsync(long afterId, int pageSize, int batchNumber,
bool dryRun)
{
string query = new ApicalypseQueryBuilder().Fields("id,name")
string query = new ApicalypseQueryBuilder().Fields("id,name,country,start_date,start_date_format," +
"change_date,change_date_format,status,parent," +
"websites.url,websites.category,description,logo.image_id," +
"company_type_histories")
.Where($"id > {afterId}")
.Sort("id asc")
.Limit(pageSize)
@@ -87,10 +90,45 @@ public class CompanyMirrorService
context.IgdbCompanies.Add(new IgdbCompany
{
IgdbId = igdbId,
Name = name,
MatchStatus = IgdbMatchStatus.Pending,
BatchNumber = batchNumber
IgdbId = igdbId,
Name = name,
MatchStatus = IgdbMatchStatus.Pending,
BatchNumber = batchNumber,
Country = element.TryGetProperty("country", out JsonElement country)
? (short)country.GetInt32()
: null,
StartDate = element.TryGetProperty("start_date", out JsonElement startDate)
? startDate.GetInt64()
: null,
StartDateFormat = element.TryGetProperty("start_date_format", out JsonElement startFmt)
? startFmt.GetInt32()
: null,
ChangeDate = element.TryGetProperty("change_date", out JsonElement changeDate)
? changeDate.GetInt64()
: null,
ChangeDateFormat = element.TryGetProperty("change_date_format", out JsonElement changeFmt)
? changeFmt.GetInt32()
: null,
Status = element.TryGetProperty("status", out JsonElement status)
? status.GetInt32()
: null,
ParentIgdbId = element.TryGetProperty("parent", out JsonElement parent)
? parent.GetInt64()
: null,
LogoImageId = element.TryGetProperty("logo", out JsonElement logo) &&
logo.TryGetProperty("image_id", out JsonElement imageId)
? imageId.GetString()
: null,
DescriptionRaw = element.TryGetProperty("description", out JsonElement description)
? description.GetString()
: null,
WebsitesJson = element.TryGetProperty("websites", out JsonElement websites)
? websites.GetRawText()
: null,
CompanyTypeHistoryJson = element.TryGetProperty("company_type_histories",
out JsonElement typeHistories)
? typeHistories.GetRawText()
: null
});
lastId = igdbId;

3
Marechai.Igdb/appsettings.json
View File

@@ -1,5 +1,6 @@
{
"Import": {
"BatchSize": 500
}
},
"LogoCachePath": "state/igdb-logo-cache"
}