feat: implement PromoArtTabParser and migrate chunk handling to fixed slots

This commit is contained in:
2026-06-06 01:25:41 +01:00
parent 56c382d394
commit a0a37944a4
10 changed files with 502 additions and 61 deletions

View File

@@ -0,0 +1,224 @@
/******************************************************************************
// MARECHAI: Master repository of computing history artifacts information
// ----------------------------------------------------------------------------
//
// Author(s) : Natalia Portillo <claunia@claunia.com>
//
// --[ License ] --------------------------------------------------------------
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
// ----------------------------------------------------------------------------
// Copyright © 2003-2026 Natalia Portillo
*******************************************************************************/
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using HtmlAgilityPack;
using Marechai.MobyGames.Models;
namespace Marechai.MobyGames.Parsers.NewSite;
/// <summary>
/// Parses the new MobyGames promo art page (post-2023 Vue redesign).
/// Each promo group lives in a <c>&lt;section id="promo-image-group-{id}"&gt;</c>
/// containing an <c>&lt;h2&gt;</c> with the group name and link, followed by
/// one or more <c>&lt;figure&gt;</c> elements inside a
/// <c>&lt;div class="img-holder mb"&gt;</c>.
/// </summary>
public static partial class PromoArtTabParser
{
[GeneratedRegex(@"group-(\d+)", RegexOptions.Compiled)]
private static partial Regex GroupIdRegex();
[GeneratedRegex(@"image-(\d+)", RegexOptions.Compiled)]
private static partial Regex ImageIdRegex();
/// <summary>
/// Regex fallback: extracts image references directly from raw HTML,
/// bypassing HtmlAgilityPack entirely. Used when DOM parsing fails
/// (e.g. due to malformed Vue component attributes corrupting the tree).
/// </summary>
[GeneratedRegex(
@"href=""[^""]*?/promo/group-(\d+)/image-(\d+)/""",
RegexOptions.Compiled | RegexOptions.IgnoreCase)]
private static partial Regex ImageHrefRegex();
/// <summary>Extracts group name from any anchor linking to /promo/group-{id}/.</summary>
[GeneratedRegex(
@"<a[^>]+href=""[^""]*?/promo/group-(\d+)/""[^>]*>\s*([^<]+?)\s*</a>",
RegexOptions.Compiled | RegexOptions.IgnoreCase)]
private static partial Regex GroupLinkRegex();
public static List<ParsedPromoArtGroup> Parse(string html)
{
var doc = new HtmlDocument();
doc.LoadHtml(html);
var groups = Parse(doc);
// If XPath-based parsing found nothing, try regex fallback
if(groups.Count == 0 && html.Contains("/promo/group-"))
groups = ParseWithRegex(html);
return groups;
}
public static List<ParsedPromoArtGroup> Parse(HtmlDocument doc)
{
var groups = new List<ParsedPromoArtGroup>();
// Target the <section id="promo-image-group-{id}"> containers directly.
// This bypasses the navbar/header Vue components whose massive JSON attributes
// can corrupt HAP's DOM tree.
var sections = doc.DocumentNode.SelectNodes("//section[starts-with(@id, 'promo-image-group-')]");
if(sections is null) return groups;
foreach(var section in sections)
{
// Extract group name and ID from the h2 anchor
var groupLink = section.SelectSingleNode(".//h2//a[contains(@href, '/promo/group-')]");
if(groupLink is null) continue;
string groupName = WebUtility.HtmlDecode(groupLink.InnerText).Trim();
string groupHref = groupLink.GetAttributeValue("href", "");
var groupMatch = GroupIdRegex().Match(groupHref);
string groupId = groupMatch.Success ? groupMatch.Groups[1].Value : null;
// Also try extracting from the section id attribute
if(groupId is null)
{
string sectionId = section.GetAttributeValue("id", "");
// id="promo-image-group-90458" → extract 90458
if(sectionId.StartsWith("promo-image-group-"))
groupId = sectionId["promo-image-group-".Length..];
}
var group = new ParsedPromoArtGroup
{
GroupName = groupName,
GroupId = groupId
};
// Find all <figure> elements within this section (may be nested in <div class="img-holder">)
var figures = section.SelectNodes(".//figure");
if(figures is not null)
{
foreach(var figure in figures)
{
var a = figure.SelectSingleNode(".//a[@href]");
if(a is null) continue;
string href = a.GetAttributeValue("href", "");
var imageMatch = ImageIdRegex().Match(href);
if(!imageMatch.Success) continue;
string imageId = imageMatch.Groups[1].Value;
// Skip duplicates
if(group.Images.Any(i => i.ImageId == imageId)) continue;
// Caption from <figcaption>
var fc = figure.SelectSingleNode(".//figcaption");
string caption = fc is null ? null : WebUtility.HtmlDecode(fc.InnerText).Trim();
if(string.IsNullOrWhiteSpace(caption)) caption = null;
// Thumbnail URL from <img> src
var img = figure.SelectSingleNode(".//img");
string thumbnailUrl = img?.GetAttributeValue("src", null);
group.Images.Add(new ParsedPromoArtImage
{
ImageId = imageId,
DetailPageUrl = href,
Caption = caption,
ThumbnailUrl = thumbnailUrl
});
}
}
if(group.Images.Count > 0)
groups.Add(group);
}
return groups;
}
/// <summary>
/// Regex-only fallback that scans raw HTML for promo image href patterns.
/// Groups images by their group ID. Used when HAP DOM parsing fails.
/// Groups whose name cannot be determined are skipped entirely.
/// </summary>
static List<ParsedPromoArtGroup> ParseWithRegex(string html)
{
var groupMap = new Dictionary<string, ParsedPromoArtGroup>();
// Extract group names from <a href="...promo/group-{id}/">Name</a> links
var headingNames = new Dictionary<string, string>();
foreach(Match hm in GroupLinkRegex().Matches(html))
{
string gid = hm.Groups[1].Value;
string name = WebUtility.HtmlDecode(hm.Groups[2].Value).Trim();
if(!string.IsNullOrWhiteSpace(name))
headingNames.TryAdd(gid, name);
}
// Extract all image references — only for groups whose name we found
foreach(Match m in ImageHrefRegex().Matches(html))
{
string groupId = m.Groups[1].Value;
string imageId = m.Groups[2].Value;
// Skip groups where we couldn't determine the real name
if(!headingNames.ContainsKey(groupId)) continue;
if(!groupMap.TryGetValue(groupId, out var group))
{
group = new ParsedPromoArtGroup
{
GroupId = groupId,
GroupName = headingNames[groupId]
};
groupMap[groupId] = group;
}
if(group.Images.Any(i => i.ImageId == imageId)) continue;
// Reconstruct the detail page URL from the matched href
string fullHref = m.Value.Length > 6
? m.Value[6..^1] // strip href=" and trailing "
: $"/promo/group-{groupId}/image-{imageId}/";
group.Images.Add(new ParsedPromoArtImage
{
ImageId = imageId,
DetailPageUrl = fullHref
});
}
return groupMap.Values.Where(g => g.Images.Count > 0).ToList();
}
}

View File

@@ -1,4 +1,6 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Marechai.Database;
using Marechai.Database.Models;
@@ -776,6 +778,94 @@ class Program
break;
}
case "migrate-chunks":
{
Console.WriteLine("\n Migrating dynamically-allocated chunks to fixed slot numbers...\n");
Console.WriteLine(" Fixed slots: Promo=10, Screenshots=11, Media=12\n");
var misplaced = await sourceDb.GetMisplacedChunksAsync();
Console.WriteLine($" Found {misplaced.Count} chunk(s) outside fixed slots\n");
int migrated = 0;
int skipped = 0;
int errors = 0;
var skipReasons = new Dictionary<string, int>();
for(int i = 0; i < misplaced.Count; i++)
{
var (gameId, chunk) = misplaced[i];
if((i + 1) % 1000 == 0)
Console.WriteLine($" [{i + 1}/{misplaced.Count}] migrated={migrated} skipped={skipped} errors={errors}");
string body = await sourceDb.GetChunkBodyAsync(gameId, chunk);
if(body is null)
{
skipped++;
continue;
}
var (tab, _) = Parsers.TabDetector.DetectWithLayout(body);
int? targetChunk = tab switch
{
Parsers.MobyTab.PromoArt => NewGameRawFetcher.ChunkPromo,
Parsers.MobyTab.Screenshots => NewGameRawFetcher.ChunkScreenshots,
Parsers.MobyTab.Media => NewGameRawFetcher.ChunkMedia,
_ => null
};
if(targetChunk is null)
{
string reason = tab.ToString();
skipReasons.TryGetValue(reason, out int cnt);
skipReasons[reason] = cnt + 1;
skipped++;
continue;
}
// Don't overwrite if target slot already has data
if(await sourceDb.ChunkExistsAsync(gameId, targetChunk.Value))
{
string reason = $"{tab} → {targetChunk.Value} (slot occupied)";
skipReasons.TryGetValue(reason, out int cnt);
skipReasons[reason] = cnt + 1;
skipped++;
continue;
}
try
{
await sourceDb.MoveChunkAsync(gameId, chunk, targetChunk.Value);
migrated++;
}
catch(Exception ex)
{
Console.WriteLine($" ERROR {gameId} chunk {chunk} → {targetChunk.Value}: {ex.Message}");
errors++;
}
}
Console.WriteLine($"\n ────────────────────────────────────");
Console.WriteLine($" Chunks scanned: {misplaced.Count}");
Console.WriteLine($" Migrated: {migrated}");
Console.WriteLine($" Skipped: {skipped}");
Console.WriteLine($" Errors: {errors}");
if(skipReasons.Count > 0)
{
Console.WriteLine(" Skip breakdown:");
foreach(var kv in skipReasons.OrderByDescending(kv => kv.Value))
Console.WriteLine($" {kv.Key}: {kv.Value}");
}
Console.WriteLine($" ────────────────────────────────────\n");
break;
}
default:
Console.WriteLine(" Usage:");
Console.WriteLine(" import [--batch-size N] [--unattended] [--yes-to-all]");
@@ -831,6 +921,8 @@ class Program
Console.WriteLine(" cached rows under both '<slug>' and '-<slug>' first. Use to evict stale");
Console.WriteLine(" legacy-layout captures (e.g. since-corrected MobyGames data).");
Console.WriteLine(" reset --game <id> Reset a game to unprocessed");
Console.WriteLine(" migrate-chunks Move dynamically-allocated chunks (6-9, 13+) to fixed slots:");
Console.WriteLine(" Promo→10, Screenshots→11, Media→12. Run once after upgrading.");
break;
}

View File

@@ -137,9 +137,7 @@ public class MediaScraper
bool hasVideos = html.Contains("lazyframe", StringComparison.OrdinalIgnoreCase);
// Store in mobygames_raw as a new chunk (even without videos, to avoid re-fetching)
int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0;
await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html);
await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkMedia, html);
if(hasVideos)
{

View File

@@ -61,6 +61,9 @@ public class NewGameRawFetcher
const int ChunkSpecs = 3;
const int ChunkCovers = 4;
const int ChunkReviews = 5;
internal const int ChunkPromo = 10;
internal const int ChunkScreenshots = 11;
internal const int ChunkMedia = 12;
readonly DiscoveryStateService _discovery;
readonly MobyGamesHttpClient _http;
readonly SourceDatabaseService _sourceDb;

View File

@@ -7,6 +7,7 @@ using Marechai.Data;
using Marechai.Database.Models;
using Marechai.MobyGames.Models;
using Marechai.MobyGames.Parsers;
using NewSite = Marechai.MobyGames.Parsers.NewSite;
using Microsoft.EntityFrameworkCore;
namespace Marechai.MobyGames.Services;
@@ -70,6 +71,7 @@ public class PromoArtDownloadService
int skippedCount = 0;
int failedCount = 0;
int noPromoPage = 0;
int parseFailed = 0;
int gamesProcessed = 0;
bool aborted = false;
@@ -78,20 +80,8 @@ public class PromoArtDownloadService
if(aborted) break;
gamesProcessed++;
var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId);
// Find the promo art page chunk (new MobyGames HTML stored by scraper)
string promoHtml = null;
foreach(var row in rows)
{
// New site promo pages contain /promo/group- pattern
if(row.Body.Contains("/promo/group-"))
{
promoHtml = row.Body;
break;
}
}
// Fetch the promo art page directly from the fixed chunk slot
string promoHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkPromo);
if(promoHtml is null)
{
@@ -106,11 +96,18 @@ public class PromoArtDownloadService
continue;
}
var promoGroups = PromoArtPageParser.Parse(promoHtml);
var promoGroups = NewSite.PromoArtTabParser.Parse(promoHtml);
if(promoGroups.Count == 0)
{
Console.WriteLine($" [{gamesProcessed}/{Math.Min(batchSize, importedGames.Count)}] {game.MobyGameId}: Promo page found but no groups parsed");
parseFailed++;
if(dryRun || parseFailed <= 20)
Console.WriteLine($" [{gamesProcessed}/{Math.Min(batchSize, importedGames.Count)}] {game.MobyGameId}: Promo page found but no groups parsed");
if(parseFailed == 20 && !dryRun)
Console.WriteLine(" ... suppressing further 'no groups parsed' messages ...");
continue;
}
@@ -130,7 +127,7 @@ public class PromoArtDownloadService
int groupId = 0;
if(!dryRun)
groupId = await GetOrCreateGroupAsync(group.GroupName);
groupId = await GetOrCreateGroupAsync(group.GroupName, group.GroupId);
foreach(var image in group.Images)
{
@@ -317,12 +314,14 @@ public class PromoArtDownloadService
Console.WriteLine(" \e[33;1m[DRY RUN]\e[0m No changes made");
Console.WriteLine($" Games scanned: {gamesProcessed}");
Console.WriteLine($" No promo page: {noPromoPage}");
Console.WriteLine($" Parse failed: {parseFailed}");
Console.WriteLine($" Total images found: {totalImages}");
}
else
{
Console.WriteLine($" Games processed: {gamesProcessed}");
Console.WriteLine($" No promo page: {noPromoPage}");
Console.WriteLine($" Parse failed: {parseFailed}");
Console.WriteLine($" Total images found: {totalImages}");
Console.WriteLine($" Downloaded: {downloadedCount}");
Console.WriteLine($" Skipped (existing): {skippedCount}");
@@ -332,10 +331,57 @@ public class PromoArtDownloadService
Console.WriteLine(" ────────────────────────────────────\n");
}
async Task<int> GetOrCreateGroupAsync(string groupName)
async Task<int> GetOrCreateGroupAsync(string groupName, string groupId = null)
{
await using var context = await _contextFactory.CreateDbContextAsync();
// If we have a groupId, check if a previous run created a bad "Group {id}" entry
// and rename it to the real name. Skip if the current name IS the bad pattern
// (regex fallback couldn't extract the real name — nothing to repair).
if(groupId is not null)
{
string badName = $"Group {groupId}";
if(groupName != badName)
{
var badEntry = await context.SoftwarePromoArtGroups
.FirstOrDefaultAsync(g => g.Name == badName);
if(badEntry is not null)
{
// Check if the real name already exists too
var realEntry = await context.SoftwarePromoArtGroups
.FirstOrDefaultAsync(g => g.Name == groupName);
if(realEntry is null)
{
// Just rename the bad entry
badEntry.Name = groupName;
await context.SaveChangesAsync();
Console.WriteLine($" \e[33mRepaired\e[0m group \"{badName}\" → \"{groupName}\"");
return badEntry.Id;
}
// Both exist — migrate promo art from bad group to real group, then delete bad
var orphaned = await context.SoftwarePromoArt
.Where(p => p.GroupId == badEntry.Id)
.ToListAsync();
foreach(var art in orphaned)
art.GroupId = realEntry.Id;
context.SoftwarePromoArtGroups.Remove(badEntry);
await context.SaveChangesAsync();
if(orphaned.Count > 0)
Console.WriteLine($" \e[33mRepaired\e[0m migrated {orphaned.Count} images from \"{badName}\" → \"{groupName}\"");
return realEntry.Id;
}
}
}
var existing = await context.SoftwarePromoArtGroups
.FirstOrDefaultAsync(g => g.Name == groupName);

View File

@@ -113,9 +113,22 @@ public class PromoArtScraper
Console.WriteLine($" [{gamesChecked}/{Math.Min(batchSize, importedGames.Count)}] \e[36;1m{game.MobyGameId}\e[0m: Has promo art");
// Check if we already scraped this game's promo page
// New site pages contain /promo/group- pattern (old site doesn't)
bool alreadyScraped = rows.Any(r => r.Body.Contains("/promo/group-"));
// Check if we already scraped this game's promo page.
// Must verify via TabDetector that the row IS actually a promo page,
// not just the main page which also contains /promo/group- image links.
bool alreadyScraped = false;
foreach(var row in rows)
{
var (tab, _) = TabDetector.DetectWithLayout(row.Body);
if(tab == MobyTab.PromoArt)
{
alreadyScraped = true;
break;
}
}
if(alreadyScraped)
{
@@ -185,10 +198,8 @@ public class PromoArtScraper
continue;
}
// Store in mobygames_raw as a new chunk
int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0;
await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html);
// Store in mobygames_raw at the fixed promo chunk slot
await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkPromo, html);
Console.WriteLine(" \e[32mOK\e[0m");
scraped++;

View File

@@ -81,21 +81,8 @@ public class ScreenshotDownloadService
gamesProcessed++;
var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId);
// Find the screenshot page chunk (new MobyGames HTML)
// The gtag content_type "game-list-screenshots" is always present on screenshot list pages
string screenshotHtml = null;
foreach(var row in rows)
{
if(row.Body.Contains("game-list-screenshots"))
{
screenshotHtml = row.Body;
break;
}
}
// Fetch the screenshot page directly from the fixed chunk slot
string screenshotHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkScreenshots);
if(screenshotHtml is null)
{

View File

@@ -197,9 +197,7 @@ public class ScreenshotScraper
}
// Store in mobygames_raw as a new chunk
int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0;
await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html);
await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkScreenshots, html);
Console.WriteLine(" \e[32mOK\e[0m");
scraped++;

View File

@@ -110,6 +110,101 @@ public class SourceDatabaseService
await cmd.ExecuteNonQueryAsync();
}
/// <summary>
/// Moves a row from one chunk number to another. Used by the chunk migrator
/// to reassign dynamically-allocated chunks to their fixed slot numbers.
/// Returns the number of affected rows (0 if the source chunk didn't exist,
/// or the target chunk already exists via <c>INSERT IGNORE</c>).
/// </summary>
public async Task<int> MoveChunkAsync(string gameId, int fromChunk, int toChunk)
{
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
// Copy to new chunk (IGNORE if target already exists), then delete old
await using var insert = new MySqlCommand(
"INSERT IGNORE INTO mobygames_raw (id, chunk, body) SELECT id, @toChunk, body FROM mobygames_raw WHERE id = @id AND chunk = @fromChunk",
connection);
insert.Parameters.AddWithValue("@id", gameId);
insert.Parameters.AddWithValue("@fromChunk", fromChunk);
insert.Parameters.AddWithValue("@toChunk", toChunk);
int inserted = await insert.ExecuteNonQueryAsync();
await using var delete = new MySqlCommand(
"DELETE FROM mobygames_raw WHERE id = @id AND chunk = @fromChunk", connection);
delete.Parameters.AddWithValue("@id", gameId);
delete.Parameters.AddWithValue("@fromChunk", fromChunk);
await delete.ExecuteNonQueryAsync();
return inserted;
}
/// <summary>
/// Returns all (id, chunk) pairs whose chunk number is outside the known fixed slots
/// (0-5 and 10-12). These are candidates for chunk migration.
/// </summary>
public async Task<List<(string Id, int Chunk)>> GetMisplacedChunksAsync()
{
var results = new List<(string, int)>();
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
await using var cmd = new MySqlCommand(
"SELECT id, chunk FROM mobygames_raw WHERE chunk NOT IN (0,1,2,3,4,5,10,11,12) ORDER BY id, chunk",
connection);
await using var reader = await cmd.ExecuteReaderAsync();
while(await reader.ReadAsync())
results.Add((reader.GetString(0), reader.GetInt32(1)));
return results;
}
/// <summary>
/// Returns the body of a single (id, chunk) row. Used by the chunk migrator
/// to detect the page type without loading all chunks for a game.
/// </summary>
public async Task<string> GetChunkBodyAsync(string gameId, int chunk)
{
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
await using var cmd = new MySqlCommand(
"SELECT body FROM mobygames_raw WHERE id = @id AND chunk = @chunk LIMIT 1", connection);
cmd.Parameters.AddWithValue("@id", gameId);
cmd.Parameters.AddWithValue("@chunk", chunk);
var result = await cmd.ExecuteScalarAsync();
return result as string;
}
/// <summary>
/// Returns true if a row exists for the given (id, chunk) pair.
/// </summary>
public async Task<bool> ChunkExistsAsync(string gameId, int chunk)
{
await using var connection = new MySqlConnection(_connectionString);
await connection.OpenAsync();
await using var cmd = new MySqlCommand(
"SELECT EXISTS (SELECT 1 FROM mobygames_raw WHERE id = @id AND chunk = @chunk LIMIT 1)", connection);
cmd.Parameters.AddWithValue("@id", gameId);
cmd.Parameters.AddWithValue("@chunk", chunk);
var result = await cmd.ExecuteScalarAsync();
return System.Convert.ToInt32(result) == 1;
}
/// <summary>
/// Delete every cached chunk for the given slug. Used to evict stale legacy-layout
/// captures before re-scraping the current new-layout page (see

View File

@@ -61,21 +61,8 @@ public class VideoImportService
{
gamesProcessed++;
var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId);
// Find the media page chunk (new MobyGames HTML)
// The gtag content_type "game-list-media" is present on media pages
string mediaHtml = null;
foreach(var row in rows)
{
if(row.Body.Contains("game-list-media"))
{
mediaHtml = row.Body;
break;
}
}
// Fetch the media page directly from the fixed chunk slot
string mediaHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkMedia);
if(mediaHtml is null)
{