From a0a37944a4a69daf9857bfeeedd00b27fdfd314a Mon Sep 17 00:00:00 2001 From: Natalia Portillo Date: Sat, 6 Jun 2026 01:25:41 +0100 Subject: [PATCH] feat: implement PromoArtTabParser and migrate chunk handling to fixed slots --- .../Parsers/NewSite/PromoArtTabParser.cs | 224 ++++++++++++++++++ Marechai.MobyGames/Program.cs | 92 +++++++ Marechai.MobyGames/Services/MediaScraper.cs | 4 +- .../Services/NewGameRawFetcher.cs | 3 + .../Services/PromoArtDownloadService.cs | 82 +++++-- .../Services/PromoArtScraper.cs | 25 +- .../Services/ScreenshotDownloadService.cs | 17 +- .../Services/ScreenshotScraper.cs | 4 +- .../Services/SourceDatabaseService.cs | 95 ++++++++ .../Services/VideoImportService.cs | 17 +- 10 files changed, 502 insertions(+), 61 deletions(-) create mode 100644 Marechai.MobyGames/Parsers/NewSite/PromoArtTabParser.cs diff --git a/Marechai.MobyGames/Parsers/NewSite/PromoArtTabParser.cs b/Marechai.MobyGames/Parsers/NewSite/PromoArtTabParser.cs new file mode 100644 index 00000000..d7e919e5 --- /dev/null +++ b/Marechai.MobyGames/Parsers/NewSite/PromoArtTabParser.cs @@ -0,0 +1,224 @@ +/****************************************************************************** +// MARECHAI: Master repository of computing history artifacts information +// ---------------------------------------------------------------------------- +// +// Author(s) : Natalia Portillo +// +// --[ License ] -------------------------------------------------------------- +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . +// +// ---------------------------------------------------------------------------- +// Copyright © 2003-2026 Natalia Portillo +*******************************************************************************/ + +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using HtmlAgilityPack; +using Marechai.MobyGames.Models; + +namespace Marechai.MobyGames.Parsers.NewSite; + +/// +/// Parses the new MobyGames promo art page (post-2023 Vue redesign). +/// Each promo group lives in a <section id="promo-image-group-{id}"> +/// containing an <h2> with the group name and link, followed by +/// one or more <figure> elements inside a +/// <div class="img-holder mb">. +/// +public static partial class PromoArtTabParser +{ + [GeneratedRegex(@"group-(\d+)", RegexOptions.Compiled)] + private static partial Regex GroupIdRegex(); + + [GeneratedRegex(@"image-(\d+)", RegexOptions.Compiled)] + private static partial Regex ImageIdRegex(); + + /// + /// Regex fallback: extracts image references directly from raw HTML, + /// bypassing HtmlAgilityPack entirely. Used when DOM parsing fails + /// (e.g. due to malformed Vue component attributes corrupting the tree). + /// + [GeneratedRegex( + @"href=""[^""]*?/promo/group-(\d+)/image-(\d+)/""", + RegexOptions.Compiled | RegexOptions.IgnoreCase)] + private static partial Regex ImageHrefRegex(); + + /// Extracts group name from any anchor linking to /promo/group-{id}/. + [GeneratedRegex( + @"]+href=""[^""]*?/promo/group-(\d+)/""[^>]*>\s*([^<]+?)\s*", + RegexOptions.Compiled | RegexOptions.IgnoreCase)] + private static partial Regex GroupLinkRegex(); + + public static List Parse(string html) + { + var doc = new HtmlDocument(); + doc.LoadHtml(html); + + var groups = Parse(doc); + + // If XPath-based parsing found nothing, try regex fallback + if(groups.Count == 0 && html.Contains("/promo/group-")) + groups = ParseWithRegex(html); + + return groups; + } + + public static List Parse(HtmlDocument doc) + { + var groups = new List(); + + // Target the
containers directly. + // This bypasses the navbar/header Vue components whose massive JSON attributes + // can corrupt HAP's DOM tree. + var sections = doc.DocumentNode.SelectNodes("//section[starts-with(@id, 'promo-image-group-')]"); + + if(sections is null) return groups; + + foreach(var section in sections) + { + // Extract group name and ID from the h2 anchor + var groupLink = section.SelectSingleNode(".//h2//a[contains(@href, '/promo/group-')]"); + + if(groupLink is null) continue; + + string groupName = WebUtility.HtmlDecode(groupLink.InnerText).Trim(); + string groupHref = groupLink.GetAttributeValue("href", ""); + var groupMatch = GroupIdRegex().Match(groupHref); + string groupId = groupMatch.Success ? groupMatch.Groups[1].Value : null; + + // Also try extracting from the section id attribute + if(groupId is null) + { + string sectionId = section.GetAttributeValue("id", ""); + + // id="promo-image-group-90458" → extract 90458 + if(sectionId.StartsWith("promo-image-group-")) + groupId = sectionId["promo-image-group-".Length..]; + } + + var group = new ParsedPromoArtGroup + { + GroupName = groupName, + GroupId = groupId + }; + + // Find all
elements within this section (may be nested in
) + var figures = section.SelectNodes(".//figure"); + + if(figures is not null) + { + foreach(var figure in figures) + { + var a = figure.SelectSingleNode(".//a[@href]"); + + if(a is null) continue; + + string href = a.GetAttributeValue("href", ""); + var imageMatch = ImageIdRegex().Match(href); + + if(!imageMatch.Success) continue; + + string imageId = imageMatch.Groups[1].Value; + + // Skip duplicates + if(group.Images.Any(i => i.ImageId == imageId)) continue; + + // Caption from
+ var fc = figure.SelectSingleNode(".//figcaption"); + string caption = fc is null ? null : WebUtility.HtmlDecode(fc.InnerText).Trim(); + + if(string.IsNullOrWhiteSpace(caption)) caption = null; + + // Thumbnail URL from src + var img = figure.SelectSingleNode(".//img"); + string thumbnailUrl = img?.GetAttributeValue("src", null); + + group.Images.Add(new ParsedPromoArtImage + { + ImageId = imageId, + DetailPageUrl = href, + Caption = caption, + ThumbnailUrl = thumbnailUrl + }); + } + } + + if(group.Images.Count > 0) + groups.Add(group); + } + + return groups; + } + + /// + /// Regex-only fallback that scans raw HTML for promo image href patterns. + /// Groups images by their group ID. Used when HAP DOM parsing fails. + /// Groups whose name cannot be determined are skipped entirely. + /// + static List ParseWithRegex(string html) + { + var groupMap = new Dictionary(); + + // Extract group names from Name links + var headingNames = new Dictionary(); + + foreach(Match hm in GroupLinkRegex().Matches(html)) + { + string gid = hm.Groups[1].Value; + string name = WebUtility.HtmlDecode(hm.Groups[2].Value).Trim(); + + if(!string.IsNullOrWhiteSpace(name)) + headingNames.TryAdd(gid, name); + } + + // Extract all image references — only for groups whose name we found + foreach(Match m in ImageHrefRegex().Matches(html)) + { + string groupId = m.Groups[1].Value; + string imageId = m.Groups[2].Value; + + // Skip groups where we couldn't determine the real name + if(!headingNames.ContainsKey(groupId)) continue; + + if(!groupMap.TryGetValue(groupId, out var group)) + { + group = new ParsedPromoArtGroup + { + GroupId = groupId, + GroupName = headingNames[groupId] + }; + + groupMap[groupId] = group; + } + + if(group.Images.Any(i => i.ImageId == imageId)) continue; + + // Reconstruct the detail page URL from the matched href + string fullHref = m.Value.Length > 6 + ? m.Value[6..^1] // strip href=" and trailing " + : $"/promo/group-{groupId}/image-{imageId}/"; + + group.Images.Add(new ParsedPromoArtImage + { + ImageId = imageId, + DetailPageUrl = fullHref + }); + } + + return groupMap.Values.Where(g => g.Images.Count > 0).ToList(); + } +} diff --git a/Marechai.MobyGames/Program.cs b/Marechai.MobyGames/Program.cs index 159d140e..b579b9f8 100644 --- a/Marechai.MobyGames/Program.cs +++ b/Marechai.MobyGames/Program.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using Marechai.Database; using Marechai.Database.Models; @@ -776,6 +778,94 @@ class Program break; } + case "migrate-chunks": + { + Console.WriteLine("\n Migrating dynamically-allocated chunks to fixed slot numbers...\n"); + Console.WriteLine(" Fixed slots: Promo=10, Screenshots=11, Media=12\n"); + + var misplaced = await sourceDb.GetMisplacedChunksAsync(); + + Console.WriteLine($" Found {misplaced.Count} chunk(s) outside fixed slots\n"); + + int migrated = 0; + int skipped = 0; + int errors = 0; + var skipReasons = new Dictionary(); + + for(int i = 0; i < misplaced.Count; i++) + { + var (gameId, chunk) = misplaced[i]; + + if((i + 1) % 1000 == 0) + Console.WriteLine($" [{i + 1}/{misplaced.Count}] migrated={migrated} skipped={skipped} errors={errors}"); + + string body = await sourceDb.GetChunkBodyAsync(gameId, chunk); + + if(body is null) + { + skipped++; + continue; + } + + var (tab, _) = Parsers.TabDetector.DetectWithLayout(body); + + int? targetChunk = tab switch + { + Parsers.MobyTab.PromoArt => NewGameRawFetcher.ChunkPromo, + Parsers.MobyTab.Screenshots => NewGameRawFetcher.ChunkScreenshots, + Parsers.MobyTab.Media => NewGameRawFetcher.ChunkMedia, + _ => null + }; + + if(targetChunk is null) + { + string reason = tab.ToString(); + skipReasons.TryGetValue(reason, out int cnt); + skipReasons[reason] = cnt + 1; + skipped++; + continue; + } + + // Don't overwrite if target slot already has data + if(await sourceDb.ChunkExistsAsync(gameId, targetChunk.Value)) + { + string reason = $"{tab} → {targetChunk.Value} (slot occupied)"; + skipReasons.TryGetValue(reason, out int cnt); + skipReasons[reason] = cnt + 1; + skipped++; + continue; + } + + try + { + await sourceDb.MoveChunkAsync(gameId, chunk, targetChunk.Value); + migrated++; + } + catch(Exception ex) + { + Console.WriteLine($" ERROR {gameId} chunk {chunk} → {targetChunk.Value}: {ex.Message}"); + errors++; + } + } + + Console.WriteLine($"\n ────────────────────────────────────"); + Console.WriteLine($" Chunks scanned: {misplaced.Count}"); + Console.WriteLine($" Migrated: {migrated}"); + Console.WriteLine($" Skipped: {skipped}"); + Console.WriteLine($" Errors: {errors}"); + + if(skipReasons.Count > 0) + { + Console.WriteLine(" Skip breakdown:"); + + foreach(var kv in skipReasons.OrderByDescending(kv => kv.Value)) + Console.WriteLine($" {kv.Key}: {kv.Value}"); + } + Console.WriteLine($" ────────────────────────────────────\n"); + + break; + } + default: Console.WriteLine(" Usage:"); Console.WriteLine(" import [--batch-size N] [--unattended] [--yes-to-all]"); @@ -831,6 +921,8 @@ class Program Console.WriteLine(" cached rows under both '' and '-' first. Use to evict stale"); Console.WriteLine(" legacy-layout captures (e.g. since-corrected MobyGames data)."); Console.WriteLine(" reset --game Reset a game to unprocessed"); + Console.WriteLine(" migrate-chunks Move dynamically-allocated chunks (6-9, 13+) to fixed slots:"); + Console.WriteLine(" Promo→10, Screenshots→11, Media→12. Run once after upgrading."); break; } diff --git a/Marechai.MobyGames/Services/MediaScraper.cs b/Marechai.MobyGames/Services/MediaScraper.cs index 66aa346a..60a80ca6 100644 --- a/Marechai.MobyGames/Services/MediaScraper.cs +++ b/Marechai.MobyGames/Services/MediaScraper.cs @@ -137,9 +137,7 @@ public class MediaScraper bool hasVideos = html.Contains("lazyframe", StringComparison.OrdinalIgnoreCase); // Store in mobygames_raw as a new chunk (even without videos, to avoid re-fetching) - int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0; - - await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html); + await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkMedia, html); if(hasVideos) { diff --git a/Marechai.MobyGames/Services/NewGameRawFetcher.cs b/Marechai.MobyGames/Services/NewGameRawFetcher.cs index 201fd77b..cefc1051 100644 --- a/Marechai.MobyGames/Services/NewGameRawFetcher.cs +++ b/Marechai.MobyGames/Services/NewGameRawFetcher.cs @@ -61,6 +61,9 @@ public class NewGameRawFetcher const int ChunkSpecs = 3; const int ChunkCovers = 4; const int ChunkReviews = 5; + internal const int ChunkPromo = 10; + internal const int ChunkScreenshots = 11; + internal const int ChunkMedia = 12; readonly DiscoveryStateService _discovery; readonly MobyGamesHttpClient _http; readonly SourceDatabaseService _sourceDb; diff --git a/Marechai.MobyGames/Services/PromoArtDownloadService.cs b/Marechai.MobyGames/Services/PromoArtDownloadService.cs index 45a12695..41f3e26e 100644 --- a/Marechai.MobyGames/Services/PromoArtDownloadService.cs +++ b/Marechai.MobyGames/Services/PromoArtDownloadService.cs @@ -7,6 +7,7 @@ using Marechai.Data; using Marechai.Database.Models; using Marechai.MobyGames.Models; using Marechai.MobyGames.Parsers; +using NewSite = Marechai.MobyGames.Parsers.NewSite; using Microsoft.EntityFrameworkCore; namespace Marechai.MobyGames.Services; @@ -70,6 +71,7 @@ public class PromoArtDownloadService int skippedCount = 0; int failedCount = 0; int noPromoPage = 0; + int parseFailed = 0; int gamesProcessed = 0; bool aborted = false; @@ -78,20 +80,8 @@ public class PromoArtDownloadService if(aborted) break; gamesProcessed++; - var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId); - - // Find the promo art page chunk (new MobyGames HTML stored by scraper) - string promoHtml = null; - - foreach(var row in rows) - { - // New site promo pages contain /promo/group- pattern - if(row.Body.Contains("/promo/group-")) - { - promoHtml = row.Body; - break; - } - } + // Fetch the promo art page directly from the fixed chunk slot + string promoHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkPromo); if(promoHtml is null) { @@ -106,11 +96,18 @@ public class PromoArtDownloadService continue; } - var promoGroups = PromoArtPageParser.Parse(promoHtml); + var promoGroups = NewSite.PromoArtTabParser.Parse(promoHtml); if(promoGroups.Count == 0) { - Console.WriteLine($" [{gamesProcessed}/{Math.Min(batchSize, importedGames.Count)}] {game.MobyGameId}: Promo page found but no groups parsed"); + parseFailed++; + + if(dryRun || parseFailed <= 20) + Console.WriteLine($" [{gamesProcessed}/{Math.Min(batchSize, importedGames.Count)}] {game.MobyGameId}: Promo page found but no groups parsed"); + + if(parseFailed == 20 && !dryRun) + Console.WriteLine(" ... suppressing further 'no groups parsed' messages ..."); + continue; } @@ -130,7 +127,7 @@ public class PromoArtDownloadService int groupId = 0; if(!dryRun) - groupId = await GetOrCreateGroupAsync(group.GroupName); + groupId = await GetOrCreateGroupAsync(group.GroupName, group.GroupId); foreach(var image in group.Images) { @@ -317,12 +314,14 @@ public class PromoArtDownloadService Console.WriteLine(" \e[33;1m[DRY RUN]\e[0m No changes made"); Console.WriteLine($" Games scanned: {gamesProcessed}"); Console.WriteLine($" No promo page: {noPromoPage}"); + Console.WriteLine($" Parse failed: {parseFailed}"); Console.WriteLine($" Total images found: {totalImages}"); } else { Console.WriteLine($" Games processed: {gamesProcessed}"); Console.WriteLine($" No promo page: {noPromoPage}"); + Console.WriteLine($" Parse failed: {parseFailed}"); Console.WriteLine($" Total images found: {totalImages}"); Console.WriteLine($" Downloaded: {downloadedCount}"); Console.WriteLine($" Skipped (existing): {skippedCount}"); @@ -332,10 +331,57 @@ public class PromoArtDownloadService Console.WriteLine(" ────────────────────────────────────\n"); } - async Task GetOrCreateGroupAsync(string groupName) + async Task GetOrCreateGroupAsync(string groupName, string groupId = null) { await using var context = await _contextFactory.CreateDbContextAsync(); + // If we have a groupId, check if a previous run created a bad "Group {id}" entry + // and rename it to the real name. Skip if the current name IS the bad pattern + // (regex fallback couldn't extract the real name — nothing to repair). + if(groupId is not null) + { + string badName = $"Group {groupId}"; + + if(groupName != badName) + { + var badEntry = await context.SoftwarePromoArtGroups + .FirstOrDefaultAsync(g => g.Name == badName); + + if(badEntry is not null) + { + // Check if the real name already exists too + var realEntry = await context.SoftwarePromoArtGroups + .FirstOrDefaultAsync(g => g.Name == groupName); + + if(realEntry is null) + { + // Just rename the bad entry + badEntry.Name = groupName; + await context.SaveChangesAsync(); + Console.WriteLine($" \e[33mRepaired\e[0m group \"{badName}\" → \"{groupName}\""); + + return badEntry.Id; + } + + // Both exist — migrate promo art from bad group to real group, then delete bad + var orphaned = await context.SoftwarePromoArt + .Where(p => p.GroupId == badEntry.Id) + .ToListAsync(); + + foreach(var art in orphaned) + art.GroupId = realEntry.Id; + + context.SoftwarePromoArtGroups.Remove(badEntry); + await context.SaveChangesAsync(); + + if(orphaned.Count > 0) + Console.WriteLine($" \e[33mRepaired\e[0m migrated {orphaned.Count} images from \"{badName}\" → \"{groupName}\""); + + return realEntry.Id; + } + } + } + var existing = await context.SoftwarePromoArtGroups .FirstOrDefaultAsync(g => g.Name == groupName); diff --git a/Marechai.MobyGames/Services/PromoArtScraper.cs b/Marechai.MobyGames/Services/PromoArtScraper.cs index ba32361e..da9967eb 100644 --- a/Marechai.MobyGames/Services/PromoArtScraper.cs +++ b/Marechai.MobyGames/Services/PromoArtScraper.cs @@ -113,9 +113,22 @@ public class PromoArtScraper Console.WriteLine($" [{gamesChecked}/{Math.Min(batchSize, importedGames.Count)}] \e[36;1m{game.MobyGameId}\e[0m: Has promo art"); - // Check if we already scraped this game's promo page - // New site pages contain /promo/group- pattern (old site doesn't) - bool alreadyScraped = rows.Any(r => r.Body.Contains("/promo/group-")); + // Check if we already scraped this game's promo page. + // Must verify via TabDetector that the row IS actually a promo page, + // not just the main page which also contains /promo/group- image links. + bool alreadyScraped = false; + + foreach(var row in rows) + { + var (tab, _) = TabDetector.DetectWithLayout(row.Body); + + if(tab == MobyTab.PromoArt) + { + alreadyScraped = true; + + break; + } + } if(alreadyScraped) { @@ -185,10 +198,8 @@ public class PromoArtScraper continue; } - // Store in mobygames_raw as a new chunk - int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0; - - await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html); + // Store in mobygames_raw at the fixed promo chunk slot + await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkPromo, html); Console.WriteLine(" \e[32mOK\e[0m"); scraped++; diff --git a/Marechai.MobyGames/Services/ScreenshotDownloadService.cs b/Marechai.MobyGames/Services/ScreenshotDownloadService.cs index 3d6f358c..066614b8 100644 --- a/Marechai.MobyGames/Services/ScreenshotDownloadService.cs +++ b/Marechai.MobyGames/Services/ScreenshotDownloadService.cs @@ -81,21 +81,8 @@ public class ScreenshotDownloadService gamesProcessed++; - var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId); - - // Find the screenshot page chunk (new MobyGames HTML) - // The gtag content_type "game-list-screenshots" is always present on screenshot list pages - string screenshotHtml = null; - - foreach(var row in rows) - { - if(row.Body.Contains("game-list-screenshots")) - { - screenshotHtml = row.Body; - - break; - } - } + // Fetch the screenshot page directly from the fixed chunk slot + string screenshotHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkScreenshots); if(screenshotHtml is null) { diff --git a/Marechai.MobyGames/Services/ScreenshotScraper.cs b/Marechai.MobyGames/Services/ScreenshotScraper.cs index 7fc29779..ddbcad5a 100644 --- a/Marechai.MobyGames/Services/ScreenshotScraper.cs +++ b/Marechai.MobyGames/Services/ScreenshotScraper.cs @@ -197,9 +197,7 @@ public class ScreenshotScraper } // Store in mobygames_raw as a new chunk - int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0; - - await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html); + await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkScreenshots, html); Console.WriteLine(" \e[32mOK\e[0m"); scraped++; diff --git a/Marechai.MobyGames/Services/SourceDatabaseService.cs b/Marechai.MobyGames/Services/SourceDatabaseService.cs index c5313756..2413efee 100644 --- a/Marechai.MobyGames/Services/SourceDatabaseService.cs +++ b/Marechai.MobyGames/Services/SourceDatabaseService.cs @@ -110,6 +110,101 @@ public class SourceDatabaseService await cmd.ExecuteNonQueryAsync(); } + /// + /// Moves a row from one chunk number to another. Used by the chunk migrator + /// to reassign dynamically-allocated chunks to their fixed slot numbers. + /// Returns the number of affected rows (0 if the source chunk didn't exist, + /// or the target chunk already exists via INSERT IGNORE). + /// + public async Task MoveChunkAsync(string gameId, int fromChunk, int toChunk) + { + await using var connection = new MySqlConnection(_connectionString); + await connection.OpenAsync(); + + // Copy to new chunk (IGNORE if target already exists), then delete old + await using var insert = new MySqlCommand( + "INSERT IGNORE INTO mobygames_raw (id, chunk, body) SELECT id, @toChunk, body FROM mobygames_raw WHERE id = @id AND chunk = @fromChunk", + connection); + + insert.Parameters.AddWithValue("@id", gameId); + insert.Parameters.AddWithValue("@fromChunk", fromChunk); + insert.Parameters.AddWithValue("@toChunk", toChunk); + + int inserted = await insert.ExecuteNonQueryAsync(); + + await using var delete = new MySqlCommand( + "DELETE FROM mobygames_raw WHERE id = @id AND chunk = @fromChunk", connection); + + delete.Parameters.AddWithValue("@id", gameId); + delete.Parameters.AddWithValue("@fromChunk", fromChunk); + + await delete.ExecuteNonQueryAsync(); + + return inserted; + } + + /// + /// Returns all (id, chunk) pairs whose chunk number is outside the known fixed slots + /// (0-5 and 10-12). These are candidates for chunk migration. + /// + public async Task> GetMisplacedChunksAsync() + { + var results = new List<(string, int)>(); + + await using var connection = new MySqlConnection(_connectionString); + await connection.OpenAsync(); + + await using var cmd = new MySqlCommand( + "SELECT id, chunk FROM mobygames_raw WHERE chunk NOT IN (0,1,2,3,4,5,10,11,12) ORDER BY id, chunk", + connection); + + await using var reader = await cmd.ExecuteReaderAsync(); + + while(await reader.ReadAsync()) + results.Add((reader.GetString(0), reader.GetInt32(1))); + + return results; + } + + /// + /// Returns the body of a single (id, chunk) row. Used by the chunk migrator + /// to detect the page type without loading all chunks for a game. + /// + public async Task GetChunkBodyAsync(string gameId, int chunk) + { + await using var connection = new MySqlConnection(_connectionString); + await connection.OpenAsync(); + + await using var cmd = new MySqlCommand( + "SELECT body FROM mobygames_raw WHERE id = @id AND chunk = @chunk LIMIT 1", connection); + + cmd.Parameters.AddWithValue("@id", gameId); + cmd.Parameters.AddWithValue("@chunk", chunk); + + var result = await cmd.ExecuteScalarAsync(); + + return result as string; + } + + /// + /// Returns true if a row exists for the given (id, chunk) pair. + /// + public async Task ChunkExistsAsync(string gameId, int chunk) + { + await using var connection = new MySqlConnection(_connectionString); + await connection.OpenAsync(); + + await using var cmd = new MySqlCommand( + "SELECT EXISTS (SELECT 1 FROM mobygames_raw WHERE id = @id AND chunk = @chunk LIMIT 1)", connection); + + cmd.Parameters.AddWithValue("@id", gameId); + cmd.Parameters.AddWithValue("@chunk", chunk); + + var result = await cmd.ExecuteScalarAsync(); + + return System.Convert.ToInt32(result) == 1; + } + /// /// Delete every cached chunk for the given slug. Used to evict stale legacy-layout /// captures before re-scraping the current new-layout page (see diff --git a/Marechai.MobyGames/Services/VideoImportService.cs b/Marechai.MobyGames/Services/VideoImportService.cs index 208f8e86..dd5658c5 100644 --- a/Marechai.MobyGames/Services/VideoImportService.cs +++ b/Marechai.MobyGames/Services/VideoImportService.cs @@ -61,21 +61,8 @@ public class VideoImportService { gamesProcessed++; - var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId); - - // Find the media page chunk (new MobyGames HTML) - // The gtag content_type "game-list-media" is present on media pages - string mediaHtml = null; - - foreach(var row in rows) - { - if(row.Body.Contains("game-list-media")) - { - mediaHtml = row.Body; - - break; - } - } + // Fetch the media page directly from the fixed chunk slot + string mediaHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkMedia); if(mediaHtml is null) {