mirror of
https://github.com/claunia/marechai.git
synced 2026-07-08 17:57:08 +00:00
feat: implement PromoArtTabParser and migrate chunk handling to fixed slots
This commit is contained in:
224
Marechai.MobyGames/Parsers/NewSite/PromoArtTabParser.cs
Normal file
224
Marechai.MobyGames/Parsers/NewSite/PromoArtTabParser.cs
Normal file
@@ -0,0 +1,224 @@
|
||||
/******************************************************************************
|
||||
// MARECHAI: Master repository of computing history artifacts information
|
||||
// ----------------------------------------------------------------------------
|
||||
//
|
||||
// Author(s) : Natalia Portillo <claunia@claunia.com>
|
||||
//
|
||||
// --[ License ] --------------------------------------------------------------
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as
|
||||
// published by the Free Software Foundation, either version 3 of the
|
||||
// License, or (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
//
|
||||
// ----------------------------------------------------------------------------
|
||||
// Copyright © 2003-2026 Natalia Portillo
|
||||
*******************************************************************************/
|
||||
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using Marechai.MobyGames.Models;
|
||||
|
||||
namespace Marechai.MobyGames.Parsers.NewSite;
|
||||
|
||||
/// <summary>
|
||||
/// Parses the new MobyGames promo art page (post-2023 Vue redesign).
|
||||
/// Each promo group lives in a <c><section id="promo-image-group-{id}"></c>
|
||||
/// containing an <c><h2></c> with the group name and link, followed by
|
||||
/// one or more <c><figure></c> elements inside a
|
||||
/// <c><div class="img-holder mb"></c>.
|
||||
/// </summary>
|
||||
public static partial class PromoArtTabParser
|
||||
{
|
||||
[GeneratedRegex(@"group-(\d+)", RegexOptions.Compiled)]
|
||||
private static partial Regex GroupIdRegex();
|
||||
|
||||
[GeneratedRegex(@"image-(\d+)", RegexOptions.Compiled)]
|
||||
private static partial Regex ImageIdRegex();
|
||||
|
||||
/// <summary>
|
||||
/// Regex fallback: extracts image references directly from raw HTML,
|
||||
/// bypassing HtmlAgilityPack entirely. Used when DOM parsing fails
|
||||
/// (e.g. due to malformed Vue component attributes corrupting the tree).
|
||||
/// </summary>
|
||||
[GeneratedRegex(
|
||||
@"href=""[^""]*?/promo/group-(\d+)/image-(\d+)/""",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)]
|
||||
private static partial Regex ImageHrefRegex();
|
||||
|
||||
/// <summary>Extracts group name from any anchor linking to /promo/group-{id}/.</summary>
|
||||
[GeneratedRegex(
|
||||
@"<a[^>]+href=""[^""]*?/promo/group-(\d+)/""[^>]*>\s*([^<]+?)\s*</a>",
|
||||
RegexOptions.Compiled | RegexOptions.IgnoreCase)]
|
||||
private static partial Regex GroupLinkRegex();
|
||||
|
||||
public static List<ParsedPromoArtGroup> Parse(string html)
|
||||
{
|
||||
var doc = new HtmlDocument();
|
||||
doc.LoadHtml(html);
|
||||
|
||||
var groups = Parse(doc);
|
||||
|
||||
// If XPath-based parsing found nothing, try regex fallback
|
||||
if(groups.Count == 0 && html.Contains("/promo/group-"))
|
||||
groups = ParseWithRegex(html);
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
public static List<ParsedPromoArtGroup> Parse(HtmlDocument doc)
|
||||
{
|
||||
var groups = new List<ParsedPromoArtGroup>();
|
||||
|
||||
// Target the <section id="promo-image-group-{id}"> containers directly.
|
||||
// This bypasses the navbar/header Vue components whose massive JSON attributes
|
||||
// can corrupt HAP's DOM tree.
|
||||
var sections = doc.DocumentNode.SelectNodes("//section[starts-with(@id, 'promo-image-group-')]");
|
||||
|
||||
if(sections is null) return groups;
|
||||
|
||||
foreach(var section in sections)
|
||||
{
|
||||
// Extract group name and ID from the h2 anchor
|
||||
var groupLink = section.SelectSingleNode(".//h2//a[contains(@href, '/promo/group-')]");
|
||||
|
||||
if(groupLink is null) continue;
|
||||
|
||||
string groupName = WebUtility.HtmlDecode(groupLink.InnerText).Trim();
|
||||
string groupHref = groupLink.GetAttributeValue("href", "");
|
||||
var groupMatch = GroupIdRegex().Match(groupHref);
|
||||
string groupId = groupMatch.Success ? groupMatch.Groups[1].Value : null;
|
||||
|
||||
// Also try extracting from the section id attribute
|
||||
if(groupId is null)
|
||||
{
|
||||
string sectionId = section.GetAttributeValue("id", "");
|
||||
|
||||
// id="promo-image-group-90458" → extract 90458
|
||||
if(sectionId.StartsWith("promo-image-group-"))
|
||||
groupId = sectionId["promo-image-group-".Length..];
|
||||
}
|
||||
|
||||
var group = new ParsedPromoArtGroup
|
||||
{
|
||||
GroupName = groupName,
|
||||
GroupId = groupId
|
||||
};
|
||||
|
||||
// Find all <figure> elements within this section (may be nested in <div class="img-holder">)
|
||||
var figures = section.SelectNodes(".//figure");
|
||||
|
||||
if(figures is not null)
|
||||
{
|
||||
foreach(var figure in figures)
|
||||
{
|
||||
var a = figure.SelectSingleNode(".//a[@href]");
|
||||
|
||||
if(a is null) continue;
|
||||
|
||||
string href = a.GetAttributeValue("href", "");
|
||||
var imageMatch = ImageIdRegex().Match(href);
|
||||
|
||||
if(!imageMatch.Success) continue;
|
||||
|
||||
string imageId = imageMatch.Groups[1].Value;
|
||||
|
||||
// Skip duplicates
|
||||
if(group.Images.Any(i => i.ImageId == imageId)) continue;
|
||||
|
||||
// Caption from <figcaption>
|
||||
var fc = figure.SelectSingleNode(".//figcaption");
|
||||
string caption = fc is null ? null : WebUtility.HtmlDecode(fc.InnerText).Trim();
|
||||
|
||||
if(string.IsNullOrWhiteSpace(caption)) caption = null;
|
||||
|
||||
// Thumbnail URL from <img> src
|
||||
var img = figure.SelectSingleNode(".//img");
|
||||
string thumbnailUrl = img?.GetAttributeValue("src", null);
|
||||
|
||||
group.Images.Add(new ParsedPromoArtImage
|
||||
{
|
||||
ImageId = imageId,
|
||||
DetailPageUrl = href,
|
||||
Caption = caption,
|
||||
ThumbnailUrl = thumbnailUrl
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if(group.Images.Count > 0)
|
||||
groups.Add(group);
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Regex-only fallback that scans raw HTML for promo image href patterns.
|
||||
/// Groups images by their group ID. Used when HAP DOM parsing fails.
|
||||
/// Groups whose name cannot be determined are skipped entirely.
|
||||
/// </summary>
|
||||
static List<ParsedPromoArtGroup> ParseWithRegex(string html)
|
||||
{
|
||||
var groupMap = new Dictionary<string, ParsedPromoArtGroup>();
|
||||
|
||||
// Extract group names from <a href="...promo/group-{id}/">Name</a> links
|
||||
var headingNames = new Dictionary<string, string>();
|
||||
|
||||
foreach(Match hm in GroupLinkRegex().Matches(html))
|
||||
{
|
||||
string gid = hm.Groups[1].Value;
|
||||
string name = WebUtility.HtmlDecode(hm.Groups[2].Value).Trim();
|
||||
|
||||
if(!string.IsNullOrWhiteSpace(name))
|
||||
headingNames.TryAdd(gid, name);
|
||||
}
|
||||
|
||||
// Extract all image references — only for groups whose name we found
|
||||
foreach(Match m in ImageHrefRegex().Matches(html))
|
||||
{
|
||||
string groupId = m.Groups[1].Value;
|
||||
string imageId = m.Groups[2].Value;
|
||||
|
||||
// Skip groups where we couldn't determine the real name
|
||||
if(!headingNames.ContainsKey(groupId)) continue;
|
||||
|
||||
if(!groupMap.TryGetValue(groupId, out var group))
|
||||
{
|
||||
group = new ParsedPromoArtGroup
|
||||
{
|
||||
GroupId = groupId,
|
||||
GroupName = headingNames[groupId]
|
||||
};
|
||||
|
||||
groupMap[groupId] = group;
|
||||
}
|
||||
|
||||
if(group.Images.Any(i => i.ImageId == imageId)) continue;
|
||||
|
||||
// Reconstruct the detail page URL from the matched href
|
||||
string fullHref = m.Value.Length > 6
|
||||
? m.Value[6..^1] // strip href=" and trailing "
|
||||
: $"/promo/group-{groupId}/image-{imageId}/";
|
||||
|
||||
group.Images.Add(new ParsedPromoArtImage
|
||||
{
|
||||
ImageId = imageId,
|
||||
DetailPageUrl = fullHref
|
||||
});
|
||||
}
|
||||
|
||||
return groupMap.Values.Where(g => g.Images.Count > 0).ToList();
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Marechai.Database;
|
||||
using Marechai.Database.Models;
|
||||
@@ -776,6 +778,94 @@ class Program
|
||||
break;
|
||||
}
|
||||
|
||||
case "migrate-chunks":
|
||||
{
|
||||
Console.WriteLine("\n Migrating dynamically-allocated chunks to fixed slot numbers...\n");
|
||||
Console.WriteLine(" Fixed slots: Promo=10, Screenshots=11, Media=12\n");
|
||||
|
||||
var misplaced = await sourceDb.GetMisplacedChunksAsync();
|
||||
|
||||
Console.WriteLine($" Found {misplaced.Count} chunk(s) outside fixed slots\n");
|
||||
|
||||
int migrated = 0;
|
||||
int skipped = 0;
|
||||
int errors = 0;
|
||||
var skipReasons = new Dictionary<string, int>();
|
||||
|
||||
for(int i = 0; i < misplaced.Count; i++)
|
||||
{
|
||||
var (gameId, chunk) = misplaced[i];
|
||||
|
||||
if((i + 1) % 1000 == 0)
|
||||
Console.WriteLine($" [{i + 1}/{misplaced.Count}] migrated={migrated} skipped={skipped} errors={errors}");
|
||||
|
||||
string body = await sourceDb.GetChunkBodyAsync(gameId, chunk);
|
||||
|
||||
if(body is null)
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var (tab, _) = Parsers.TabDetector.DetectWithLayout(body);
|
||||
|
||||
int? targetChunk = tab switch
|
||||
{
|
||||
Parsers.MobyTab.PromoArt => NewGameRawFetcher.ChunkPromo,
|
||||
Parsers.MobyTab.Screenshots => NewGameRawFetcher.ChunkScreenshots,
|
||||
Parsers.MobyTab.Media => NewGameRawFetcher.ChunkMedia,
|
||||
_ => null
|
||||
};
|
||||
|
||||
if(targetChunk is null)
|
||||
{
|
||||
string reason = tab.ToString();
|
||||
skipReasons.TryGetValue(reason, out int cnt);
|
||||
skipReasons[reason] = cnt + 1;
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Don't overwrite if target slot already has data
|
||||
if(await sourceDb.ChunkExistsAsync(gameId, targetChunk.Value))
|
||||
{
|
||||
string reason = $"{tab} → {targetChunk.Value} (slot occupied)";
|
||||
skipReasons.TryGetValue(reason, out int cnt);
|
||||
skipReasons[reason] = cnt + 1;
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await sourceDb.MoveChunkAsync(gameId, chunk, targetChunk.Value);
|
||||
migrated++;
|
||||
}
|
||||
catch(Exception ex)
|
||||
{
|
||||
Console.WriteLine($" ERROR {gameId} chunk {chunk} → {targetChunk.Value}: {ex.Message}");
|
||||
errors++;
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine($"\n ────────────────────────────────────");
|
||||
Console.WriteLine($" Chunks scanned: {misplaced.Count}");
|
||||
Console.WriteLine($" Migrated: {migrated}");
|
||||
Console.WriteLine($" Skipped: {skipped}");
|
||||
Console.WriteLine($" Errors: {errors}");
|
||||
|
||||
if(skipReasons.Count > 0)
|
||||
{
|
||||
Console.WriteLine(" Skip breakdown:");
|
||||
|
||||
foreach(var kv in skipReasons.OrderByDescending(kv => kv.Value))
|
||||
Console.WriteLine($" {kv.Key}: {kv.Value}");
|
||||
}
|
||||
Console.WriteLine($" ────────────────────────────────────\n");
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
Console.WriteLine(" Usage:");
|
||||
Console.WriteLine(" import [--batch-size N] [--unattended] [--yes-to-all]");
|
||||
@@ -831,6 +921,8 @@ class Program
|
||||
Console.WriteLine(" cached rows under both '<slug>' and '-<slug>' first. Use to evict stale");
|
||||
Console.WriteLine(" legacy-layout captures (e.g. since-corrected MobyGames data).");
|
||||
Console.WriteLine(" reset --game <id> Reset a game to unprocessed");
|
||||
Console.WriteLine(" migrate-chunks Move dynamically-allocated chunks (6-9, 13+) to fixed slots:");
|
||||
Console.WriteLine(" Promo→10, Screenshots→11, Media→12. Run once after upgrading.");
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -137,9 +137,7 @@ public class MediaScraper
|
||||
bool hasVideos = html.Contains("lazyframe", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// Store in mobygames_raw as a new chunk (even without videos, to avoid re-fetching)
|
||||
int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0;
|
||||
|
||||
await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html);
|
||||
await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkMedia, html);
|
||||
|
||||
if(hasVideos)
|
||||
{
|
||||
|
||||
@@ -61,6 +61,9 @@ public class NewGameRawFetcher
|
||||
const int ChunkSpecs = 3;
|
||||
const int ChunkCovers = 4;
|
||||
const int ChunkReviews = 5;
|
||||
internal const int ChunkPromo = 10;
|
||||
internal const int ChunkScreenshots = 11;
|
||||
internal const int ChunkMedia = 12;
|
||||
readonly DiscoveryStateService _discovery;
|
||||
readonly MobyGamesHttpClient _http;
|
||||
readonly SourceDatabaseService _sourceDb;
|
||||
|
||||
@@ -7,6 +7,7 @@ using Marechai.Data;
|
||||
using Marechai.Database.Models;
|
||||
using Marechai.MobyGames.Models;
|
||||
using Marechai.MobyGames.Parsers;
|
||||
using NewSite = Marechai.MobyGames.Parsers.NewSite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace Marechai.MobyGames.Services;
|
||||
@@ -70,6 +71,7 @@ public class PromoArtDownloadService
|
||||
int skippedCount = 0;
|
||||
int failedCount = 0;
|
||||
int noPromoPage = 0;
|
||||
int parseFailed = 0;
|
||||
int gamesProcessed = 0;
|
||||
bool aborted = false;
|
||||
|
||||
@@ -78,20 +80,8 @@ public class PromoArtDownloadService
|
||||
if(aborted) break;
|
||||
gamesProcessed++;
|
||||
|
||||
var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId);
|
||||
|
||||
// Find the promo art page chunk (new MobyGames HTML stored by scraper)
|
||||
string promoHtml = null;
|
||||
|
||||
foreach(var row in rows)
|
||||
{
|
||||
// New site promo pages contain /promo/group- pattern
|
||||
if(row.Body.Contains("/promo/group-"))
|
||||
{
|
||||
promoHtml = row.Body;
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Fetch the promo art page directly from the fixed chunk slot
|
||||
string promoHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkPromo);
|
||||
|
||||
if(promoHtml is null)
|
||||
{
|
||||
@@ -106,11 +96,18 @@ public class PromoArtDownloadService
|
||||
continue;
|
||||
}
|
||||
|
||||
var promoGroups = PromoArtPageParser.Parse(promoHtml);
|
||||
var promoGroups = NewSite.PromoArtTabParser.Parse(promoHtml);
|
||||
|
||||
if(promoGroups.Count == 0)
|
||||
{
|
||||
Console.WriteLine($" [{gamesProcessed}/{Math.Min(batchSize, importedGames.Count)}] {game.MobyGameId}: Promo page found but no groups parsed");
|
||||
parseFailed++;
|
||||
|
||||
if(dryRun || parseFailed <= 20)
|
||||
Console.WriteLine($" [{gamesProcessed}/{Math.Min(batchSize, importedGames.Count)}] {game.MobyGameId}: Promo page found but no groups parsed");
|
||||
|
||||
if(parseFailed == 20 && !dryRun)
|
||||
Console.WriteLine(" ... suppressing further 'no groups parsed' messages ...");
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -130,7 +127,7 @@ public class PromoArtDownloadService
|
||||
int groupId = 0;
|
||||
|
||||
if(!dryRun)
|
||||
groupId = await GetOrCreateGroupAsync(group.GroupName);
|
||||
groupId = await GetOrCreateGroupAsync(group.GroupName, group.GroupId);
|
||||
|
||||
foreach(var image in group.Images)
|
||||
{
|
||||
@@ -317,12 +314,14 @@ public class PromoArtDownloadService
|
||||
Console.WriteLine(" \e[33;1m[DRY RUN]\e[0m No changes made");
|
||||
Console.WriteLine($" Games scanned: {gamesProcessed}");
|
||||
Console.WriteLine($" No promo page: {noPromoPage}");
|
||||
Console.WriteLine($" Parse failed: {parseFailed}");
|
||||
Console.WriteLine($" Total images found: {totalImages}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($" Games processed: {gamesProcessed}");
|
||||
Console.WriteLine($" No promo page: {noPromoPage}");
|
||||
Console.WriteLine($" Parse failed: {parseFailed}");
|
||||
Console.WriteLine($" Total images found: {totalImages}");
|
||||
Console.WriteLine($" Downloaded: {downloadedCount}");
|
||||
Console.WriteLine($" Skipped (existing): {skippedCount}");
|
||||
@@ -332,10 +331,57 @@ public class PromoArtDownloadService
|
||||
Console.WriteLine(" ────────────────────────────────────\n");
|
||||
}
|
||||
|
||||
async Task<int> GetOrCreateGroupAsync(string groupName)
|
||||
async Task<int> GetOrCreateGroupAsync(string groupName, string groupId = null)
|
||||
{
|
||||
await using var context = await _contextFactory.CreateDbContextAsync();
|
||||
|
||||
// If we have a groupId, check if a previous run created a bad "Group {id}" entry
|
||||
// and rename it to the real name. Skip if the current name IS the bad pattern
|
||||
// (regex fallback couldn't extract the real name — nothing to repair).
|
||||
if(groupId is not null)
|
||||
{
|
||||
string badName = $"Group {groupId}";
|
||||
|
||||
if(groupName != badName)
|
||||
{
|
||||
var badEntry = await context.SoftwarePromoArtGroups
|
||||
.FirstOrDefaultAsync(g => g.Name == badName);
|
||||
|
||||
if(badEntry is not null)
|
||||
{
|
||||
// Check if the real name already exists too
|
||||
var realEntry = await context.SoftwarePromoArtGroups
|
||||
.FirstOrDefaultAsync(g => g.Name == groupName);
|
||||
|
||||
if(realEntry is null)
|
||||
{
|
||||
// Just rename the bad entry
|
||||
badEntry.Name = groupName;
|
||||
await context.SaveChangesAsync();
|
||||
Console.WriteLine($" \e[33mRepaired\e[0m group \"{badName}\" → \"{groupName}\"");
|
||||
|
||||
return badEntry.Id;
|
||||
}
|
||||
|
||||
// Both exist — migrate promo art from bad group to real group, then delete bad
|
||||
var orphaned = await context.SoftwarePromoArt
|
||||
.Where(p => p.GroupId == badEntry.Id)
|
||||
.ToListAsync();
|
||||
|
||||
foreach(var art in orphaned)
|
||||
art.GroupId = realEntry.Id;
|
||||
|
||||
context.SoftwarePromoArtGroups.Remove(badEntry);
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
if(orphaned.Count > 0)
|
||||
Console.WriteLine($" \e[33mRepaired\e[0m migrated {orphaned.Count} images from \"{badName}\" → \"{groupName}\"");
|
||||
|
||||
return realEntry.Id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var existing = await context.SoftwarePromoArtGroups
|
||||
.FirstOrDefaultAsync(g => g.Name == groupName);
|
||||
|
||||
|
||||
@@ -113,9 +113,22 @@ public class PromoArtScraper
|
||||
|
||||
Console.WriteLine($" [{gamesChecked}/{Math.Min(batchSize, importedGames.Count)}] \e[36;1m{game.MobyGameId}\e[0m: Has promo art");
|
||||
|
||||
// Check if we already scraped this game's promo page
|
||||
// New site pages contain /promo/group- pattern (old site doesn't)
|
||||
bool alreadyScraped = rows.Any(r => r.Body.Contains("/promo/group-"));
|
||||
// Check if we already scraped this game's promo page.
|
||||
// Must verify via TabDetector that the row IS actually a promo page,
|
||||
// not just the main page which also contains /promo/group- image links.
|
||||
bool alreadyScraped = false;
|
||||
|
||||
foreach(var row in rows)
|
||||
{
|
||||
var (tab, _) = TabDetector.DetectWithLayout(row.Body);
|
||||
|
||||
if(tab == MobyTab.PromoArt)
|
||||
{
|
||||
alreadyScraped = true;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(alreadyScraped)
|
||||
{
|
||||
@@ -185,10 +198,8 @@ public class PromoArtScraper
|
||||
continue;
|
||||
}
|
||||
|
||||
// Store in mobygames_raw as a new chunk
|
||||
int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0;
|
||||
|
||||
await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html);
|
||||
// Store in mobygames_raw at the fixed promo chunk slot
|
||||
await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkPromo, html);
|
||||
|
||||
Console.WriteLine(" \e[32mOK\e[0m");
|
||||
scraped++;
|
||||
|
||||
@@ -81,21 +81,8 @@ public class ScreenshotDownloadService
|
||||
|
||||
gamesProcessed++;
|
||||
|
||||
var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId);
|
||||
|
||||
// Find the screenshot page chunk (new MobyGames HTML)
|
||||
// The gtag content_type "game-list-screenshots" is always present on screenshot list pages
|
||||
string screenshotHtml = null;
|
||||
|
||||
foreach(var row in rows)
|
||||
{
|
||||
if(row.Body.Contains("game-list-screenshots"))
|
||||
{
|
||||
screenshotHtml = row.Body;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Fetch the screenshot page directly from the fixed chunk slot
|
||||
string screenshotHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkScreenshots);
|
||||
|
||||
if(screenshotHtml is null)
|
||||
{
|
||||
|
||||
@@ -197,9 +197,7 @@ public class ScreenshotScraper
|
||||
}
|
||||
|
||||
// Store in mobygames_raw as a new chunk
|
||||
int maxChunk = rows.Count > 0 ? rows.Max(r => r.Chunk) : 0;
|
||||
|
||||
await _sourceDb.InsertRowAsync(game.MobyGameId, maxChunk + 1, html);
|
||||
await _sourceDb.InsertRowAsync(game.MobyGameId, NewGameRawFetcher.ChunkScreenshots, html);
|
||||
|
||||
Console.WriteLine(" \e[32mOK\e[0m");
|
||||
scraped++;
|
||||
|
||||
@@ -110,6 +110,101 @@ public class SourceDatabaseService
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves a row from one chunk number to another. Used by the chunk migrator
|
||||
/// to reassign dynamically-allocated chunks to their fixed slot numbers.
|
||||
/// Returns the number of affected rows (0 if the source chunk didn't exist,
|
||||
/// or the target chunk already exists via <c>INSERT IGNORE</c>).
|
||||
/// </summary>
|
||||
public async Task<int> MoveChunkAsync(string gameId, int fromChunk, int toChunk)
|
||||
{
|
||||
await using var connection = new MySqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
// Copy to new chunk (IGNORE if target already exists), then delete old
|
||||
await using var insert = new MySqlCommand(
|
||||
"INSERT IGNORE INTO mobygames_raw (id, chunk, body) SELECT id, @toChunk, body FROM mobygames_raw WHERE id = @id AND chunk = @fromChunk",
|
||||
connection);
|
||||
|
||||
insert.Parameters.AddWithValue("@id", gameId);
|
||||
insert.Parameters.AddWithValue("@fromChunk", fromChunk);
|
||||
insert.Parameters.AddWithValue("@toChunk", toChunk);
|
||||
|
||||
int inserted = await insert.ExecuteNonQueryAsync();
|
||||
|
||||
await using var delete = new MySqlCommand(
|
||||
"DELETE FROM mobygames_raw WHERE id = @id AND chunk = @fromChunk", connection);
|
||||
|
||||
delete.Parameters.AddWithValue("@id", gameId);
|
||||
delete.Parameters.AddWithValue("@fromChunk", fromChunk);
|
||||
|
||||
await delete.ExecuteNonQueryAsync();
|
||||
|
||||
return inserted;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns all (id, chunk) pairs whose chunk number is outside the known fixed slots
|
||||
/// (0-5 and 10-12). These are candidates for chunk migration.
|
||||
/// </summary>
|
||||
public async Task<List<(string Id, int Chunk)>> GetMisplacedChunksAsync()
|
||||
{
|
||||
var results = new List<(string, int)>();
|
||||
|
||||
await using var connection = new MySqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = new MySqlCommand(
|
||||
"SELECT id, chunk FROM mobygames_raw WHERE chunk NOT IN (0,1,2,3,4,5,10,11,12) ORDER BY id, chunk",
|
||||
connection);
|
||||
|
||||
await using var reader = await cmd.ExecuteReaderAsync();
|
||||
|
||||
while(await reader.ReadAsync())
|
||||
results.Add((reader.GetString(0), reader.GetInt32(1)));
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the body of a single (id, chunk) row. Used by the chunk migrator
|
||||
/// to detect the page type without loading all chunks for a game.
|
||||
/// </summary>
|
||||
public async Task<string> GetChunkBodyAsync(string gameId, int chunk)
|
||||
{
|
||||
await using var connection = new MySqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = new MySqlCommand(
|
||||
"SELECT body FROM mobygames_raw WHERE id = @id AND chunk = @chunk LIMIT 1", connection);
|
||||
|
||||
cmd.Parameters.AddWithValue("@id", gameId);
|
||||
cmd.Parameters.AddWithValue("@chunk", chunk);
|
||||
|
||||
var result = await cmd.ExecuteScalarAsync();
|
||||
|
||||
return result as string;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns true if a row exists for the given (id, chunk) pair.
|
||||
/// </summary>
|
||||
public async Task<bool> ChunkExistsAsync(string gameId, int chunk)
|
||||
{
|
||||
await using var connection = new MySqlConnection(_connectionString);
|
||||
await connection.OpenAsync();
|
||||
|
||||
await using var cmd = new MySqlCommand(
|
||||
"SELECT EXISTS (SELECT 1 FROM mobygames_raw WHERE id = @id AND chunk = @chunk LIMIT 1)", connection);
|
||||
|
||||
cmd.Parameters.AddWithValue("@id", gameId);
|
||||
cmd.Parameters.AddWithValue("@chunk", chunk);
|
||||
|
||||
var result = await cmd.ExecuteScalarAsync();
|
||||
|
||||
return System.Convert.ToInt32(result) == 1;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Delete every cached chunk for the given slug. Used to evict stale legacy-layout
|
||||
/// captures before re-scraping the current new-layout page (see
|
||||
|
||||
@@ -61,21 +61,8 @@ public class VideoImportService
|
||||
{
|
||||
gamesProcessed++;
|
||||
|
||||
var rows = await _sourceDb.GetRowsForGameAsync(game.MobyGameId);
|
||||
|
||||
// Find the media page chunk (new MobyGames HTML)
|
||||
// The gtag content_type "game-list-media" is present on media pages
|
||||
string mediaHtml = null;
|
||||
|
||||
foreach(var row in rows)
|
||||
{
|
||||
if(row.Body.Contains("game-list-media"))
|
||||
{
|
||||
mediaHtml = row.Body;
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
// Fetch the media page directly from the fixed chunk slot
|
||||
string mediaHtml = await _sourceDb.GetChunkBodyAsync(game.MobyGameId, NewGameRawFetcher.ChunkMedia);
|
||||
|
||||
if(mediaHtml is null)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user