claunia/marechai
1
0
Fork 0
mirror of https://github.com/claunia/marechai.git synced 2026-07-08 17:57:08 +00:00
Code Issues 15 Packages Projects Releases Wiki Activity

Enhance logging in CategorizeEnricher and DescribeEnricher for better processing visibility

Browse Source
This commit is contained in:
Natalia Portillo
2026-06-03 03:37:13 +01:00
parent f9b937f191
commit ccf46b49c9
2 changed files with 33 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
Marechai.WinWorld/Services/CategorizeEnricher.cs
View File

@@ -70,6 +70,7 @@ public sealed class CategorizeEnricher
.Take(limit) .Take(limit)
.ToListAsync(); .ToListAsync();
Console.WriteLine($"\e[36mCategorize: {rows.Count} Described row(s) to process (limit {limit}).\e[0m");
if(rows.Count == 0) return 0; if(rows.Count == 0) return 0;
var categories = await db.SoftwareGenres var categories = await db.SoftwareGenres
@@ -77,14 +78,17 @@ public sealed class CategorizeEnricher
.OrderBy(g => g.Name) .OrderBy(g => g.Name)
.Select(g => new { g.Id, g.Name }) .Select(g => new { g.Id, g.Name })
.ToListAsync(); .ToListAsync();
Console.WriteLine($"\e[36m loaded {categories.Count} category allow-list entries\e[0m");
string allow = string.Join("\n", categories.Select(c => $" {c.Id} = {c.Name}")); string allow = string.Join("\n", categories.Select(c => $" {c.Id} = {c.Name}"));
int done = 0; int done = 0;
foreach(WwpcSoftware row in rows) for(int idx = 0; idx < rows.Count; idx++)
{ {
WwpcSoftware row = rows[idx];
try try
{ {
Console.WriteLine($"\e[36m[{idx + 1}/{rows.Count}] #{row.Id} {row.Name}\e[0m");
string userPrompt = $""" string userPrompt = $"""
Allow-list (id = name): Allow-list (id = name):
{allow} {allow}
@@ -97,6 +101,8 @@ public sealed class CategorizeEnricher
{row.EnglishDescriptionMuseum} {row.EnglishDescriptionMuseum}
"""; """;
Console.Write(" openai… ");
System.Diagnostics.Stopwatch sw = System.Diagnostics.Stopwatch.StartNew();
string raw = await _openAi.CompleteAsync(SystemPrompt, userPrompt, ResponseSchema); string raw = await _openAi.CompleteAsync(SystemPrompt, userPrompt, ResponseSchema);
List<int> picked = ExtractIds(raw) List<int> picked = ExtractIds(raw)
.Where(id => categories.Any(c => c.Id == id)) .Where(id => categories.Any(c => c.Id == id))
@@ -109,13 +115,13 @@ public sealed class CategorizeEnricher
row.LastError = null; row.LastError = null;
await db.SaveChangesAsync(); await db.SaveChangesAsync();
done++; done++;
Console.WriteLine($" Categorized #{row.Id}: {row.Name} → [{string.Join(", ", picked)}]"); Console.WriteLine($"\e[32m{sw.ElapsedMilliseconds} ms → [{string.Join(", ", picked)}] (ReadyForReview)\e[0m");
} }
catch(Exception ex) catch(Exception ex)
{ {
row.LastError = "categorize: " + ex.Message; row.LastError = "categorize: " + ex.Message;
await db.SaveChangesAsync(); await db.SaveChangesAsync();
Console.WriteLine($"\e[31m Categorize failed #{row.Id}: {ex.Message}\e[0m"); Console.WriteLine($"\e[31m FAILED: {ex.Message}\e[0m");
} }
} }
return done; return done;

29
Marechai.WinWorld/Services/DescribeEnricher.cs
View File

@@ -61,13 +61,30 @@ public sealed class DescribeEnricher
.Take(limit) .Take(limit)
.ToListAsync(); .ToListAsync();
Console.WriteLine($"\e[36mDescribe: {rows.Count} Crawled row(s) to process (limit {limit}).\e[0m");
int done = 0; int done = 0;
foreach(WwpcSoftware row in rows) for(int idx = 0; idx < rows.Count; idx++)
{ {
WwpcSoftware row = rows[idx];
try try
{ {
string wikipedia = _wikipediaEnabled ? await FetchWikipediaSummaryAsync(row.Name) : null; Console.WriteLine($"\e[36m[{idx + 1}/{rows.Count}] #{row.Id} {row.Name}\e[0m");
string search = _searchEnabled ? await FetchSearchSnippetAsync(row.Name) : null;
string wikipedia = null;
if(_wikipediaEnabled)
{
Console.Write(" wikipedia… ");
wikipedia = await FetchWikipediaSummaryAsync(row.Name);
Console.WriteLine(wikipedia is null ? "(none)" : $"{wikipedia.Length} chars");
}
string search = null;
if(_searchEnabled)
{
Console.Write(" search… ");
search = await FetchSearchSnippetAsync(row.Name);
Console.WriteLine(search is null ? "(none)" : $"{search.Length} chars");
}
string userPrompt = $""" string userPrompt = $"""
Software name: {row.Name} Software name: {row.Name}
@@ -87,19 +104,21 @@ public sealed class DescribeEnricher
{search} {search}
"""; """;
Console.Write(" openai… ");
System.Diagnostics.Stopwatch sw = System.Diagnostics.Stopwatch.StartNew();
row.EnglishDescriptionMuseum = (await _openAi.CompleteAsync(SystemPrompt, userPrompt))?.Trim(); row.EnglishDescriptionMuseum = (await _openAi.CompleteAsync(SystemPrompt, userPrompt))?.Trim();
row.MuseumDescriptionPromptVersion = PromptVersion; row.MuseumDescriptionPromptVersion = PromptVersion;
row.Status = WwpcSoftwareStatus.Described; row.Status = WwpcSoftwareStatus.Described;
row.LastError = null; row.LastError = null;
await db.SaveChangesAsync(); await db.SaveChangesAsync();
done++; done++;
Console.WriteLine($" Described #{row.Id}: {row.Name}"); Console.WriteLine($"\e[32m{sw.ElapsedMilliseconds} ms, {row.EnglishDescriptionMuseum?.Length ?? 0} chars → Described\e[0m");
} }
catch(Exception ex) catch(Exception ex)
{ {
row.LastError = "describe: " + ex.Message; row.LastError = "describe: " + ex.Message;
await db.SaveChangesAsync(); await db.SaveChangesAsync();
Console.WriteLine($"\e[31m Describe failed #{row.Id}: {ex.Message}\e[0m"); Console.WriteLine($"\e[31m FAILED: {ex.Message}\e[0m");
} }
} }
return done; return done;