mirror of
https://github.com/claunia/marechai.git
synced 2026-07-08 17:57:08 +00:00
feat: integrate IndexNow service for URL submissions across various services
This commit is contained in:
@@ -144,6 +144,7 @@ public sealed class OldDosPromotionService
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
ulong targetSoftwareId;
|
ulong targetSoftwareId;
|
||||||
|
string softwareName;
|
||||||
int insertedDescriptions = 0;
|
int insertedDescriptions = 0;
|
||||||
int insertedGenres = 0;
|
int insertedGenres = 0;
|
||||||
int insertedVersions = 0;
|
int insertedVersions = 0;
|
||||||
@@ -159,6 +160,7 @@ public sealed class OldDosPromotionService
|
|||||||
return new AcceptOldDosImportResultDto { Success = false, Error = "Target software not found." };
|
return new AcceptOldDosImportResultDto { Success = false, Error = "Target software not found." };
|
||||||
|
|
||||||
targetSoftwareId = target.Id;
|
targetSoftwareId = target.Id;
|
||||||
|
softwareName = target.Name;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -170,6 +172,7 @@ public sealed class OldDosPromotionService
|
|||||||
_db.Softwares.Add(fresh);
|
_db.Softwares.Add(fresh);
|
||||||
await _db.SaveChangesAsync();
|
await _db.SaveChangesAsync();
|
||||||
targetSoftwareId = fresh.Id;
|
targetSoftwareId = fresh.Id;
|
||||||
|
softwareName = fresh.Name;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Per-version attach (Version + Release + DEV company role).
|
// Per-version attach (Version + Release + DEV company role).
|
||||||
@@ -275,6 +278,17 @@ public sealed class OldDosPromotionService
|
|||||||
staging.PromotedSoftwareId = targetSoftwareId;
|
staging.PromotedSoftwareId = targetSoftwareId;
|
||||||
staging.ReviewedBy = adminUserId;
|
staging.ReviewedBy = adminUserId;
|
||||||
staging.ReviewedOn = DateTime.UtcNow;
|
staging.ReviewedOn = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_db.News.Add(new News
|
||||||
|
{
|
||||||
|
AddedId = (long)targetSoftwareId,
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Type = dto.Mode == OldDosAcceptMode.MergeIntoExisting
|
||||||
|
? NewsType.UpdatedSoftwareInDb
|
||||||
|
: NewsType.NewSoftwareInDb,
|
||||||
|
Name = softwareName
|
||||||
|
});
|
||||||
|
|
||||||
await _db.SaveChangesAsync();
|
await _db.SaveChangesAsync();
|
||||||
await tx.CommitAsync();
|
await tx.CommitAsync();
|
||||||
|
|
||||||
|
|||||||
@@ -223,6 +223,7 @@ public sealed class WwpcPromotionService
|
|||||||
int insertedGenres = 0;
|
int insertedGenres = 0;
|
||||||
|
|
||||||
ulong targetSoftwareId;
|
ulong targetSoftwareId;
|
||||||
|
string softwareName;
|
||||||
if(dto.Mode == WwpcAcceptMode.MergeIntoExisting)
|
if(dto.Mode == WwpcAcceptMode.MergeIntoExisting)
|
||||||
{
|
{
|
||||||
if(dto.TargetSoftwareId is not { } tid)
|
if(dto.TargetSoftwareId is not { } tid)
|
||||||
@@ -230,6 +231,7 @@ public sealed class WwpcPromotionService
|
|||||||
Software target = await _db.Softwares.FindAsync(tid);
|
Software target = await _db.Softwares.FindAsync(tid);
|
||||||
if(target == null) return Fail("Target software not found.");
|
if(target == null) return Fail("Target software not found.");
|
||||||
targetSoftwareId = target.Id;
|
targetSoftwareId = target.Id;
|
||||||
|
softwareName = target.Name;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@@ -241,6 +243,7 @@ public sealed class WwpcPromotionService
|
|||||||
_db.Softwares.Add(fresh);
|
_db.Softwares.Add(fresh);
|
||||||
await _db.SaveChangesAsync();
|
await _db.SaveChangesAsync();
|
||||||
targetSoftwareId = fresh.Id;
|
targetSoftwareId = fresh.Id;
|
||||||
|
softwareName = fresh.Name;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---- Company → Software link ----
|
// ---- Company → Software link ----
|
||||||
@@ -420,6 +423,17 @@ public sealed class WwpcPromotionService
|
|||||||
staging.PromotedSoftwareId = targetSoftwareId;
|
staging.PromotedSoftwareId = targetSoftwareId;
|
||||||
staging.ReviewedBy = adminUserId;
|
staging.ReviewedBy = adminUserId;
|
||||||
staging.ReviewedOn = DateTime.UtcNow;
|
staging.ReviewedOn = DateTime.UtcNow;
|
||||||
|
|
||||||
|
_db.News.Add(new News
|
||||||
|
{
|
||||||
|
AddedId = (long)targetSoftwareId,
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Type = dto.Mode == WwpcAcceptMode.MergeIntoExisting
|
||||||
|
? NewsType.UpdatedSoftwareInDb
|
||||||
|
: NewsType.NewSoftwareInDb,
|
||||||
|
Name = softwareName
|
||||||
|
});
|
||||||
|
|
||||||
await _db.SaveChangesAsync();
|
await _db.SaveChangesAsync();
|
||||||
await tx.CommitAsync();
|
await tx.CommitAsync();
|
||||||
|
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class BooksService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData)
|
public class BooksService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<int> GetBooksCountAsync(IReadOnlyList<string> filters = null,
|
public async Task<int> GetBooksCountAsync(IReadOnlyList<string> filters = null,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
@@ -352,6 +352,8 @@ public class BooksService(Marechai.ApiClient.Client client, ReferenceDataCache r
|
|||||||
{
|
{
|
||||||
long? id = await client.Books.PostAsync(dto);
|
long? id = await client.Books.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/book/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -370,6 +372,8 @@ public class BooksService(Marechai.ApiClient.Client client, ReferenceDataCache r
|
|||||||
{
|
{
|
||||||
await client.Books[id].PutAsync(dto);
|
await client.Books[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/book/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class CompaniesService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData)
|
public class CompaniesService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<List<CompanyDto>> GetAsync()
|
public async Task<List<CompanyDto>> GetAsync()
|
||||||
{
|
{
|
||||||
@@ -104,6 +104,8 @@ public class CompaniesService(Marechai.ApiClient.Client client, ReferenceDataCac
|
|||||||
{
|
{
|
||||||
int? id = await client.Companies.PostAsync(dto);
|
int? id = await client.Companies.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/company/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -122,6 +124,8 @@ public class CompaniesService(Marechai.ApiClient.Client client, ReferenceDataCac
|
|||||||
{
|
{
|
||||||
await client.Companies[id].PutAsync(dto);
|
await client.Companies[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/company/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class DocumentsService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData)
|
public class DocumentsService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<int> GetDocumentsCountAsync(IReadOnlyList<string> filters = null,
|
public async Task<int> GetDocumentsCountAsync(IReadOnlyList<string> filters = null,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
@@ -333,6 +333,8 @@ public class DocumentsService(Marechai.ApiClient.Client client, ReferenceDataCac
|
|||||||
{
|
{
|
||||||
long? id = await client.Documents.PostAsync(dto);
|
long? id = await client.Documents.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/document/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -351,6 +353,8 @@ public class DocumentsService(Marechai.ApiClient.Client client, ReferenceDataCac
|
|||||||
{
|
{
|
||||||
await client.Documents[id].PutAsync(dto);
|
await client.Documents[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/document/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class GpusService(Marechai.ApiClient.Client client)
|
public class GpusService(Marechai.ApiClient.Client client, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<List<GpuDto>> GetAllAsync(int? skip = null, int? take = null,
|
public async Task<List<GpuDto>> GetAllAsync(int? skip = null, int? take = null,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
@@ -142,6 +142,8 @@ public class GpusService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
long? id = await client.Gpus.PostAsync(dto);
|
long? id = await client.Gpus.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/gpu/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -160,6 +162,8 @@ public class GpusService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
await client.Gpus[id].PutAsync(dto);
|
await client.Gpus[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/gpu/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
166
Marechai/Services/IndexNowBackgroundService.cs
Normal file
166
Marechai/Services/IndexNowBackgroundService.cs
Normal file
@@ -0,0 +1,166 @@
|
|||||||
|
/******************************************************************************
|
||||||
|
// MARECHAI: Master repository of computing history artifacts information
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// Author(s) : Natalia Portillo <claunia@claunia.com>
|
||||||
|
//
|
||||||
|
// --[ License ] --------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as
|
||||||
|
// published by the Free Software Foundation, either version 3 of the
|
||||||
|
// License, or (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
//
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Copyright © 2003-2026 Natalia Portillo
|
||||||
|
*******************************************************************************/
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Net.Http;
|
||||||
|
using System.Net.Http.Json;
|
||||||
|
using System.Text.Json;
|
||||||
|
using System.Text.Json.Serialization;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using Marechai.Helpers;
|
||||||
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace Marechai.Services;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Background worker that periodically drains the <see cref="IndexNowService" /> queue
|
||||||
|
/// and POSTs batches of changed URLs to every IndexNow-participating search engine
|
||||||
|
/// endpoint in parallel.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class IndexNowBackgroundService(
|
||||||
|
IndexNowService indexNowService,
|
||||||
|
IHttpClientFactory httpClientFactory,
|
||||||
|
IOptions<IndexNowOptions> options,
|
||||||
|
ILogger<IndexNowBackgroundService> logger) : BackgroundService
|
||||||
|
{
|
||||||
|
// How often we drain the queue and submit.
|
||||||
|
static readonly TimeSpan Interval = TimeSpan.FromSeconds(30);
|
||||||
|
|
||||||
|
// IndexNow caps a single POST at 10 000 URLs.
|
||||||
|
const int MaxBatchSize = 10_000;
|
||||||
|
|
||||||
|
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||||
|
{
|
||||||
|
IndexNowOptions opts = options.Value;
|
||||||
|
|
||||||
|
if(!opts.Enabled || string.IsNullOrWhiteSpace(opts.Key))
|
||||||
|
{
|
||||||
|
logger.LogInformation("IndexNow is disabled or no key configured — background worker will not run");
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.LogInformation("IndexNow background worker started, submitting to {Count} endpoints every {Seconds}s",
|
||||||
|
opts.Endpoints.Length, Interval.TotalSeconds);
|
||||||
|
|
||||||
|
while(!stoppingToken.IsCancellationRequested)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await Task.Delay(Interval, stoppingToken);
|
||||||
|
}
|
||||||
|
catch(TaskCanceledException)
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
List<string> urls = indexNowService.DrainQueue();
|
||||||
|
|
||||||
|
if(urls.Count == 0)
|
||||||
|
continue;
|
||||||
|
|
||||||
|
// Split into IndexNow-legal batches of 10 000.
|
||||||
|
for(int i = 0; i < urls.Count; i += MaxBatchSize)
|
||||||
|
{
|
||||||
|
List<string> batch = urls.GetRange(i, Math.Min(MaxBatchSize, urls.Count - i));
|
||||||
|
|
||||||
|
await SubmitBatchAsync(batch, opts, stoppingToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task SubmitBatchAsync(List<string> urls, IndexNowOptions opts, CancellationToken ct)
|
||||||
|
{
|
||||||
|
// Parse the canonical host to extract just the hostname for the payload.
|
||||||
|
string host = new Uri(SeoMeta.CanonicalHost).Host;
|
||||||
|
|
||||||
|
string keyLocation = $"{SeoMeta.CanonicalHost}/{opts.Key}.txt";
|
||||||
|
|
||||||
|
var payload = new IndexNowPayload
|
||||||
|
{
|
||||||
|
Host = host,
|
||||||
|
Key = opts.Key,
|
||||||
|
KeyLocation = keyLocation,
|
||||||
|
UrlList = urls
|
||||||
|
};
|
||||||
|
|
||||||
|
logger.LogInformation("IndexNow: submitting {Count} URL(s) to {Endpoints} endpoints",
|
||||||
|
urls.Count, opts.Endpoints.Length);
|
||||||
|
|
||||||
|
// Fire requests to all endpoints in parallel.
|
||||||
|
Task[] tasks = opts.Endpoints.Select(endpoint => SubmitToEndpointAsync(endpoint, payload, ct)).ToArray();
|
||||||
|
|
||||||
|
await Task.WhenAll(tasks);
|
||||||
|
}
|
||||||
|
|
||||||
|
async Task SubmitToEndpointAsync(string endpoint, IndexNowPayload payload, CancellationToken ct)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
HttpClient client = httpClientFactory.CreateClient("IndexNow");
|
||||||
|
|
||||||
|
string json = JsonSerializer.Serialize(payload);
|
||||||
|
using var content = new StringContent(json, System.Text.Encoding.UTF8, "application/json");
|
||||||
|
|
||||||
|
HttpResponseMessage response = await client.PostAsync(endpoint, content, ct);
|
||||||
|
|
||||||
|
if(response.IsSuccessStatusCode)
|
||||||
|
{
|
||||||
|
logger.LogInformation("IndexNow: {Endpoint} accepted {Count} URL(s) (HTTP {Status})",
|
||||||
|
endpoint, payload.UrlList.Count, (int)response.StatusCode);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
logger.LogWarning("IndexNow: {Endpoint} returned HTTP {Status} for {Count} URL(s)",
|
||||||
|
endpoint, (int)response.StatusCode, payload.UrlList.Count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
catch(Exception ex)
|
||||||
|
{
|
||||||
|
logger.LogWarning(ex, "IndexNow: failed to submit to {Endpoint}", endpoint);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>JSON payload shape for the IndexNow POST API.</summary>
|
||||||
|
sealed class IndexNowPayload
|
||||||
|
{
|
||||||
|
[JsonPropertyName("host")]
|
||||||
|
public string Host { get; init; }
|
||||||
|
|
||||||
|
[JsonPropertyName("key")]
|
||||||
|
public string Key { get; init; }
|
||||||
|
|
||||||
|
[JsonPropertyName("keyLocation")]
|
||||||
|
public string KeyLocation { get; init; }
|
||||||
|
|
||||||
|
[JsonPropertyName("urlList")]
|
||||||
|
public List<string> UrlList { get; init; }
|
||||||
|
}
|
||||||
|
}
|
||||||
42
Marechai/Services/IndexNowOptions.cs
Normal file
42
Marechai/Services/IndexNowOptions.cs
Normal file
@@ -0,0 +1,42 @@
|
|||||||
|
/******************************************************************************
|
||||||
|
// MARECHAI: Master repository of computing history artifacts information
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// Author(s) : Natalia Portillo <claunia@claunia.com>
|
||||||
|
//
|
||||||
|
// --[ License ] --------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as
|
||||||
|
// published by the Free Software Foundation, either version 3 of the
|
||||||
|
// License, or (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
//
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Copyright © 2003-2026 Natalia Portillo
|
||||||
|
*******************************************************************************/
|
||||||
|
|
||||||
|
namespace Marechai.Services;
|
||||||
|
|
||||||
|
public sealed class IndexNowOptions
|
||||||
|
{
|
||||||
|
public bool Enabled { get; set; }
|
||||||
|
public string Key { get; set; }
|
||||||
|
public string[] Endpoints { get; set; } =
|
||||||
|
[
|
||||||
|
"https://www.bing.com/indexnow",
|
||||||
|
"https://yandex.com/indexnow",
|
||||||
|
"https://search.seznam.cz/indexnow",
|
||||||
|
"https://searchadvisor.naver.com/indexnow",
|
||||||
|
"https://indexnow.yep.com/indexnow",
|
||||||
|
"https://internetarchive.indexnow.org/indexnow",
|
||||||
|
"https://indexnow.amazonbot.amazon/indexnow"
|
||||||
|
];
|
||||||
|
}
|
||||||
69
Marechai/Services/IndexNowService.cs
Normal file
69
Marechai/Services/IndexNowService.cs
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
/******************************************************************************
|
||||||
|
// MARECHAI: Master repository of computing history artifacts information
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// Author(s) : Natalia Portillo <claunia@claunia.com>
|
||||||
|
//
|
||||||
|
// --[ License ] --------------------------------------------------------------
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU General Public License as
|
||||||
|
// published by the Free Software Foundation, either version 3 of the
|
||||||
|
// License, or (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
//
|
||||||
|
// ----------------------------------------------------------------------------
|
||||||
|
// Copyright © 2003-2026 Natalia Portillo
|
||||||
|
*******************************************************************************/
|
||||||
|
|
||||||
|
using System;
|
||||||
|
using System.Collections.Concurrent;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using Marechai.Helpers;
|
||||||
|
using Microsoft.Extensions.Logging;
|
||||||
|
using Microsoft.Extensions.Options;
|
||||||
|
|
||||||
|
namespace Marechai.Services;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Thread-safe in-memory queue for IndexNow URL submissions. Singleton service that
|
||||||
|
/// accepts relative URLs from admin CRUD operations and exposes them for the background
|
||||||
|
/// worker to drain and POST to each participating search engine.
|
||||||
|
/// </summary>
|
||||||
|
public sealed class IndexNowService(IOptions<IndexNowOptions> options, ILogger<IndexNowService> logger)
|
||||||
|
{
|
||||||
|
readonly ConcurrentQueue<string> _queue = new();
|
||||||
|
|
||||||
|
/// <summary>Enqueue a relative URL (e.g. <c>/machine/42</c>) for submission to IndexNow.</summary>
|
||||||
|
public void EnqueueUrl(string relativeUrl)
|
||||||
|
{
|
||||||
|
IndexNowOptions opts = options.Value;
|
||||||
|
|
||||||
|
if(!opts.Enabled || string.IsNullOrWhiteSpace(opts.Key))
|
||||||
|
return;
|
||||||
|
|
||||||
|
string absoluteUrl = $"{SeoMeta.CanonicalHost}{relativeUrl}";
|
||||||
|
|
||||||
|
_queue.Enqueue(absoluteUrl);
|
||||||
|
|
||||||
|
logger.LogDebug("IndexNow: enqueued {Url}", absoluteUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Drain all queued URLs into a deduplicated list. Called by the background worker.</summary>
|
||||||
|
internal List<string> DrainQueue()
|
||||||
|
{
|
||||||
|
var urls = new HashSet<string>(StringComparer.Ordinal);
|
||||||
|
|
||||||
|
while(_queue.TryDequeue(out string url))
|
||||||
|
urls.Add(url);
|
||||||
|
|
||||||
|
return [.. urls];
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -33,7 +33,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class MachinesService(Marechai.ApiClient.Client client)
|
public class MachinesService(Marechai.ApiClient.Client client, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
static string ExtractErrorMessage(ApiException ex)
|
static string ExtractErrorMessage(ApiException ex)
|
||||||
{
|
{
|
||||||
@@ -134,6 +134,8 @@ public class MachinesService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
long? id = await client.Machines.PostAsync(dto);
|
long? id = await client.Machines.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/machine/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -152,6 +154,8 @@ public class MachinesService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
await client.Machines[id].PutAsync(dto);
|
await client.Machines[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/machine/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class MagazinesService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData)
|
public class MagazinesService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<int> GetMagazinesCountAsync(IReadOnlyList<string> filters = null,
|
public async Task<int> GetMagazinesCountAsync(IReadOnlyList<string> filters = null,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
@@ -350,6 +350,8 @@ public class MagazinesService(Marechai.ApiClient.Client client, ReferenceDataCac
|
|||||||
{
|
{
|
||||||
long? id = await client.Magazines.PostAsync(dto);
|
long? id = await client.Magazines.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/magazine/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -368,6 +370,8 @@ public class MagazinesService(Marechai.ApiClient.Client client, ReferenceDataCac
|
|||||||
{
|
{
|
||||||
await client.Magazines[id].PutAsync(dto);
|
await client.Magazines[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/magazine/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
/// <summary>Client wrapper around the <c>/old-dos</c> admin endpoints.</summary>
|
/// <summary>Client wrapper around the <c>/old-dos</c> admin endpoints.</summary>
|
||||||
public sealed class OldDosImportsService(Client client, ILogger<OldDosImportsService> logger)
|
public sealed class OldDosImportsService(Client client, ILogger<OldDosImportsService> logger, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<List<OldDosPendingListItemDto>> GetPendingAsync(int skip, int take,
|
public async Task<List<OldDosPendingListItemDto>> GetPendingAsync(int skip, int take,
|
||||||
OldDosSoftwareStatus? status,
|
OldDosSoftwareStatus? status,
|
||||||
@@ -114,6 +114,10 @@ public sealed class OldDosImportsService(Client client, ILogger<OldDosImportsSer
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
AcceptOldDosImportResultDto result = await client.OldDos.Pending[(int)id].Accept.PostAsync(dto);
|
AcceptOldDosImportResultDto result = await client.OldDos.Pending[(int)id].Accept.PostAsync(dto);
|
||||||
|
|
||||||
|
if(result?.Success == true && result.PromotedSoftwareId.HasValue)
|
||||||
|
indexNow.EnqueueUrl($"/software/{result.PromotedSoftwareId.Value}");
|
||||||
|
|
||||||
return (result, result?.Success == false ? result.Error : null);
|
return (result, result?.Success == false ? result.Error : null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class PeopleService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData)
|
public class PeopleService(Marechai.ApiClient.Client client, ReferenceDataCache referenceData, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
static string ExtractDetail(ApiException ex)
|
static string ExtractDetail(ApiException ex)
|
||||||
{
|
{
|
||||||
@@ -295,6 +295,8 @@ public class PeopleService(Marechai.ApiClient.Client client, ReferenceDataCache
|
|||||||
{
|
{
|
||||||
long? id = await client.People.PostAsync(dto);
|
long? id = await client.People.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/person/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -313,6 +315,8 @@ public class PeopleService(Marechai.ApiClient.Client client, ReferenceDataCache
|
|||||||
{
|
{
|
||||||
await client.People[id].PutAsync(dto);
|
await client.People[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/person/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class ProcessorsService(Marechai.ApiClient.Client client)
|
public class ProcessorsService(Marechai.ApiClient.Client client, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<List<ProcessorDto>> GetAllAsync(int? skip = null, int? take = null,
|
public async Task<List<ProcessorDto>> GetAllAsync(int? skip = null, int? take = null,
|
||||||
CancellationToken cancellationToken = default)
|
CancellationToken cancellationToken = default)
|
||||||
@@ -142,6 +142,8 @@ public class ProcessorsService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
long? id = await client.Processors.PostAsync(dto);
|
long? id = await client.Processors.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/processor/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -160,6 +162,8 @@ public class ProcessorsService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
await client.Processors[id].PutAsync(dto);
|
await client.Processors[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/processor/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -91,5 +91,8 @@ public static class Register
|
|||||||
services.AddScoped<ThemeFontLoader>();
|
services.AddScoped<ThemeFontLoader>();
|
||||||
|
|
||||||
services.AddSingleton<SitemapService>();
|
services.AddSingleton<SitemapService>();
|
||||||
|
|
||||||
|
services.AddSingleton<IndexNowService>();
|
||||||
|
services.AddHostedService<IndexNowBackgroundService>();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -51,7 +51,7 @@ public enum AddonPrefixMode
|
|||||||
StartsWith
|
StartsWith
|
||||||
}
|
}
|
||||||
|
|
||||||
public class SoftwareService(Marechai.ApiClient.Client client, IRequestAdapter requestAdapter, ReferenceDataCache referenceData)
|
public class SoftwareService(Marechai.ApiClient.Client client, IRequestAdapter requestAdapter, ReferenceDataCache referenceData, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
static string ExtractErrorMessage(ApiException ex)
|
static string ExtractErrorMessage(ApiException ex)
|
||||||
{
|
{
|
||||||
@@ -69,6 +69,8 @@ public class SoftwareService(Marechai.ApiClient.Client client, IRequestAdapter r
|
|||||||
{
|
{
|
||||||
int? id = await client.Software.PostAsync(dto);
|
int? id = await client.Software.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/software/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -87,6 +89,8 @@ public class SoftwareService(Marechai.ApiClient.Client client, IRequestAdapter r
|
|||||||
{
|
{
|
||||||
await client.Software[id].PutAsync(dto);
|
await client.Software[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/software/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
|
|
||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
public class SoundSynthsService(Marechai.ApiClient.Client client)
|
public class SoundSynthsService(Marechai.ApiClient.Client client, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
static string ExtractErrorMessage(ApiException ex)
|
static string ExtractErrorMessage(ApiException ex)
|
||||||
{
|
{
|
||||||
@@ -151,6 +151,8 @@ public class SoundSynthsService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
long? id = await client.SoundSynths.PostAsync(dto);
|
long? id = await client.SoundSynths.PostAsync(dto);
|
||||||
|
|
||||||
|
if(id.HasValue) indexNow.EnqueueUrl($"/soundsynth/{id}");
|
||||||
|
|
||||||
return (id, null);
|
return (id, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
@@ -169,6 +171,8 @@ public class SoundSynthsService(Marechai.ApiClient.Client client)
|
|||||||
{
|
{
|
||||||
await client.SoundSynths[id].PutAsync(dto);
|
await client.SoundSynths[id].PutAsync(dto);
|
||||||
|
|
||||||
|
indexNow.EnqueueUrl($"/soundsynth/{id}");
|
||||||
|
|
||||||
return (true, null);
|
return (true, null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ using Microsoft.Kiota.Abstractions;
|
|||||||
namespace Marechai.Services;
|
namespace Marechai.Services;
|
||||||
|
|
||||||
/// <summary>Client wrapper around the <c>/wwpc</c> admin endpoints.</summary>
|
/// <summary>Client wrapper around the <c>/wwpc</c> admin endpoints.</summary>
|
||||||
public sealed class WwpcImportsService(Client client, ILogger<WwpcImportsService> logger)
|
public sealed class WwpcImportsService(Client client, ILogger<WwpcImportsService> logger, IndexNowService indexNow)
|
||||||
{
|
{
|
||||||
public async Task<List<WwpcPendingListItemDto>> GetPendingAsync(int skip, int take,
|
public async Task<List<WwpcPendingListItemDto>> GetPendingAsync(int skip, int take,
|
||||||
WwpcSoftwareStatus? status,
|
WwpcSoftwareStatus? status,
|
||||||
@@ -125,6 +125,10 @@ public sealed class WwpcImportsService(Client client, ILogger<WwpcImportsService
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
AcceptWwpcImportResultDto result = await client.Wwpc.Pending[(int)id].Accept.PostAsync(dto);
|
AcceptWwpcImportResultDto result = await client.Wwpc.Pending[(int)id].Accept.PostAsync(dto);
|
||||||
|
|
||||||
|
if(result?.Success == true && result.PromotedSoftwareId.HasValue)
|
||||||
|
indexNow.EnqueueUrl($"/software/{result.PromotedSoftwareId.Value}");
|
||||||
|
|
||||||
return (result, result?.Success == false ? result.Error : null);
|
return (result, result?.Success == false ? result.Error : null);
|
||||||
}
|
}
|
||||||
catch(ApiException ex)
|
catch(ApiException ex)
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ using System.IO.Compression;
|
|||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Net.Http;
|
using System.Net.Http;
|
||||||
using System.Net.Http.Headers;
|
using System.Net.Http.Headers;
|
||||||
|
using System.Threading.Tasks;
|
||||||
using Microsoft.AspNetCore.Http;
|
using Microsoft.AspNetCore.Http;
|
||||||
using Microsoft.AspNetCore.ResponseCompression;
|
using Microsoft.AspNetCore.ResponseCompression;
|
||||||
using Microsoft.AspNetCore.StaticFiles;
|
using Microsoft.AspNetCore.StaticFiles;
|
||||||
@@ -171,6 +172,9 @@ public class Startup(IConfiguration configuration)
|
|||||||
client.BaseAddress = new Uri(apiUrl);
|
client.BaseAddress = new Uri(apiUrl);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
services.Configure<IndexNowOptions>(Configuration.GetSection("IndexNow"));
|
||||||
|
services.AddHttpClient("IndexNow");
|
||||||
|
|
||||||
// OpenAI + NLLB HttpClients (each registered only when its Url is configured) plus the
|
// OpenAI + NLLB HttpClients (each registered only when its Url is configured) plus the
|
||||||
// shared TranslationService singleton. Lives in Marechai.Translation so the API server
|
// shared TranslationService singleton. Lives in Marechai.Translation so the API server
|
||||||
// (Marechai.Server) can reuse the same provider for the genre translation worker.
|
// (Marechai.Server) can reuse the same provider for the genre translation worker.
|
||||||
@@ -344,6 +348,27 @@ public class Startup(IConfiguration configuration)
|
|||||||
$"User-agent: *\nAllow: /\n\nSitemap: {Helpers.SeoMeta.CanonicalHost}/sitemap.xml\n");
|
$"User-agent: *\nAllow: /\n\nSitemap: {Helpers.SeoMeta.CanonicalHost}/sitemap.xml\n");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// IndexNow key verification file — search engines crawl /{key}.txt to prove
|
||||||
|
// we own the domain. Served dynamically so the key lives only in appsettings.
|
||||||
|
endpoints.MapGet("/{filename}.txt", context =>
|
||||||
|
{
|
||||||
|
string filename = (string)context.Request.RouteValues["filename"];
|
||||||
|
string configKey = Configuration["IndexNow:Key"];
|
||||||
|
|
||||||
|
if(string.IsNullOrWhiteSpace(configKey) ||
|
||||||
|
!string.Equals(filename, configKey, StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
context.Response.StatusCode = 404;
|
||||||
|
|
||||||
|
return Task.CompletedTask;
|
||||||
|
}
|
||||||
|
|
||||||
|
context.Response.ContentType = "text/plain; charset=utf-8";
|
||||||
|
context.Response.Headers["Cache-Control"] = "public, max-age=86400";
|
||||||
|
|
||||||
|
return context.Response.WriteAsync(configKey);
|
||||||
|
});
|
||||||
|
|
||||||
endpoints.MapGet("/sitemap.xml", async context =>
|
endpoints.MapGet("/sitemap.xml", async context =>
|
||||||
{
|
{
|
||||||
var sitemapService = context.RequestServices.GetRequiredService<SitemapService>();
|
var sitemapService = context.RequestServices.GetRequiredService<SitemapService>();
|
||||||
|
|||||||
@@ -70,6 +70,10 @@
|
|||||||
"Description": "A normal user role."
|
"Description": "A normal user role."
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
"IndexNow": {
|
||||||
|
"Enabled": true,
|
||||||
|
"Key": null
|
||||||
|
},
|
||||||
"Smtp": {
|
"Smtp": {
|
||||||
"Host": "",
|
"Host": "",
|
||||||
"Port": 587,
|
"Port": 587,
|
||||||
|
|||||||
Reference in New Issue
Block a user