From f1b2024f82c7cb5131d7d6bc8bdfcbd609c387f0 Mon Sep 17 00:00:00 2001
From: Atanas Korchev <akorchev@gmail.com>
Date: Fri, 29 May 2026 20:25:50 +0300
Subject: [PATCH] Don't disallow llms.txt and .md in robots.txt

They already send X-Robots-Tag: noindex, which keeps them out of search results while staying readable by AI agents. A robots.txt Disallow blocks fetching entirely (so the noindex is never seen) and blocks agents from llms.txt, defeating its purpose.
---
 RadzenBlazorDemos.Tools/Program.cs | 2 --
 1 file changed, 2 deletions(-)

diff --git a/RadzenBlazorDemos.Tools/Program.cs b/RadzenBlazorDemos.Tools/Program.cs
index 23c1e4445..313e9ddf9 100644
--- a/RadzenBlazorDemos.Tools/Program.cs
+++ b/RadzenBlazorDemos.Tools/Program.cs
@@ -1528,8 +1528,6 @@ class Program
         sb.AppendLine("User-agent: *");
         sb.AppendLine("Content-Signal: ai-train=yes, search=yes, ai-input=yes");
         sb.AppendLine($"Sitemap: {BaseUrl}/sitemap.xml");
-        sb.AppendLine("Disallow: /llms.txt");
-        sb.AppendLine("Disallow: /*.md$");
         File.WriteAllText(filePath, sb.ToString(), new UTF8Encoding(false));
     }
 }