diff --git a/src/pages/robots.txt.ts b/src/pages/robots.txt.ts index 9ab742ae6..b4b777650 100644 --- a/src/pages/robots.txt.ts +++ b/src/pages/robots.txt.ts @@ -1,6 +1,40 @@ import type { APIRoute } from "astro"; const robotsTxt = ` +# AI Search & Citation Bots — block these = disappear from AI answers +User-agent: OAI-SearchBot +Allow: / + +User-agent: ChatGPT-User +Allow: / + +User-agent: Claude-SearchBot +Allow: / + +User-agent: Claude-User +Allow: / + +User-agent: PerplexityBot +Allow: / + +# AI Training Bots — feeds model knowledge of Akash +User-agent: GPTBot +Allow: / + +User-agent: ClaudeBot +Allow: / + +User-agent: Google-Extended +Allow: / + +User-agent: CCBot +Allow: / + +# Block low-value scrapers +User-agent: Bytespider +Disallow: / + +# Standard crawlers User-agent: * Allow: /