# arundo.com — production robots.txt (content/48 §A2).
# The preview deployment never serves this file: when the Worker's SITE_ENV var is not
# "production" it answers /robots.txt with a disallow-all body and adds X-Robots-Tag: noindex
# to every response (src/worker.js, wrangler.jsonc).
#
# Search and AI crawlers are allowed explicitly. Each named group repeats the same rules
# because a crawler that finds its own group ignores the catch-all.
# Pages that must not be indexed carry <meta name="robots" content="noindex"> instead of a
# Disallow here — a disallowed URL can never show its noindex.

User-agent: Googlebot
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: Bingbot
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: GPTBot
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: OAI-SearchBot
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: ChatGPT-User
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: ClaudeBot
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: Claude-SearchBot
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: Claude-User
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: PerplexityBot
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: Perplexity-User
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: Google-Extended
Allow: /
Disallow: /review/
Disallow: /*.draft

User-agent: *
Allow: /
Disallow: /review/
Disallow: /*.draft

Sitemap: https://arundo.com/sitemap.xml