# Aurora — robots.txt
#
# Training and search are SEPARATE controls at every vendor. They are granted
# separately below, on purpose:
#
#   search bots   — allowed without qualification. Aurora cannot be recommended
#                   by an assistant that is not permitted to read the site.
#   training bots — allowed for the public marketing site and the blog only.
#                   Nothing here is user data; it is prose Aurora published
#                   deliberately. Refusing training would buy no user privacy.
#
# WARNING: robots.txt is not the only gate. A CDN can block AI crawlers before
# the request reaches this file — Cloudflare has done so by default on new
# domains since 2025-07-01. Verify with a real request per user-agent, and check
# the origin logs, before trusting anything in this file.
#
# Changes here take roughly 24 hours to propagate to OpenAI's systems.

# ── OpenAI ───────────────────────────────────────────────────────────
# Search: governs whether Aurora can appear in ChatGPT search answers.
User-agent: OAI-SearchBot
Allow: /

# User-directed: a person asked ChatGPT to open a page.
User-agent: ChatGPT-User
Allow: /

# Training.
User-agent: GPTBot
Allow: /

# ── Anthropic ────────────────────────────────────────────────────────
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

# ── Google ───────────────────────────────────────────────────────────
# Googlebot also feeds AI Overviews and AI Mode.
User-agent: Googlebot
Allow: /

# Google-Extended is a control token with no user-agent of its own. It governs
# Gemini training AND grounding — the content Gemini is given at prompt time.
# Disallowing it removes Aurora from Gemini's grounding without any gain.
User-agent: Google-Extended
Allow: /

# ── Microsoft / Copilot ──────────────────────────────────────────────
User-agent: bingbot
Allow: /

# ── Perplexity ───────────────────────────────────────────────────────
User-agent: PerplexityBot
Allow: /

# Perplexity-User is user-initiated and generally ignores robots.txt anyway.
User-agent: Perplexity-User
Allow: /

# ── Others ───────────────────────────────────────────────────────────
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: MistralAI-User
Allow: /

# xAI publishes no crawler documentation and Grok's retrieval has been reported
# using rotating residential IPs and spoofed browser user-agents. There is no
# token here that would reliably reach it. The defence is not a robots rule: it
# is that every page works for an anonymous, JavaScript-free, single-shot GET.

# ── Everything else ──────────────────────────────────────────────────
User-agent: *
Allow: /

Sitemap: https://auroraisyours.com/sitemap.xml