# MiPrep AI — the interview copilot that runs privately (miprep.ai)
# Maximum visibility: allow all search + AI citation bots.
# Signals: ai-train=yes, search=yes, ai-input=yes
#
# IMPORTANT — robots.txt group-matching gotcha (Google spec):
#   A bot uses ONLY its most-specific User-agent group. A stanza like
#   `User-agent: Googlebot \n Allow: /` with no Disallow rules SILENTLY
#   overrides the Default block's Disallows for that bot. Every explicit
#   per-bot stanza below carries BOTH Allow AND Disallow — the welcomed
#   bots not in an explicit stanza inherit Default (they're documented
#   as comments only).
#
#   To DISALLOW a specific bot in the future, add a stanza with
#   User-agent + Disallow: /. Don't add User-agent stanzas with only
#   "Allow: /" — they're no-ops that break the Default group for that bot.

# === DEFAULT (applies to all bots without an explicit override below) ===
User-agent: *
Allow: /
Disallow: /api/
Disallow: /_next/data/

# === BOTS WE WELCOME (documentation only; Default group above applies) ===
#
# Traditional search:
#   Googlebot, Bingbot, YandexBot, Baiduspider (+ render/image/mobile),
#   DuckDuckBot, BraveBot, NaverBot, SeznamBot, Sogou web spider,
#   HaosouSpider, coccocbot, Daumoa, MojeekBot, Qwantify
#
# AI search (citations with links):
#   OAI-SearchBot, ChatGPT-User, Claude-SearchBot, Claude-User, Claude-Web,
#   PerplexityBot, Perplexity-User, MistralAI-User, meta-webindexer,
#   GrokBot, xAI-Grok, Grok-DeepSearch, DuckAssistBot, Applebot, YouBot
#
# AI training (allowed during brand-awareness phase; flip to Disallow
# per-bot in 6-12 months by adding a stanza with both User-agent AND
# Disallow: / — single-line "Allow: /" stanzas don't work, see header):
#   GPTBot, ClaudeBot, Google-Extended, GoogleOther, Applebot-Extended,
#   meta-externalagent, anthropic-ai, cohere-ai, CCBot, Amazonbot,
#   Bytespider, Diffbot, ia_archiver
#
# Social link unfurls:
#   facebookexternalhit, FacebookBot, Twitterbot, LinkedInBot, Slackbot,
#   TelegramBot, WhatsApp, Discordbot, redditbot, PinterestBot

# === CITATION-BOT EXPLICIT ALLOWS ===
# Belt-and-suspenders welcome signal for the AI search/citation bots
# driving entry traffic. Each carries BOTH Allow AND real Disallow rules
# — Allow-only stanzas silently break Default (see gotcha at top).

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: MistralAI-User
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: Applebot
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: Google-Extended
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /_next/data/

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /_next/data/

# === AI CONTEXT & LLM DISCOVERY ===
# LLM-optimized context (short summary)
# https://miprep.ai/llms.txt
# LLM-optimized context (full detail)
# https://miprep.ai/llms-full.txt
# Anthropic-style context file
# https://miprep.ai/llms-ctx.txt

# === .well-known (agent/LLM auto-discovery conventions) ===
# https://miprep.ai/.well-known/agents.json
# https://miprep.ai/.well-known/ai-plugin.json
# https://miprep.ai/.well-known/mcp.json
# https://miprep.ai/.well-known/mcp/server-card.json
# https://miprep.ai/.well-known/security.txt

# === MACHINE-READABLE PRODUCT MANIFEST ===
# https://miprep.ai/llms.json

# === SITEMAPS ===
# One per serving surface. miprep.ai is five static-first surfaces behind
# Caddy (apps/web + miprep-{content,jobs,news,farm}); each owns its own sitemap
# and a URL appears in exactly one of them. Adding a surface without adding
# its Sitemap line here is how ~580 URLs stayed undeclared until 2026-08-04
# — see docs/specs/seo-geo-gaps-2026-08-04.md.
#
# NOTE on /content-sitemap.xml: miprep-content is routed by per-shape regex
# handlers (@alt_shape, ^/blog/([^/]+)/?$), so a file at /blog/sitemap.xml
# would be swallowed by the blog-slug handler. It is therefore published at
# the apex path instead. A sitemap at the apex may list any path on the host.
#
# miprep-farm (/answers/*) — LIVE 2026-08-09. Was deferred while the surface
# held only a noindex placeholder, because declaring a 404ing sitemap is a
# standing Search Console error against the whole host. That condition is gone:
# the first farm (resume synonyms) shipped 105 URLs and /answers/sitemap.xml
# returns 200. Declared in the same change that shipped it, which is the
# discipline the 2026-08-04 gap was missing.
Sitemap: https://miprep.ai/answers/sitemap.xml
Sitemap: https://miprep.ai/sitemap.xml
Sitemap: https://miprep.ai/content-sitemap.xml
Sitemap: https://miprep.ai/jobs/sitemap.xml
Sitemap: https://miprep.ai/news/sitemap-news.xml