# robots.txt — Tenet AI (tenetai.dev)
# https://tenetai.dev/robots.txt
# Last updated: 2026-04-27 (2026 AI bot taxonomy per Anthropic/OpenAI/Google/Bing docs)

# ── Traditional search crawlers ────────────────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# ── AI search & retrieval bots — ALLOW (these decide AI citations) ──────────
# ChatGPT Search index (Bing-powered) — critical for ChatGPT search citations
User-agent: OAI-SearchBot
Allow: /

# User-initiated ChatGPT web fetches
User-agent: ChatGPT-User
Allow: /

# Anthropic search index — determines Claude citations
User-agent: Claude-SearchBot
Allow: /

# User-initiated Claude web fetches
User-agent: Claude-User
Allow: /

# Perplexity search index
User-agent: PerplexityBot
Allow: /

# Perplexity live retrieval
User-agent: Perplexity-User
Allow: /

# Google Gemini / AI Overviews ground truth
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# DuckDuckGo AI Assistant
User-agent: DuckAssistBot
Allow: /

User-agent: DuckDuckBot
Allow: /

# Mistral AI
User-agent: MistralAI-User
Allow: /

# Amazon Alexa / Bedrock agents
User-agent: Amazonbot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Yahoo Slurp
User-agent: Slurp
Allow: /

# ── AI training-only bots ──────────────────────────────────────────────────
User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

# Common Crawl — limited B2B value, block training use
User-agent: CCBot
Disallow: /

# TikTok crawler — limited B2B value
User-agent: Bytespider
Disallow: /

# Meta training crawler
User-agent: Meta-ExternalAgent
Disallow: /

# ── Global catch-all ───────────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /api/internal/
Disallow: /checkout/
Disallow: /account/
Disallow: /search?

# ── LLM / GEO discovery ───────────────────────────────────────────────────
# Structured information for LLMs: https://tenetai.dev/llms.txt
Sitemap: https://tenetai.dev/sitemap.xml