# BuddyLeader — robots.txt
# Last updated: 2026-06-01
# Generative engine optimisation: AI crawlers are explicitly welcomed.
# Canonical entity description for LLMs lives at /llms.txt.

# ============================================================
# Default policy — search engines and well-behaved crawlers
# ============================================================
User-agent: *
Allow: /
Disallow: /demo
Disallow: /pilot-internal
Disallow: /admin
Disallow: /api
Crawl-delay: 1

# ============================================================
# Generative AI / LLM crawlers — explicitly allowed
# ============================================================

# OpenAI — ChatGPT browsing, GPTBot, OAI-SearchBot
User-agent: GPTBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: ChatGPT-User
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: OAI-SearchBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Anthropic — Claude
User-agent: ClaudeBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: anthropic-ai
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: Claude-Web
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: Perplexity-User
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Google — Gemini / extended
User-agent: Google-Extended
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: GoogleOther
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Common Crawl (used by many models for training and grounding)
User-agent: CCBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Apple Intelligence
User-agent: Applebot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: Applebot-Extended
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /demo
Disallow: /pilot-internal

User-agent: FacebookBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Mistral / Le Chat
User-agent: MistralAI-User
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# You.com
User-agent: YouBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Bytespider (ByteDance)
User-agent: Bytespider
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# DuckDuckGo
User-agent: DuckAssistBot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Microsoft Copilot
User-agent: MSNBot-Media
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# Diffbot (used in enterprise grounding)
User-agent: Diffbot
Allow: /
Disallow: /demo
Disallow: /pilot-internal

# ============================================================
# Sitemap and llms.txt pointers
# ============================================================
Sitemap: https://buddyleader.com/sitemap.xml

# LLM canonical answer surface — non-standard, well-recognised hint
# A growing set of AI agents respect a /llms.txt convention.
# See: https://llmstxt.org
# Canonical: https://buddyleader.com/llms.txt

# MCP product feed — machine-actionable entity + FAQ for AI agents
# https://buddyleader.com/mcp.json