# robots.txt for rfab.ai
# NOTE: per the Robots Exclusion Protocol a crawler obeys ONLY its most
# specific matching group — named groups do NOT inherit rules from "*".
# Any Disallow added below must be added to BOTH groups.

# --- AI crawlers (named so they can be tuned independently) ---
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /email
Disallow: /game
Disallow: /settings
Disallow: /tokens/
Disallow: /companions/new
Disallow: /companions/*/edit
Disallow: /chat/companion/
Disallow: /custom-adventures
Disallow: /adventure-creator
Disallow: /nsfw-scenario-creator
Disallow: /my-scenarios
Disallow: /saves

# --- Everyone else ---
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /email
Disallow: /game
Disallow: /settings
Disallow: /tokens/
Disallow: /companions/new
Disallow: /companions/*/edit
Disallow: /chat/companion/
Disallow: /custom-adventures
Disallow: /adventure-creator
Disallow: /nsfw-scenario-creator
Disallow: /my-scenarios
Disallow: /saves

# Dynamic sitemap (all scenarios, agents, recipes, hreflang matrices)
Sitemap: https://api.rfab.ai/api/sitemap.xml
# Static fallback sitemap on this host
Sitemap: https://rfab.ai/sitemap.xml

# --- AI-readable documentation ---
# Machine-readable platform + API docs (llmstxt.org convention):
#   https://rfab.ai/llms.txt
#   https://rfab.ai/llms-full.txt
#   https://rfab.ai/.well-known/llms.txt
#   https://rfab.ai/.well-known/llms-full.txt
#   https://rfab.ai/ai-guide.html
#
# Platform: Autonomous AI Agent Platform & Developer API
# Developer API: 30+ endpoints — image gen (100+ models), video, VTuber avatars, lip-sync, SAM2/GroundedSAM segmentation, face parsing, bg removal
# API Signup: Free at https://rfab.ai → Settings → API Keys → Create
# API Base URL: https://api.rfab.ai
# Unique Features: Alloy Mode, OpenAI Realtime Voices, 70+ Actions, Plain Language Chaining
# Models: 15+ (GPT-4o, Claude Fable 5, Gemini 2.5 Pro, Grok 4.5, Mistral, DeepSeek)
# Image Models: 100+ via Replicate, ModelsLab, AWS Bedrock, Atlas Cloud, Runware, Grok
# Languages: 6 (English, Spanish, Portuguese, French, German, Polish)
# Free Tier: 50K signup + 25K daily tokens