# ─── AI SEO Scanner Robots.txt ───────────────────────────────────────────────
# https://useaiseo.app/robots.txt

# Allow all well-behaved crawlers
User-agent: *
Allow: /

# Public pages — explicitly encourage indexing
Allow: /about
Allow: /features
Allow: /how-it-works
Allow: /faq
Allow: /contact
Allow: /pricing
Allow: /blog
Allow: /llms.txt

# Disallow authenticated/internal routes
Disallow: /app
Disallow: /app/
Disallow: /admin
Disallow: /admin/
Disallow: /auth
Disallow: /auth/
Disallow: /onboarding
Disallow: /onboarding/
Disallow: /api/
Disallow: /_next/

# Disallow crawler-unfriendly query strings
Disallow: /*?*utm_
Disallow: /*?*ref=

# ─── AI Search Crawlers ───────────────────────────────────────────────────────
# Allow AI search crawlers full access to public content so they can cite us.

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

# Google Gemini & AI Overviews
User-agent: Google-Extended
Allow: /

# Microsoft Copilot (via Bing)
User-agent: Bingbot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

# Common Crawl (training data — allow public pages)
User-agent: CCBot
Allow: /
Disallow: /app/
Disallow: /admin/
Disallow: /api/

# Sitemap
Sitemap: https://useaiseo.app/sitemap.xml

# Host
Host: https://useaiseo.app