# robots.txt for https://aidaide.app
# Last updated: 2026-05-19

# ---------------------------------------------------------------
# Default policy — allow all crawlers on public pages
# ---------------------------------------------------------------
User-agent: *
Allow: /

# Account / auth flow — gated content, no crawl value
Disallow: /login
Disallow: /signup
Disallow: /signup-success
Disallow: /dashboard
Disallow: /dashboard-redesign-mockup
Disallow: /dashboard-redesign-mockup.html
Disallow: /checkout
Disallow: /success

# Admin & internal APIs
Disallow: /admin
Disallow: /api/

# Retired routes (302 -> /pricing). Block defensively so cached copies
# don't compete with the live pricing page in search results.
Disallow: /beta
Disallow: /beta.html
Disallow: /notify
Disallow: /notify.html

# R2 release downloads & latest manifest are rewrites, not pages.
Disallow: /download/
Disallow: /latest.json

# ---------------------------------------------------------------
# Image crawlers — explicit allow on social/og + screenshots
# (the default Allow: / above already permits these; this section
# is documentation, not policy change)
# ---------------------------------------------------------------
User-agent: Googlebot-Image
Allow: /og-image.png
Allow: /screenshots/
Allow: /favicon-32.png
Allow: /favicon-16.png
Allow: /apple-touch-icon.png
Allow: /icon-192.png
Allow: /icon-512.png
Allow: /logo.svg

# ---------------------------------------------------------------
# AI / GEO crawlers — allow indexing for generative search citations
# (GPTBot, Anthropic, Perplexity, Google-Extended, Common Crawl)
# Aligns with the SEO strategy doc's GEO/RAG content tactics.
# ---------------------------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /checkout
Disallow: /success
Disallow: /signup
Disallow: /signup-success
Disallow: /login

User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /checkout
Disallow: /success
Disallow: /signup
Disallow: /signup-success
Disallow: /login

User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /checkout
Disallow: /success
Disallow: /signup
Disallow: /signup-success
Disallow: /login

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /checkout
Disallow: /success
Disallow: /signup
Disallow: /signup-success
Disallow: /login

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /checkout
Disallow: /success
Disallow: /signup
Disallow: /signup-success
Disallow: /login

User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /checkout
Disallow: /success
Disallow: /signup
Disallow: /signup-success
Disallow: /login

User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /api/
Disallow: /dashboard
Disallow: /checkout
Disallow: /success
Disallow: /signup
Disallow: /signup-success
Disallow: /login

# ---------------------------------------------------------------
# Aggressive/scraper bots — block
# ---------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# ---------------------------------------------------------------
# Sitemap
# ---------------------------------------------------------------
Sitemap: https://aidaide.app/sitemap.xml