# === AI Search & Discovery Bots ===

# OpenAI - ChatGPT search indexing
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing
Disallow: /login
Disallow: /register

# OpenAI - ChatGPT real-time browsing
User-agent: ChatGPT-User
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing
Disallow: /login
Disallow: /register

# OpenAI - GPT model training
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing
Disallow: /login
Disallow: /register

# Anthropic - Claude
User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing
Disallow: /login
Disallow: /register

User-agent: anthropic-ai
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing

User-agent: Claude-User
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing

# Perplexity
User-agent: PerplexityBot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing
Disallow: /login
Disallow: /register

# Google - Gemini AI training
User-agent: Google-Extended
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing

# Microsoft - Bing + Copilot
User-agent: bingbot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing
Disallow: /login
Disallow: /register

# Apple - Siri / Spotlight / Apple Intelligence
User-agent: Applebot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing

# Amazon - Alexa
User-agent: Amazonbot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/
Disallow: /billing

# === Blocked Crawlers ===
# Commercial data harvesters / SEO scrapers / regional crawlers with
# no referral value. Enforced at nginx + fail2ban as well.

User-agent: panscient.com
Disallow: /

User-agent: panscient
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: SeekportBot
Disallow: /

# === Standard Crawlers ===

User-agent: *
Allow: /
Disallow: /admin/
Disallow: /training/
Disallow: /dashboard/
Disallow: /api/

# Disallow authentication pages
Disallow: /login
Disallow: /register
Disallow: /reset_password_request
Disallow: /forgot_password
Disallow: /reset_password/
Disallow: /confirm/
Disallow: /set-password/
Disallow: /set_password/

# Disallow checkout and billing
Disallow: /create-checkout-session
Disallow: /checkout-success
Disallow: /checkout-cancel
Disallow: /stripe_webhook
Disallow: /billing
Disallow: /cancel-subscription

# Disallow download endpoints
Disallow: /download_results
Disallow: /download_job_result/

# Disallow tracking endpoints
Disallow: /track/

# Disallow internal API endpoints
Disallow: /internal/
Disallow: /webhook/
Disallow: /graphql

# Sitemap location
Sitemap: https://verirouteintel.com/sitemap.xml
Sitemap: https://verirouteintel.com/sitemap-data.xml
Sitemap: https://verirouteintel.com/sitemap-lookup-pass.xml