# ========================================
# AIVO — AI Crawler & Search Engine Configuration
# ========================================
# Domain: www.tryaivo.com
# Last Updated: 2026-09-09
# Strategy: Maximum AI visibility — allow search, browsing, and training bots
# AI Contract: /api/llms (LLMS.txt) | /api/llms-full (LLMS-FULL.txt)

# ========================================
# OPENAI — ChatGPT Search, Training, Browsing
# Source: https://developers.openai.com/api/docs/bots
# ========================================

# ChatGPT search results — CRITICAL for AI search visibility
User-agent: OAI-SearchBot
Allow: /
Allow: /api/ai-endpoints/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 1

# GPT foundation model training
User-agent: GPTBot
Allow: /
Allow: /api/ai-endpoints/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 2

# User-triggered ChatGPT browsing (may ignore robots.txt)
User-agent: ChatGPT-User
Allow: /
Allow: /api/ai-endpoints/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 2

# ChatGPT ads landing page validation
User-agent: OAI-AdsBot
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026

# ========================================
# ANTHROPIC — Claude Search, Training, Browsing
# Source: docs.anthropic.com (ClaudeBot verified Dec 2025)
# Deprecated: anthropic-ai, Claude-Web (July 2024)
# ========================================

# Claude model training + citation retrieval
User-agent: ClaudeBot
Allow: /
Allow: /api/ai-endpoints/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 2

# Claude search indexing
User-agent: Claude-SearchBot
Allow: /
Allow: /api/ai-endpoints/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 1

# User-triggered Claude browsing (may ignore robots.txt)
User-agent: Claude-User
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 1

# ========================================
# GOOGLE — Search, Gemini Training, Vertex AI
# Source: https://developers.google.com/crawling/docs/crawlers-fetchers/google-common-crawlers
# ========================================

# Google Search (also powers AI Overviews)
# Note: /_next/ (JS/CSS) is intentionally crawlable — blocking it harms how
# Google renders and indexes client-rendered content.
User-agent: Googlebot
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Disallow: /api/internal/

# Gemini model training + grounding (does not affect Search ranking)
User-agent: Google-Extended
Allow: /
Disallow: /private/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 1

# Vertex AI Agent Builder (site-owner requested crawls)
User-agent: Google-CloudVertexBot
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 1

# Google R&D crawler
User-agent: GoogleOther
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 2

# ========================================
# PERPLEXITY — Search Indexing, User Browsing
# Source: https://docs.perplexity.ai/docs/resources/perplexity-crawlers
# ========================================

# Perplexity search indexing (not for training)
User-agent: PerplexityBot
Allow: /
Allow: /api/ai-endpoints/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 3

# User-triggered Perplexity browsing (generally ignores robots.txt)
User-agent: Perplexity-User
Allow: /
Allow: /api/ai-endpoints/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 1

# ========================================
# APPLE — Siri, Spotlight, Apple Intelligence
# Source: https://support.apple.com/en-us/119829
# ========================================

# Apple Search (Siri, Spotlight, Safari)
User-agent: Applebot
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 1

# Apple Intelligence foundation model training (control token, not a separate crawler)
User-agent: Applebot-Extended
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026

# ========================================
# MICROSOFT — Bing Search + Copilot
# ========================================

User-agent: Bingbot
Allow: /
Allow: /api/ai-endpoints/
Disallow: /admin/
Disallow: /private/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026

# ========================================
# OTHER AI PLATFORMS
# ========================================

# Meta AI (Llama models) — 19% of AI crawler traffic
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 3

# Facebook link previews
User-agent: FacebookBot
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026

# Amazon AI (Alexa, Amazon AI services) — 11% of AI crawler traffic
User-agent: Amazonbot
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 2

# ByteDance AI (TikTok AI, Doubao)
User-agent: Bytespider
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 3

# Common Crawl (open corpus used by many AI labs)
User-agent: CCBot
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 3

# Cohere LLM training
User-agent: cohere-ai
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 2

# DuckDuckGo AI answers
User-agent: DuckAssistBot
Allow: /
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Crawl-delay: 2

# ========================================
# DEFAULT — All Other Crawlers
# ========================================

User-agent: *
Disallow: /admin/
Disallow: /private/
Disallow: /downloads/
Disallow: /snapshot
Disallow: /pricing
Disallow: /platform
Disallow: /possible-2026
Disallow: /resources/hotel-playbook/thank-you
Disallow: /resources/beauty-representation-report/thank-you
Disallow: /analytics/
Disallow: /internal/
Disallow: /staging/
Disallow: /dev/
Disallow: /backup/
Disallow: /temp/
Disallow: /cache/
Disallow: /logs/
Disallow: /api/internal/
Allow: /api/llms
Allow: /api/llms-full
Allow: /.well-known/ai-contracts
Allow: /api/ai-endpoints
Allow: /api/robots
Crawl-delay: 1

# ========================================
# AI CONTRACT REFERENCE
# ========================================
# Primary Index: /api/llms (LLMS.txt navigation layer)
# Full Content: /api/llms-full (LLMS-FULL.txt comprehensive access)
# AI Endpoints: /api/ai-endpoints/ (structured data for AI platforms)
# AI Contracts: /.well-known/ai-contracts

# ========================================
# SITEMAPS
# ========================================
# === SITEMAPS ===
# Main site sitemap
Sitemap: https://www.tryaivo.com/sitemap.xml
# Custom sitemap
Sitemap: https://www.tryaivo.com/perspectives/sitemap.xml
# Custom sitemap
Sitemap: https://www.tryaivo.com/case-studies/sitemap.xml
# Custom sitemap
Sitemap: https://www.tryaivo.com/resources/research/sitemap.xml
# AI-optimized sitemap
Sitemap: https://www.tryaivo.com/ai-sitemap.xml
# Blog content sitemap
Sitemap: https://www.tryaivo.com/blog/sitemap.xml
# Custom sitemap
Sitemap: https://www.tryaivo.com/partners/sitemap.xml

# ========================================
# CONTACT
# ========================================
# Organization: AIVO — Strategic AI Visibility Consultancy
# Technical Contact: team@tryaivo.com
# AI Partnership: team@tryaivo.com
Host: www.tryaivo.com