### AI BOTS CONFIGURATION ###
# OpenAI
User-agent: GPTBot
Allow: /
Crawl-delay: 5

User-agent: ChatGPT-User
Allow: /
Disallow: /checkout/
Disallow: /cart/

# Anthropic
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: Anthropic-ai
Allow: /
Crawl-delay: 3

# Common Crawl
User-agent: CCBot
Allow: /
Disallow: /private/

# Google AI
User-agent: Google-Extended
Allow: /

# Perplexity
User-agent: Omgilibot
Allow: /

# Other AI
User-agent: Gemini
User-agent: MistralAI
User-agent: NeevaBot
User-agent: Anthropic-ai
User-agent: PerplexityBot

Allow: /
### SEARCH ENGINES ###
# Google
User-agent: Googlebot
User-agent: Googlebot-Image
Allow: /
Allow: /*.css$
Allow: /*.js$

# Bing
User-agent: bingbot
User-agent: MSNBot
Allow: /
Crawl-delay: 3

# Others
User-agent: DuckDuckBot
User-agent: Yandex
Allow: /

### SOCIAL MEDIA ###
User-agent: FacebookBot
User-agent: Twitterbot
User-agent: LinkedInBot
User-agent: Pinterestbot
Allow: /
Disallow: /account/

### SPECIAL BOTS ###
User-agent: ia_archiver  # Wayback Machine
Allow: /
Request-rate: 1/3600     # 1 page/hour

User-agent: Applebot      # Apple Spotlight
Allow: /