# robots.txt ottimizzato per crawler AI
# Ultimo aggiornamento: 2025-07-31

# =====================================
# REGOLE GENERALI PER TUTTI I WEB CRAWLER
# =====================================
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /private/
Disallow: /tmp/
Disallow: /cgi-bin/
Disallow: /wp-admin/
Disallow: /search?
Crawl-delay: 3

# =====================================
# REGOLE SPECIFICHE PER AGENTI AI
# =====================================

# OpenAI (ChatGPT)
User-agent: GPTBot
Allow: /
Disallow: /private/
Disallow: /user-data/
Crawl-delay: 2
Request-rate: 1/5
AI-Policy: https://www.traduzionelibri.it/ai-policy

# Google AI (Gemini)
User-agent: Google-Extended
Allow: /
Disallow: /checkout/
Disallow: /account/
AI-Training: allow
AI-Generative: allow

# Anthropic (Claude)
User-agent: anthropic-ai
Allow: /
Disallow: /confidential/
AI-Opt-Out: 0
AI-Usage: training

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Disallow: /payment/
Disallow: /cookies/
AI-Indexing: full
AI-Data-Collection: permitted

# Microsoft AI (Bing/Copilot)
User-agent: BingBot
Allow: /
Disallow: /api/
AI-Integration: enabled
AI-Search-Enabled: true

# Common Crawl (usato da molti sistemi AI)
User-agent: CCBot
Allow: /
Disallow: /logs/
Disallow: /analytics/

# Apple AI
User-agent: Applebot
Allow: /
Disallow: /internal/
AI-Parser: allow

# Facebook AI
User-agent: facebookexternalhit/1.1
Allow: /
Disallow: /user/

# Amazon AI
User-agent: Amazonbot
Allow: /
Disallow: /partner-area/

# =====================================
# REGOLE PER ALTRI CRAWLER RILEVANTI
# =====================================
User-agent: AhrefsBot
Crawl-delay: 5

User-agent: SEMrushBot
Crawl-delay: 5

User-agent: MozBot
Crawl-delay: 5

# =====================================
# SITEMAP E POLITICHE
# =====================================
Sitemap: https://www.traduzionelibri.it/sitemap.xml
Sitemap: https://www.traduzionelibri.it/ai-sitemap.xml

AI-Policy: https://www.traduzionelibri.it/ai-policy
AI-Training-Portal: https://www.traduzionelibri.it/ai-data