User-agent: *
Allow: /
Disallow: /System.php
Disallow: /config/

Sitemap: https://shopware-free-theme.de/sitemap.xml

# ==============================================================================
# AI Crawler - Alle erlaubt für maximale Sichtbarkeit
# ==============================================================================

# OpenAI GPTBot (ChatGPT, GPT-4, etc.)
User-agent: GPTBot
Allow: /

# Anthropic Claude (Claude.ai)
User-agent: Claude-Web
Allow: /

# Google Extended (Bard, Gemini AI Training)
User-agent: Google-Extended
Allow: /

# Common Crawl (Trainingsdaten für viele AI-Modelle)
User-agent: CCBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Meta AI (LLaMA Training)
User-agent: FacebookBot
Allow: /

# Diffbot (Knowledge Graph für AI)
User-agent: Diffbot
Allow: /

# Applebot Extended (Apple Intelligence)
User-agent: Applebot-Extended
Allow: /

# ==============================================================================
# Standard Search Engine Bots
# ==============================================================================

# Google
User-agent: Googlebot
Allow: /

# Bing
User-agent: Bingbot
Allow: /

# Yandex
User-agent: YandexBot
Allow: /

# Baidu
User-agent: Baiduspider
Allow: /

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# ==============================================================================
# Crawl Delay für respektvolles Crawling
# ==============================================================================
Crawl-delay: 1