# ============================================
# robots.txt for https://chipotlmenus.com/
# Optimized for SEO, speed & protection
# ============================================


# ----------------------------------
# ALLOW GOOGLE CRAWLERS
# ----------------------------------
User-agent: Googlebot
Disallow:
Crawl-delay: 1

User-agent: Googlebot-Image
Disallow:
Crawl-delay: 1

User-agent: Googlebot-News
Disallow:
Crawl-delay: 1

User-agent: Googlebot-Video
Disallow:
Crawl-delay: 1


# ----------------------------------
# ALLOW BING
# ----------------------------------
User-agent: Bingbot
Disallow:
Crawl-delay: 1


# ----------------------------------
# BLOCK AI SCRAPER BOTS (Optional but recommended)
# ----------------------------------
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /


# ----------------------------------
# BLOCK BAD / AGGRESSIVE SCRAPERS
# ----------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /


# ----------------------------------
# DEFAULT RULES FOR ALL OTHER BOTS
# ----------------------------------
User-agent: *
Disallow: /cdn-cgi/
Allow: /
Crawl-delay: 10


# ----------------------------------
# SITEMAP
# ----------------------------------
Sitemap: https://chipotlmenus.com/sitemap_index.xml