# Dog Gear Review - robots.txt
# AI training on this content is expressly prohibited.
# AI search and assistant crawlers that cite and link back are allowed.
# See https://doggearreview.com/terms for full usage restrictions.

# ===========================================
# AI Training Crawlers - BLOCKED
# ===========================================

# OpenAI (GPT model training)
User-agent: GPTBot
Disallow: /

# Anthropic (Claude model training)
User-agent: anthropic-ai
Disallow: /

User-agent: ClaudeBot
Disallow: /

# Google (Gemini training - does NOT affect Google Search ranking or AI Overviews)
User-agent: Google-Extended
Disallow: /

# Apple (Apple Intelligence training - Applebot itself stays allowed for Siri/Spotlight)
User-agent: Applebot-Extended
Disallow: /

# Meta AI (LLaMA training)
User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

# DeepSeek (model training)
User-agent: DeepseekBot
Disallow: /

# xAI / Grok (model training)
User-agent: xAI-Bot
Disallow: /

# Amazon (Alexa for Shopping / Rufus - ingests content with no citation or
# link back; answers link only to Amazon product pages, so no value to DGR)
User-agent: Amazonbot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Common Crawl (major AI training data source)
User-agent: CCBot
Disallow: /

# Data resellers / scrapers
User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: img2dataset
Disallow: /

# ===========================================
# AI Search & Assistant Crawlers - ALLOWED
# (index/retrieve and cite with links back to the site)
# ===========================================

# OpenAI - search index + user-initiated browsing for ChatGPT answers
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic - search index + user-initiated retrieval for Claude answers
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity AI Search - cites sources with links
User-agent: PerplexityBot
Allow: /

# You.com AI Search - cites sources with links
User-agent: YouBot
Allow: /

# ===========================================
# Social Link Preview Scrapers - ALLOWED
# (needed for OG images on shared links)
# ===========================================

User-agent: FacebookBot
Allow: /

User-agent: facebookexternalhit
Allow: /

# ===========================================
# Traditional Search Engines - ALLOWED
# ===========================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: *
Allow: /

# Sitemap
Sitemap: https://doggearreview.com/sitemap-index.xml