# InsidethePen.com - robots.txt
# Updated 2026-07-04


# ---- Search engines: full access ----
# NOTE: Googlebot also powers Google AI Overviews. There is no way to stay in
# Google Search while opting out of AI Overviews; they are one crawler.
User-agent: Googlebot
User-agent: Bingbot
User-agent: Slurp
Disallow: /player_mugs/

# ---- AI citation / live-retrieval agents: full access ----
# These agents fetch pages in response to a user question and cite/link the
# source. 
#
# OAI-SearchBot      = ChatGPT Search indexing/citations (OpenAI)
# ChatGPT-User       = user-triggered live browsing (OpenAI)
# PerplexityBot      = Perplexity indexing/citations
# Perplexity-User    = user-triggered live fetch (Perplexity)
# Claude-User        = user-triggered live fetch (Anthropic)
# Claude-SearchBot   = search indexing/citations (Anthropic)
# DuckAssistBot      = DuckDuckGo AI answers (cites sources)
# MistralAI-User     = user-triggered live fetch (Mistral)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: DuckAssistBot
User-agent: MistralAI-User
Disallow: /player_mugs/

# ---- Google-Extended: limited access (training/grounding token) ----
# Kept SEPARATE from the retrieval agents above on purpose: this is not a
# fetcher, it is the control token for Gemini TRAINING and grounding, read by
# Googlebot.
User-agent: Google-Extended
Allow: /$
Allow: /index.html
Allow: /articles/
Allow: /bullpen-rankings-season-long.html
Allow: /bullpen-era-rankings.html
Allow: /premium.html
Disallow: /

# ---- AI training / dataset crawlers: blocked ----
# GPTBot             = OpenAI model training
# ClaudeBot          = Anthropic model training (the training crawler, not the
#                      citation agents listed above)
# anthropic-ai, Claude-Web = legacy Anthropic UAs, kept for safety
# CCBot              = Common Crawl (feeds many training sets)
# Meta-*             = Meta AI training and fetching
# Bytespider         = ByteDance/TikTok
# Applebot-Extended  = Apple Intelligence training (control token)
# Amazonbot          = Amazon/Alexa
# Omgilibot/omgili   = webz.io dataset vendor
# Timpibot           = Timpi index
# AI2Bot             = Allen Institute datasets
# PetalBot           = Huawei/Petal
# GoogleOther        = Google R&D crawler (not used for Search ranking)
# Diffbot            = structured data extraction
# img2dataset        = image dataset harvester
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: anthropic-ai
User-agent: Claude-Web
User-agent: CCBot
User-agent: FacebookBot
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: cohere-ai
User-agent: Bytespider
User-agent: Applebot-Extended
User-agent: Amazonbot
User-agent: Omgilibot
User-agent: omgili
User-agent: Timpibot
User-agent: AI2Bot
User-agent: PetalBot
User-agent: GoogleOther
User-agent: Diffbot
User-agent: img2dataset
User-agent: Scrapy
Disallow: /

# ---- SEO crawlers: allowed (backlink analysis, not AI) ----
User-agent: SemrushBot
User-agent: AhrefsBot
User-agent: DotBot
User-agent: MJ12bot
Allow: /

# ---- All other bots ----
User-agent: *
Disallow: /player_mugs/

# If a sitemap is added later, declare it here (verify the URL first):
# Sitemap: https://insidethepen.com/sitemap.xml