# ============================================================
# robots.txt for nontoxiclab.com
# Last updated: 2026-06-11
# Strategy: Allow retrieval/citation crawlers (search engines + AI answer
# engines that fetch live and cite with links — these send buying traffic).
# Block AI TRAINING corpora crawlers (they consume content for model
# training and send no traffic). This is an explicit OWNER DECISION made
# 2026-06-11, reversing the 2026-04-24 allow-training stance, and it
# aligns this file with the ai-train=no Content Signal in /llms.txt.
# Do NOT revert without explicit owner approval.
# Google-Extended stays ALLOWED deliberately (blocking it would cost
# Gemini-app citations; it never affects Search rankings either way).
# Enforcement note: this file is advisory only. The edge middleware
# (functions/_middleware.js + _bot-blocklist.js) stays DISARMED for all
# of these bots — never add training bots to the edge 403 layer.
# ============================================================

# Traditional search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# AI search / answer engines (allowed — they drive cited traffic)
User-agent: PerplexityBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: xAI-Bot
Allow: /

# Applebot-Extended (Apple Intelligence TRAINING token) is blocked and must
# stay listed BEFORE the plain Applebot group: naive first-match robots
# parsers would otherwise match "Applebot-Extended" against the "Applebot"
# group. Google-style longest-match parsers are order-independent.
User-agent: Applebot-Extended
Disallow: /

# Social preview crawlers (allowed — they fetch og:image/og:title for link
# previews when articles are shared on Facebook, Apple iMessage, X, LinkedIn,
# Pinterest, Telegram, Discord, etc.). These were previously inheriting the
# catch-all but are listed explicitly so the allow-list intent is self-
# documenting. Added 2026-05-02 per geo-ai-visibility audit.
# Plain Applebot (Siri/Spotlight/link previews) stays allowed.
User-agent: Applebot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: facebookcatalog
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Discordbot
Allow: /

User-agent: Pinterest
Allow: /

User-agent: Pinterestbot
Allow: /

# AI training corpora — BLOCKED per owner decision 2026-06-11.
# These crawlers consume content for model training and send no traffic.
# The retrieval/citation bots above stay allowed: ChatGPT search uses
# OAI-SearchBot + ChatGPT-User (not GPTBot), Claude search uses
# Claude-SearchBot + Claude-User (not ClaudeBot), so blocking the
# training tokens does NOT remove the site from any AI search surface.
# anthropic-ai and Claude-Web are legacy Anthropic training tokens.
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

# (Applebot-Extended is blocked above, before the plain Applebot group,
# for naive-parser ordering safety.)

User-agent: cohere-ai
Disallow: /

User-agent: MistralBot
Disallow: /

# Left allowed for now (judgment calls, owner can flip later):
# DeepSeekBot, HuggingFaceBot — primarily training-oriented but were not
# in the approved block list of 2026-06-11.
User-agent: DeepSeekBot
Allow: /

User-agent: HuggingFaceBot
Allow: /

# Nuisance / no-citation scrapers (blocked)
User-agent: Amazonbot
Disallow: /

User-agent: Amzn-SearchBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Diffbot
Disallow: /

# Everything else
User-agent: *
Allow: /

# Content Signals (contentsignals.org / IETF draft-romm-aipref-contentsignals).
# Declares AI content-usage preferences site-wide, mirroring the same signal
# already published in /llms.txt. Matches the 2026-06-11 owner decision:
#   search   = yes  (allow indexing for search/answer engines that cite)
#   ai-train = no   (do NOT use content for model training)
#   ai-input = yes  (allow live AI input/grounding/RAG that cites back)
# Keep in sync with /llms.txt and the robots allow/deny groups above.
Content-Signal: search=yes, ai-train=no, ai-input=yes

Sitemap: https://nontoxiclab.com/sitemap-index.xml