# Turner Septic Services: septic pumping, inspections and more in Conway, NH
#
# THE POLICY, in one line: this site's content may be used to ANSWER questions,
# and may not be used to TRAIN models. (Josh, 2026-10-01: "block AI learning,
# we just want it found by AI." Same rule as groundbreakermarketing.com.)
#
# Answering and training are different crawlers, often from the same company:
# OpenAI trains with GPTBot and answers with OAI-SearchBot; Anthropic trains
# with ClaudeBot and answers with Claude-User and Claude-SearchBot. Allowing a
# vendor wholesale opts the site into training; blocking one wholesale makes it
# uncitable in that vendor's answers. So each bot is listed by the job it does.
#
# Indexing on the dev host is suppressed by the X-Robots-Tag header that
# functions/_middleware.js sends on every non-production host. This file is
# written for the final domain and needs no change at cutover.

# ── Search and answer engines: allowed ────────────────────────────────────────
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Answer engines and user-initiated fetches. These retrieve a page to answer a
# question someone actually asked, and cite it. They do not feed training sets.
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: MistralAI-User
Allow: /

# Classic search crawlers. Googlebot and Bingbot also ground Google's AI
# Overviews and Microsoft Copilot, so these carry the answer use case too.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

# ── Model training: not allowed ───────────────────────────────────────────────
# Each of these exists to collect content for training or fine-tuning.

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: cohere-ai
Disallow: /

# Google-Extended is the one place the two goals cannot be fully separated.
# It governs Gemini training AND Gemini app grounding, with no way to permit one
# and refuse the other. Refusing it is the choice that matches the policy above.
# This does NOT remove the site from AI Overviews in Google Search: those are
# grounded by Googlebot, which is allowed.
User-agent: Google-Extended
Disallow: /

Sitemap: https://www.turnersepticservices.com/sitemap-index.xml