# GSTAT by TaxByKK
# Sourcing, editorial policy and corrections: https://gstat.taxbykk.com/about/editorial-standards/
#
# This site may be indexed, and may be quoted and cited when answering someone's
# question. It is not made available for model training. The Content-Signal line
# below states that in the form built for it; the per-agent groups say the same
# thing to the agents that read those instead.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /tbk-admin.html
Disallow: /storage-write.php

# Answering a user's question and citing back here — allowed.
# Claude fetching a page because a user asked about it
User-agent: Claude-User
Allow: /

# Claude's search index
User-agent: Claude-SearchBot
Allow: /

# ChatGPT's search index
User-agent: OAI-SearchBot
Allow: /

# ChatGPT fetching a page for a user
User-agent: ChatGPT-User
Allow: /

# Perplexity's index
User-agent: PerplexityBot
Allow: /

# Perplexity fetching a page for a user
User-agent: Perplexity-User
Allow: /

# Collecting content into a training corpus — not allowed.
# Anthropic model training
User-agent: ClaudeBot
Disallow: /

# OpenAI model training
User-agent: GPTBot
Disallow: /

# Gemini training and grounding — not Search, not AI Overviews
User-agent: Google-Extended
Disallow: /

# legacy Anthropic token
User-agent: anthropic-ai
Disallow: /

# Common Crawl, the corpus most training sets draw from
User-agent: CCBot
Disallow: /

# Apple model training
User-agent: Applebot-Extended
Disallow: /

# Meta model training
User-agent: meta-externalagent
Disallow: /

# ByteDance model training
User-agent: Bytespider
Disallow: /

Sitemap: https://gstat.taxbykk.com/sitemap.xml