llms.txt
Compare with other sites' llms.txt files →# llm.txt — AI crawler & model-use policy for ctobees.com
# Place this file at: https://ctobees.com/llm.txt
# Notes:
# - This is a non-standard but increasingly common convention (similar to ai.txt).
# - Keep robots.txt in sync; many bots still only read robots.txt.
# - Default stance below: allow browsing for answering/search, disallow model training & dataset building.
owner: yourcompany (contact: webmaster@ctobees.com)
updated: 2025-08-09
policy: purpose=answering,search-indexing; training=disallow; dataset-building=disallow; derivative-datasets=disallow
license: no-reuse-beyond-purpose
rate-limit: polite
# Canonical domain (avoid duplicates)
canonical: https://www.ctobees.com/
# Global allow/deny rules (mirrors robots-style paths)
allow: /
disallow: /admin/
disallow: /account/
disallow: /login/
disallow: /logout/
disallow: /checkout/
disallow: /cart/
disallow: /search
disallow: /private/
disallow: /draft/
disallow: /tmp/
disallow: /cgi-bin/
# Per-agent directives (bots commonly associated with LLMs and data collection)
# Where supported, respect the intent flags below; otherwise, treat disallow as binding.
user-agent: GPTBot
allow: /
disallow: /admin/,/account/,/login/,/logout/,/checkout/,/cart/,/search,/private/,/draft/,/tmp/,/cgi-bin/
intent: answering,search-indexing
training: disallow
user-agent: OAI-SearchBot
allow: /
disallow: /admin/,/account/,/login/,/logout/,/checkout/,/cart/,/search,/private/,/draft/,/tmp/,/cgi-bin/
intent: answering
training: disallow
user-agent: OAI-User
allow: /
disallow: /admin/,/account/,/login/,/logout/,/checkout/,/cart/,/search,/private/,/draft/,/tmp/,/cgi-bin/
intent: answering
training: disallow
user-agent: Google-Extended
# Controls content eligible for Google GenAI training/derivation (Gemini etc.).
disallow: /
user-agent: Applebot-Extended
# Controls content eligible for Apple ML training/derivation.
disallow: /
user-agent: CCBot
# Common Crawl (used by many model trainers).
disallow: /
user-agent: ClaudeBot
# Anthropic’s crawler for model training.
disallow: /
user-agent: PerplexityBot
# Used for answering; you may choose to allow answering but block training reuse.
allow: /
disallow: /admin/,/account/,/login/,/logout/,/checkout/,/cart/,/search,/private/,/draft/,/tmp/,/cgi-bin/
training: disallow
user-agent: Bytespider
# ByteDance/TikTok crawler.
disallow: /
user-agent: Amazonbot
# Amazon/AGI research.
disallow: /
user-agent: FacebookBot
user-agent: Meta-ExternalAgent
# Meta’s external agent for model data.
disallow: /
# Fallback catch-all for any unidentified AI agent
user-agent: *
allow: /
disallow: /admin/,/account/,/login/,/logout/,/checkout/,/cart/,/search,/private/,/draft/,/tmp/,/cgi-bin/
training: disallow