theregister.com

mx

aspmx.l.google.com

alt1.aspmx.l.google.com

alt2.aspmx.l.google.com

aspmx4.googlemail.com

aspmx2.googlemail.com

aspmx3.googlemail.com

aspmx5.googlemail.com

ns

gerald.ns.cloudflare.com

dahlia.ns.cloudflare.com

caa

txt

v=spf1 a:news.theregister.co.uk a:smtp.sitpub.com include:_spf.google.com include:sendgrid.net -all

google-site-verification=zCDPdjsPQ5dH4FbeLCiDrVsszXxNSmWTVXURdnpbDKQ

brevo-code:ce3ec0f3b4bdceb0a4f9c560821dc50d

MS=ms80025279

brave-ledger-verification=dd8c222f46910ffc43b06a3753f120dd2a68ebc0d672f5ff6d0d74dca9101273

linkedin-site-verification=1dcbfa33-d4f8-40cc-a4c6-e1b8605496ec

tollbit-domain-verification=1f6aa92e740a7e4e01c39e45a3815104b4a3ae619b24ed01889016894cda3f96

google-site-verification=CgoF5YVzVGhTo8fSZwIO0hWZAs8t2Gy5QZHk3topJwQ

dmarc

v=DMARC1; p=quarantine; rua=mailto:bc1115f44d2e4098997d5b426aa40a3d@dmarc-reports.cloudflare.net

# =============================================================================
# robots.txt
# Last updated: 18 June 2026
#
# POLICY: Default-deny. Any crawler not explicitly allowed below is BLOCKED.
#   - Traditional search engines ............ ALLOWED
#   - AI search crawlers that cite + refer ... ALLOWED (conditional on attribution)
#   - Operational bots (ads, social, uptime) . ALLOWED
#   - AI training / scraping crawlers ........ BLOCKED (licence required)
#   - Anything else, including unknown bots .. BLOCKED by default
#
#
# AI companies: to licence our content, see https://situationpublishing.com/ai-licensing/
# Licensing enquiries: content-licensing@sitpub.com
#
# If you are an AI company and want to use our content, contact us about
# a licensing arrangement. See: https://situationpublishing.com/ai-licensing/
# =============================================================================

# ---------------------------------------------------------------------------
# Traditional search engines — ALLOWED
# ---------------------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Applebot
Allow: /


# ---------------------------------------------------------------------------
# AI search crawlers that cite and return referral traffic — ALLOWED
# These power AI search answers and link back to source content.
# Permission is conditional on continued referral traffic and attribution.
# ---------------------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

# Anthropic (Claude). User-initiated fetches, search indexing, and the
# Claude Code CLI fetching on a developer's behalf. All three honour
# robots.txt. ClaudeBot (training) stays BLOCKED below, mirroring GPTBot.
User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: claude-code
Allow: /


# ---------------------------------------------------------------------------
# Operational bots we depend on — ALLOWED
# DO NOT REMOVE. Blocking these breaks programmatic revenue and social sharing.
# ---------------------------------------------------------------------------

# Google advertising crawlers (ad serving / quality)
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

# Social link-preview / unfurl bots
User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Pinterestbot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: WhatsApp
Allow: /

# Ad verification / brand safety / contextual vendors — ALLOWED
# Clients use different verification partners, so all the major ones are listed.
# Ad ops: review periodically and add any niche vendor a client requires.
# (UA matching is case-insensitive.)

# Integral Ad Science (IAS)
User-agent: ias_crawler
Allow: /

User-agent: ias_wombles
Allow: /

User-agent: ias_admantx
Allow: /

# DoubleVerify
User-agent: DoubleVerifyBot
Allow: /

User-agent: DVBot
Allow: /

# Comscore (contextual)
User-agent: proximic
Allow: /

# Peer39 (contextual)
User-agent: peer39_crawler
Allow: /

# Oracle / Grapeshot (contextual — winding down, kept for clients still on it)
User-agent: GrapeshotCrawler
Allow: /

User-agent: grapeshot
Allow: /

# GumGum (contextual)
User-agent: GumGum
Allow: /

# HUMAN Security (brand safety)
User-agent: GlancelotBot
Allow: /

# The Trade Desk
User-agent: TTD-Content
Allow: /

# PubMatic
User-agent: PubMatic Crawler Bot
Allow: /

# Sincera (supply-chain transparency)
User-agent: SinceraSyntheticUser
Allow: /

# Amazon advertising (distinct from Amazonbot AI-training crawler, blocked below)
User-agent: AmazonAdBot
Allow: /

# Other brand-safety / ad-tech crawlers
User-agent: Leikibot
Allow: /

User-agent: SlickBot
Allow: /


# ---------------------------------------------------------------------------
# AI training / scraping crawlers — BLOCKED  (belt-and-suspenders)
# These are already blocked by the default-deny rule at the bottom; listed
# explicitly to document intent. Contact us for licensing terms.
# ---------------------------------------------------------------------------

# OpenAI (training)
User-agent: GPTBot
Disallow: /

# Anthropic (training only — Claude-User, Claude-SearchBot and claude-code
# are allowed above)
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

# Google (AI training — separate from Googlebot search indexing)
User-agent: Google-Extended
Disallow: /

User-agent: Google-CloudVertexBot
Disallow: /

# Meta
User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

# Apple (AI training — separate from standard Applebot)
User-agent: Applebot-Extended
Disallow: /

# Amazon
User-agent: Amazonbot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Common Crawl
User-agent: CCBot
Disallow: /

# Cohere
User-agent: cohere-ai
Disallow: /

# Diffbot
User-agent: Diffbot
Disallow: /

# You.com
User-agent: YouBot
Disallow: /

# DuckDuckGo AI assistant (distinct from DuckDuckBot search)
User-agent: DuckAssistBot
Disallow: /

# Timpi
User-agent: Timpibot
Disallow: /

# Omgili / Webz.io
User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

# ImagesiftBot
User-agent: ImagesiftBot
Disallow: /

# xAI / Grok
User-agent: GrokBot
Disallow: /

# Allen Institute for AI (AI2)
User-agent: AI2Bot
Disallow: /

# Firecrawl (dev-tool scraper)
User-agent: FirecrawlAgent
Disallow: /

# Scrapy-based generic crawlers
User-agent: Scrapy
Disallow: /


# ---------------------------------------------------------------------------
# DEFAULT — everything else is BLOCKED
# Any crawler not explicitly allowed above is denied. Review server logs /
# TollBit WEEKLY and add new beneficial crawlers (a new AI search engine, or a
# new ad-verification vendor a client brings) to the ALLOWED sections above.
#
# EXCEPTION: ads.txt and app-ads.txt must stay readable by ALL bots —
# programmatic demand depends on it — so they are allowed even under default-deny.
# ---------------------------------------------------------------------------
User-agent: *
Allow: /ads.txt
Allow: /app-ads.txt
Disallow: /


# ---------------------------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------------------------
Sitemap: https://www.theregister.com/sitemapindex.xml

# Google/DFP
google.com, pub-3230938694120234, DIRECT, f08c47fec0942fa0
google.com, pub-6455808816405956, DIRECT, f08c47fec0942fa0