unloop.so

mx

smtp.google.com

ns

dns2.registrar-servers.com

dns1.registrar-servers.com

caa

txt

v=spf1 include:_spf.google.com ~all

google-site-verification=1PHFQgzlnzWHr1vLg_atlsE7u1bX5LrcdjEjhhRISeM

dmarc

v=DMARC1; p=reject; pct=100; rua=mailto:re+ddfmdlfcmf5@dmarc.postmarkapp.com; aspf=r;

# Unloop robots.txt
# Updated for 2025 LLM crawler optimization and SEO best practices

# NOTE ON /pitch AND /app-demo (and why the noindex headers still matter):
# /pitch is the unlisted investor deck; /app-demo is the unlisted demo video.
# Both are disallowed in EVERY group below, because a crawler obeys only its
# most-specific matching User-agent group and ignores the "*" group entirely.
# robots.txt is only advisory, so the real de-index guarantee is the
# X-Robots-Tag: noindex header on those two paths and everything under them
# (next.config.ts), plus the <meta name="robots"> tag in each page's layout.
# Never drop those in favour of these lines. The trade-off is accepted
# knowingly: a disallowed crawler never fetches the page, so it never reads the
# noindex. If either URL ever leaks into an external link, expect a URL-only
# listing until the link is gone.
#
# ORDERING IS LOAD-BEARING: every Disallow must stay ABOVE the "Allow: /" in
# its group. Google and Bing pick the longest matching rule, so order is moot
# to them, but older first-match crawlers take the first rule that matches.
# With "Allow: /" on top, those crawlers matched it and ignored every Disallow
# beneath it, which left /api/, /admin/ and /patterns/shared/ silently
# reachable until 2026-08-21. Keep Disallow first when adding rules.

# === SEARCH ENGINES ===

# Google - Full access
User-agent: Googlebot
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /share/
Disallow: /test*
Disallow: /dev-*
Disallow: /debug-*
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Crawl-delay: 0

# Bing - Full access
User-agent: Bingbot
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /share/
Disallow: /test*
Disallow: /dev-*
Disallow: /debug-*
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Crawl-delay: 0

# === LLM CRAWLERS ===

# OpenAI GPTBot - Allow learn content with crawl delay
User-agent: GPTBot
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /share/
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Allow: /learn/
Crawl-delay: 2

# Anthropic ClaudeBot - Allow learn content with crawl delay
User-agent: ClaudeBot
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /share/
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Allow: /learn/
Crawl-delay: 2

# Anthropic AI (legacy)
User-agent: anthropic-ai
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Allow: /learn/
Crawl-delay: 2

# Common Crawl - Allow with rate limiting
User-agent: CCBot
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Allow: /learn/
Crawl-delay: 3

# Perplexity Bot - Allow learn content
User-agent: PerplexityBot
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Allow: /learn/
Crawl-delay: 2

# Google Extended (Bard/Gemini)
User-agent: Google-Extended
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /
Allow: /learn/
Crawl-delay: 1

# === OTHER BOTS ===

# Default for all other crawlers
User-agent: *
Disallow: /api/
Disallow: /patterns/shared/
Disallow: /share/
Disallow: /test*
Disallow: /dev-*
Disallow: /debug-*
Disallow: /admin/
Disallow: /pitch
Disallow: /app-demo
Allow: /

# Block aggressive crawlers/scrapers
User-agent: AhrefsBot
Disallow: /pitch
Disallow: /app-demo
Crawl-delay: 10

User-agent: SemrushBot
Disallow: /pitch
Disallow: /app-demo
Crawl-delay: 10

User-agent: DotBot
Disallow: /pitch
Disallow: /app-demo
Crawl-delay: 10

# === SITEMAP ===
Sitemap: https://www.unloop.so/sitemap.xml