planora.today

mx

ns

ns48.domaincontrol.com

ns47.domaincontrol.com

caa

txt

google-site-verification=YIMwL_UZKBnovpQwoKHpBVB4muwSYCXuD7QTLOSKRt4

dmarc

v=DMARC1; p=none;

# ---------------------------------------------------------------------------
# Planora robots.txt
# ---------------------------------------------------------------------------
# Strategy:
#   1) Block known AI/LLM training crawlers entirely (no value to us, all cost)
#   2) Block only the broadly-abusive SEO scrapers (BLEXBot). Ahrefs / Semrush
#      / Majestic / Moz are kept ALLOWED — they index the backlink and
#      keyword graphs that marketers + competitors use to find products like
#      ours, and we'd want to use them ourselves to measure our own SEO.
#   3) For everyone else, disallow authenticated + sensitive paths but allow
#      the marketing surface (landing, login, waitlist) so search indexes
#      our public pages
#   4) Sitemap pointer at the bottom
#
# This is a hint, not an enforcement layer. Well-behaved crawlers obey it;
# bad actors ignore it. Real abuse protection lives in:
#   - src/proxy.ts (in-memory IP rate-limit on /auth, /api/waitlist)
#   - src/lib/api/rate-limit.ts (DB-backed per-user limits for mutations)
#   - Supabase per-email magic-link cap
# ---------------------------------------------------------------------------

# --- Link-preview scrapers — explicit ALLOW.
#
# These render the OG card when our URL is shared in WhatsApp, Facebook
# Messenger, iMessage, Slack, Twitter, LinkedIn, etc. They are NOT the same
# bots as the AI training crawlers blocked below. The `User-agent: *` rule
# at the bottom already lets them through, but Facebook's debugger has
# historically suggested explicit allowlisting when it sees ANY 403 — this
# removes ambiguity for any liberal parser.
User-agent: facebookexternalhit
Allow: /

User-agent: facebookcatalog
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slackbot
Allow: /

User-agent: SlackBot-LinkExpanding
Allow: /

User-agent: Discordbot
Allow: /

User-agent: TelegramBot
Allow: /

User-agent: Applebot
Allow: /

# --- AI / LLM training crawlers — block entirely ---
User-agent: GPTBot
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: OAI-SearchBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: ClaudeUser
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Perplexity-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: meta-externalagent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: cohere-ai
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: Webzio-Extended
Disallow: /

# --- Abusive scraper — broadly considered too aggressive to be worth the
#     traffic it generates; safe to block. Ahrefs / Semrush / MJ12bot / DotBot
#     are intentionally NOT in this list — they index the SEO graph and
#     blocking them makes us invisible to marketers and partners.
User-agent: BLEXBot
Disallow: /

# --- Everyone else (Googlebot, Bingbot, DuckDuckBot, Slackbot, OG previews, …) ---
User-agent: *
# Keep the authenticated app, admin tools, auth callbacks, invite acceptance,
# and the JSON API out of every index. Each of these gates on a logged-in
# session anyway, but indexing the entry URLs leaks structure + invites
# spam crawlers to retry them.
Disallow: /app/
Disallow: /admin/
Disallow: /auth/
Disallow: /invite/
Disallow: /api/
# Public surface — index away.
Allow: /
Allow: /login
Allow: /waitlist

Sitemap: https://planora.today/sitemap.xml