aidiveforge.com

mx

mail.aidiveforge.com

ns

andronicus.ns.cloudflare.com

eve.ns.cloudflare.com

caa

txt

v=spf1 ip4:187.124.240.217 ~all

dmarc

v=DMARC1; p=reject; rua=mailto:dmarc@aidiveforge.com; adkim=r; aspf=r

# robots.txt for aidiveforge.com
# Optimized for AI discovery, search engines, and accessibility

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-includes/
Disallow: /wp-content/plugins/
Disallow: /wp-login.php
Disallow: /?p=
Allow: /wp-admin/admin-ajax.php
Allow: /wp-content/uploads/

# Crawl delay and request rate (be nice, but crawl actively)
Request-rate: 10/1s

# ========================================
# AI Crawlers — Explicitly Allowed
# ========================================
# These agents are encouraged to index all content for LLM training and AI discovery

# OpenAI (ChatGPT, GPT-4, web browsing feature)
User-agent: GPTBot
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php
Crawl-delay: 0

User-agent: ChatGPT-User
Allow: /
Crawl-delay: 0

# Anthropic Claude (Claude API, web feature)
User-agent: Claude-Web
Allow: /
Crawl-delay: 0

User-agent: ClaudeBot
Allow: /
Crawl-delay: 0

# Google (Search, Bard, News, Extended)
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Google-Extended
Allow: /
Crawl-delay: 0

# Perplexity AI (Search engine)
User-agent: PerplexityBot
Allow: /
Crawl-delay: 0

User-agent: PerplexityBot-Web
Allow: /
Crawl-delay: 0

# Microsoft (Bing, Copilot)
User-agent: Bingbot
Allow: /
Crawl-delay: 0

User-agent: Slurp
Allow: /

# Apple (Siri, News)
User-agent: Applebot
Allow: /
Crawl-delay: 0

# Meta (Facebook, Instagram, Ray/AI)
User-agent: facebookexternalhit
Allow: /

# Other AI/Search Services
User-agent: DuckDuckBot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Omgili
Allow: /

User-agent: Bytespider
Allow: /

# ========================================
# Sitemaps & Feeds
# ========================================

Sitemap: https://aidiveforge.com/wp-sitemap.xml

# Machine-readable feeds (not sitemaps — listing them as Sitemap:
# directives causes Search Console parse errors):
# https://aidiveforge.com/feed.json
# https://aidiveforge.com/llms-full.txt

# ========================================
# Rate Limiting by User-Agent
# ========================================
# Default crawl delay for unspecified bots (be respectful)

User-agent: *
Crawl-delay: 1
Request-rate: 5/1s

Contact: mailto:security@aidiveforge.com
Expires: 2027-10-01T04:15:01Z
Preferred-Languages: en
Canonical: https://aidiveforge.com/.well-known/security.txt
Policy: https://aidiveforge.com/privacy-policy/

# AIDiveForge ads.txt
# Contact: hello@aidiveforge.com
ownerdomain=aidiveforge.com
google.com, pub-1314511091745616, DIRECT, f08c47fec0942fa0