robots.txt
See which AI crawlers other sites block →###############################################################################
# POLICY INTENT:
# - Traditional search (Bingbot, Applebot, etc.) is permitted only on content
# EBSCO has rights to expose for indexing (research-starters and articles).
# Googlebot is the exception: allowed on all content paths including
# research-starters, articles, and knowledge; Disallow /api/ only.
# Google-Extended is denied site-wide.
# - AI training, RAG/grounding, AI answers, generative previews = LICENSABLE.
# Public crawling is NOT an authorized acquisition path for those uses.
# Right-cleared content (e.g research-starters) can be allowed for AI usecases with
# citations/ clickbacks potential.
# - Publisher/licensed content (e.g. /articles/ or /knowledge/) is DENY-BY-DEFAULT until an
# explicit rights-cleared path or generated rule exists (right-aware url architecture).
# - robots.txt states policy; WAF/bot-management enforces non-compliant traffic.
###############################################################################
###############################################################################
# BASELINE - ALLOW (Research Starters), DENY (Articles, Topic Pages)
###############################################################################
User-agent: *
# Content-Signal is a preference declaration (Cloudflare Content Signals).
# It does NOT replace User-agent Allow/Disallow rules.
# Named User-agent groups replace this * group; they must repeat Content-Signal based on usecases.
Content-Signal: search=yes, ai-input=no, ai-train=no
# Default:
# - EBSCO-owned discovery content may be crawled for traditional search-style indexing
# by compliant crawlers that use this group.
# - Publisher content denied until rights-cleared architecture exists.
Allow: /
Disallow: /articles/
Disallow: /knowledge/
Disallow: /api/
###############################################################################
# GOOGLE - Googlebot ALLOW (all content except /api/), Google-Extended DENY (all)
###############################################################################
User-agent: Googlebot
User-agent: Googlebot-IA
User-agent: Googlebot-Image
User-agent: Googlebot-Mobile
User-agent: Googlebot-News
User-agent: Googlebot-Video
User-agent: Google Favicon
User-agent: Google Images
User-agent: Google Scholar
User-agent: Google Videos
User-agent: Google-Web-Preview
User-agent: Feedfetcher-Google
User-agent: Google-Agent
User-agent: Google-CWS
User-agent: Google-GeminiNotebook
User-agent: Google-NotebookLM
User-agent: GoogleMessages
User-agent: Google-Pinpoint
User-agent: GoogleProducer
User-agent: Google-Read-Aloud
User-agent: google-speakr
User-agent: Google-Site-Verification
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/
User-agent: Google-Extended
Content-Signal: search=yes, ai-input=no, ai-train=no
Disallow: /
#####################################################################################
# SEARCH ENGINES CRAWLERS - ALLOW (Research Starters, Articles), DENY (Topic Pages)
#####################################################################################
# Known Agents Search Engine Crawlers union WAF search_engine category.
# Path policy: Allow: / (includes /articles/); Disallow: /knowledge/ and /api/.
# WAF blocks search_engine, page_preview, and named labels on those two paths
# (Googlebot is exempt on /knowledge/; see GOOGLE section).
# Site-wide denies (next group) based on SEO team recommendations
#####################################################################################
User-agent: AdIdxBot
User-agent: Alexa Archive
User-agent: alexa site audit
User-agent: AlexandriaOrgBot
User-agent: Algolia
User-agent: Algolia Crawler
User-agent: Applebot
User-agent: AppEngine
User-agent: Atom Feed Robot
User-agent: Baiduspider
User-agent: Baiduspider-render
User-agent: Bingbot
User-agent: cludo.com bot
User-agent: coccoc
User-agent: coccocbot
User-agent: coccocbot-image
User-agent: coccocbot-web
User-agent: Coveo Bot
User-agent: Coveobot
User-agent: crawler.freespoke.com
User-agent: Crawlson
User-agent: DataForSeoBot
User-agent: Daum
User-agent: DocSpotBot
User-agent: DuckDuckBot
User-agent: DuckDuckGo-Favicons-Bot
User-agent: Fastwebcrawler
User-agent: FindFiles.net
User-agent: FindITAnswersbot
User-agent: Freespoke
User-agent: FreespokeCrawler
User-agent: fyndbot
User-agent: FyndSearchEngine-Crawler
User-agent: FyndSearchEngine-ReCrawler
User-agent: GeedoProductSearch
User-agent: Gigabot
User-agent: Greppr Web Crawler
User-agent: HaosouSpider
User-agent: Hype Machine
User-agent: IbouBot
User-agent: intelx.io_bot
User-agent: Jambot
User-agent: Jobiglo-Bot
User-agent: Jooblebot
User-agent: Kagibot
User-agent: Level9SearchBot
User-agent: Linespider
User-agent: Luxxlebot
User-agent: lyonl
User-agent: lyonl-crawler
User-agent: MagiBot
User-agent: Marginalia Search
User-agent: Mars Finder
User-agent: Mojeek
User-agent: MojeekBot
User-agent: MotoMinerBot
User-agent: MRGbot
User-agent: MSN
User-agent: msnbot
User-agent: msnbot-media
User-agent: mwmbl
User-agent: Neevabot
User-agent: NestDaddyBot
User-agent: NewsNow
User-agent: Notabot
User-agent: Petalbot
User-agent: Qwantify
User-agent: Scrubby
User-agent: search.marginalia.nu
User-agent: SemanticScholarBot
User-agent: SiteSearch360
User-agent: Slurp
User-agent: SMTnetPMBot
User-agent: SokoCrawler
User-agent: stepstoneCrawlBot
User-agent: Teoma
User-agent: WepchSearchEngine
User-agent: Y!J-BRZ/YATSHA crawler
User-agent: Yahoo
User-agent: YahooJapan
User-agent: YahooCacheSystem
User-agent: Yandex
User-agent: YandexBot
User-agent: YandexImages
User-agent: YandexMobileBot
User-agent: YandexRenderResourcesBot
User-agent: YandexVertis
User-agent: Yisouspider
User-agent: ZoomInfo
User-agent: ZumBot
Content-Signal: search=yes, ai-input=no, ai-train=no
Allow: /
Disallow: /knowledge/
Disallow: /api/
# Block Recommendations - site-wide Disallow (SEO team);
User-agent: 360Spider
User-agent: Sogou
User-agent: Yeti
User-agent: SeznamBot
User-agent: TrendictionBot
User-agent: SeekportBot
User-agent: Dataprovider.com
User-agent: Funnelback
User-agent: TinEye
User-agent: TinEye-bot
User-agent: TinEye-bot-live
User-agent: TinEye-Web
User-agent: yacybot
User-agent: Qwantbot
User-agent: Qwantbot-official
Disallow: /
###############################################################################
# AI-DATA-PROVIDER - DENY (ALL Content Paths)
###############################################################################
# AI governance notes:
# - Inventory: Known Agents AI Data Providers.
# - WAF blocks matching crawlers via named Bot Control labels and User-Agent regex
# - Public crawling is NOT an authorized acquisition path for these usecases.
###############################################################################
# Known Agents - AI Data Providers
User-agent: ApifyBot
User-agent: ApifyWebsiteContentCrawler
User-agent: Bravebot
User-agent: Brightbot
User-agent: CragCrawler
User-agent: Crawlspace
User-agent: Diffbot
User-agent: ExaBot
User-agent: FirecrawlAgent
User-agent: HenkBot
User-agent: Mozilla-Tabstack
User-agent: Querit-SearchBot
User-agent: QueritBot
User-agent: ShapBot
User-agent: Shapbot
User-agent: TavilyBot
User-agent: Terra Cotta
User-agent: TerraCotta
User-agent: YouBot
User-agent: AwarioRssBot
User-agent: AwarioSmartBot
User-agent: Awario
User-agent: AgentDataBot
User-agent: BixelBot
User-agent: CloudflareBrowserRenderingCrawler
User-agent: DesearchBot
User-agent: Keenable-User
User-agent: KeenableBot
User-agent: ScryBot
User-agent: SofyaBot
User-agent: TalarionFrontierCrawler
User-agent: artsdata-crawler
User-agent: Quartr Crawler
Disallow: /
#############################################################################################
# TRAIN-AI-AND-SCRAPER - DENY (ALL Content Paths)
#############################################################################################
# AI governance notes:
# - Inventory: Known Agents AI Data Scrapers and Bot Control intent:train_ai.
# - WAF blocks in-scope requests with intent:train_ai, plus named labels / UA regex
# - Public crawling is NOT authorized for these usecases.
#############################################################################################
# Google AI (Google-Extended is in the GOOGLE section above)
User-agent: GoogleOther
User-agent: Google-CloudVertexBot
User-agent: CloudVertexBot
Disallow: /
# Apple AI
User-agent: Applebot-Extended
Disallow: /
# OpenAI
User-agent: GPTBot
Disallow: /
# Anthropic
User-agent: ClaudeBot
User-agent: anthropic-ai
Disallow: /
# Meta
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
Disallow: /
# ByteDance
User-agent: Bytespider
User-agent: Doubaobot
User-agent: imageSpider
Disallow: /
# Amazon
User-agent: Amazonbot
User-agent: bedrockbot
Disallow: /
# Cohere
User-agent: Cohere
User-agent: cohere-ai
User-agent: CohereBot
User-agent: cohere-training-data-crawler
Disallow: /
# Known Agents - remaining AI Data Scrapers
# User-agent: Spider is longest-match only; Yisouspider, Baiduspider, 360Spider,
# Bytespider, ChatGLM-Spider, and imageSpider use their own User-agent groups.
User-agent: AI2Bot
User-agent: Ai2Bot-Dolma
User-agent: CCBot
User-agent: ChatGLM-Spider
User-agent: Cotoyogi
User-agent: Datenbank Crawler
User-agent: DeepSeekBot
User-agent: ERNIEBot
User-agent: ICC-Crawler
User-agent: Kangaroo Bot
User-agent: KimiBot
User-agent: KimiCrawler
User-agent: laion-huggingface-processor
User-agent: LCC
User-agent: micro-crawl
User-agent: MistralBot
User-agent: MoonshotBot
User-agent: MoonSpider
User-agent: netEstate Imprint Crawler
User-agent: omgili
User-agent: PanguBot
User-agent: QwenBot
User-agent: Reflectionbot
User-agent: SalamandraVLM
User-agent: SBIntuitionsBot
User-agent: Spider
User-agent: Timpibot
User-agent: VelenPublicWebCrawler
User-agent: webzio-extended
User-agent: BeansLLM-CorpusBot
User-agent: Hunyuan
User-agent: ImageBot
User-agent: recraft-image-dataset
User-agent: YiBot
User-agent: ElevenlabsCrawler
Disallow: /
#################################################################################################
# AI SEARCH / ANSWER-INDEX CRAWLERS - ALLOW (Research Starters), DENY (Articles and Topic Pages)
#################################################################################################
# AI governance notes:
# - Named crawlers commonly used for AI search indexing (not exhaustive).
# - WAF blocks on /articles/, /knowledge/, and /api/ when bot control labels available
# intent:search or a named AI-search bot. Research Starters are not matched.
# - Licensed AI access should use EBSCO APIs, feeds, discovery MCP, etc.
# - Public crawling is NOT an authorized acquisition path for these usecases.
# - Only allow for rights-cleared content
#################################################################################################
User-agent: PerplexityBot
User-agent: DuckAssistBot
User-agent: OAI-SearchBot
User-agent: Meta-WebIndexer
User-agent: Devin
User-agent: Claude-SearchBot
User-agent: Amzn-SearchBot
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /articles/
Disallow: /knowledge/
Disallow: /api/
######################################################################################
# USER-INITIATED AI ANSWER GENERATION WITH CITATIONS
# - ALLOW (Research Starters), DENY (Articles and Topic Pages)
######################################################################################
# AI governance notes:
# - Named crawlers commonly used for AI assist / RAG (not exhaustive).
# - WAF blocks on /articles/, /knowledge/, and /api/ when bot control labels available
# intent:ai_assistant or a named AI-assist bot. Research Starters are not matched.
# - Restrict access to LICENSABLE content for real-time, grounded AI answer generation and
# citations/ referrals for user triggered retrieval or query.
# - Only Allow for rights-cleared content
######################################################################################
User-agent: ChatGPT-User
User-agent: Claude-User
User-agent: Perplexity-User
User-agent: Amzn-User
User-agent: Meta-ExternalFetcher
User-agent: MistralAI-User
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /articles/
Disallow: /knowledge/
Disallow: /api/
##############################################################################
# OPERATIONAL CRAWLERS (Supports Business Usecases)
# - Allow (all content paths except /knowledge/ and /api/)
##############################################################################
User-agent: SemrushBot
User-agent: SemrushSiteAuditBot
Content-Signal: search=yes, ai-input=no, ai-train=no
Allow: /
Disallow: /knowledge/
Disallow: /api/
################################################################################################
# LOW-VALUE / UNWANTED CRAWLERS - DENY (All Content Paths)
# site-wide Disallow specific bots based on SEO team recommendations across different categories
# User-agent: * restrictions applies to any unmatched bot
################################################################################################
# Block Recommendations - Category: SEO
User-agent: DotBot
User-agent: MJ12bot
User-agent: ClearscopeBot
User-agent: SiteCheck
User-agent: SeobilityBot
User-agent: Mediatoolkitbot
User-agent: marketgoobot
User-agent: WebCEO
Disallow: /
# Block Recommendations - Category: Scraping framework
# This list is limited but WAF blocks when labeled as scraping_framework category.
User-agent: Scrapy
User-agent: harvester
User-agent: Hatena
Disallow: /
# Block Recommendations - Category: Archiver
User-agent: heritrix
Disallow: /
# Block Recommendations - Category: Content fetcher
User-agent: Blogtrottr
User-agent: Embedly
User-agent: Feedly
User-agent: Innologica
User-agent: LivelapBot
User-agent: XenuLinkSleuth
User-agent: NaverPreview
User-agent: AASA-Bot
User-agent: OpenRSS
User-agent: Censys
User-agent: rss2tg
User-agent: WebCopier
User-agent: ProtoPage
User-agent: MixrankBot
User-agent: WordPress-Namecheap
User-agent: Nooshub
User-agent: Scour
Disallow: /
# Block Recommendations - Category: Monitoring
User-agent: MontasticMonitor
User-agent: Monsido
User-agent: BrandVerity
User-agent: Foregenix
User-agent: Pandalytics
User-agent: WPMU DEV Broken Link Checker
User-agent: Miniflux
User-agent: WatchBot
User-agent: ActiveComply
User-agent: DotComMonitor
User-agent: ContentKing
Disallow: /
###############################################################################
# SITEMAPS
###############################################################################
Sitemap: https://a101334.sitemaphosting3.com/3981381/sitemap_3981381.xml