robots.txt
See which AI crawlers other sites block →# robots.txt - www.unesco.org
# Last updated: 2026-10-13 (corrected structure)
# Contact: digital.com@unesco.org
#
# As a condition of accessing this website, you agree to abide by the following
# content signals:
# (a) If a Content-Signal = yes, you may collect content for the corresponding use.
# (b) If a Content-Signal = no, you may not collect content for the corresponding use.
# (c) If the website operator does not include a Content-Signal for a
# corresponding use, the website operator neither grants nor restricts
# permission via Content-Signal with respect to the corresponding use.
# The content signals and their meanings are:
# - search: building a search index and providing search results (e.g., returning
# hyperlinks and short excerpts from your website's contents). Search does not
# include providing AI-generated search summaries.
# - ai-input: inputting content into one or more AI models (e.g., retrieval
# augmented generation, grounding, or other real-time taking of content for
# generative AI search answers).
# - ai-train: training or fine-tuning AI models.
# - use: how AI systems may consume the content (immediate, reference, or full).
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS BY UNESCO IN RESPECT OF THE MATERIALS MADE AVAILABLE ON THIS SITE,
# INCLUDING ITS RIGHTS AS MAKER OF ITS DATABASES. THEY APPLY WHETHER OR NOT
# UNESCO'S TERMS OF USE HAVE BEEN ACCEPTED. NOTHING IN THIS FILE CONSTITUTES
# A WAIVER, EXPRESS OR IMPLIED, OF THE PRIVILEGES AND IMMUNITIES OF UNESCO.
#
# UNESCO content may not be used to train, pre-train, fine-tune or calibrate
# AI models without prior authorisation from UNESCO.
# Requests: digital.com@unesco.org
# =============================================================================
# SECTION 1: AI TRAINING BOTS - FULLY BLOCKED
# These bots are explicitly blocked from all crawling for AI training purposes.
# =============================================================================
User-agent: Amazonbot
User-agent: Applebot-Extended
User-agent: Bytespider
User-agent: CCBot
User-agent: ClaudeBot
User-agent: FacebookBot
User-agent: GPTBot
User-agent: Meta-ExternalAgent
User-agent: omgili
User-agent: omgilibot
User-agent: DeepSeekBot
User-agent: ERNIEBot
User-agent: DoubaoBot
User-agent: ChatGLM-Spider
User-agent: cohere-training-data-crawler
User-agent: MistralAI-Training
User-agent: AI2Bot
# NOTE: FacebookBot is NOT facebookexternalhit.
# facebookexternalhit generates link previews when UNESCO
# content is shared on Facebook, Instagram or WhatsApp.
# It must never be blocked.
Disallow: /
# =============================================================================
# SECTION 2: ALL OTHER AGENTS
# Single group for all remaining crawlers including AI search bots.
# No separate groups needed - all follow the same rules below.
# =============================================================================
User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no,use=reference
License: https://www.unesco.org/en/terms-use
# Allow static assets (CSS, JS, images) for proper rendering
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Block system directories and files
Disallow: /core/
Disallow: /profiles/
Disallow: /README.txt
Disallow: /web.config
# Block generated CSS/JS, temp and private uploads (root + all sub-sites)
# Public content in /files/ (images, PDFs, documents) remains allowed.
Disallow: /files/css/
Disallow: /files/js/
Disallow: /files/tmp/
Disallow: /files/private/
Disallow: /*/files/css/
Disallow: /*/files/js/
Disallow: /*/files/tmp/
Disallow: /*/files/private/
# Block maps and grid views to prevent duplicate content
Disallow: /map
Disallow: /grid
Disallow: /*/map
Disallow: /*/grid
# Block admin and user account pages (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
Disallow: /media/oembed
Disallow: /*/media/oembed
# Block admin and user account pages (with index.php)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /index.php/media/oembed
Disallow: /index.php/*/media/oembed
# Block search and explore pages (all levels via wildcards)
# Wildcards cover root + all language subfolders (en, fr, es, ru, ar, zh, pt, ja, uz, id, uk, th, it, vi, ...)
# No need to add new language prefixes when adding subfolder sites.
Disallow: /search
Disallow: /search/
Disallow: /search?
Disallow: /explore/
Disallow: /*/search
Disallow: /*/search/*
Disallow: /*/search?
Disallow: /*/explore/*
# Block search via index.php (query parameter variant)
Disallow: /index.php/search?
Disallow: /index.php/*/search?
# Block event iCal feeds (root level)
Disallow: */event/*/ical
Disallow: /event/*/ical
Disallow: /en/event/*/ical
# Block query parameters that create duplicate content
Disallow: /*?*f[0]=*
Disallow: /*?*f0=*
Disallow: /*?*date_from=*
Disallow: /*?f[*
Disallow: /*&f[*
Disallow: /*?f%5B0*
Disallow: /*&f%5B1*
Disallow: *?hub=*
Disallow: *?page*
Disallow: *?fq*
Disallow: *?country*
# Block TSPD paths (restricted to directory structure only)
Disallow: /TSPD/
Disallow: /*/TSPD/
# Block lightbox paths
Disallow: */lightbox*
# Sitemap location
Sitemap: https://www.unesco.org/sitemap.xml
Disallow: /*?f[*
Disallow: /*&f[*
Disallow: /*?f%5B0*
Disallow: /*&f%5B1*
Disallow: /search
Disallow: /en/search
Disallow: /en/search/*
Disallow: /en/explore/*
Disallow: /event/*/ical
Disallow: /en/event/*/ical
Disallow: /biennaleluanda/2021/*?f[*
Disallow: /biennaleluanda/2021/*&f[*
Disallow: /biennaleluanda/2021/*?f%5B0*
Disallow: /biennaleluanda/2021/*&f%5B1*
Disallow: /biennaleluanda/2021/search
Disallow: /biennaleluanda/2021/en/search
Disallow: /biennaleluanda/2021/en/search/*
Disallow: /biennaleluanda/2021/en/explore/*
Disallow: /biennaleluanda/2021/event/*/ical
Disallow: /biennaleluanda/2021/en/event/*/ical
Disallow: /fr/search
Disallow: /fr/search/*
Disallow: /fr/explore/*
Disallow: /fr/event/*/ical
Disallow: /biennaleluanda/2021/fr/search
Disallow: /biennaleluanda/2021/fr/search/*
Disallow: /biennaleluanda/2021/fr/explore/*
Disallow: /biennaleluanda/2021/fr/event/*/ical
Disallow: /es/search
Disallow: /es/search/*
Disallow: /es/explore/*
Disallow: /es/event/*/ical
Disallow: /biennaleluanda/2021/es/search
Disallow: /biennaleluanda/2021/es/search/*
Disallow: /biennaleluanda/2021/es/explore/*
Disallow: /biennaleluanda/2021/es/event/*/ical
Disallow: /ru/search
Disallow: /ru/search/*
Disallow: /ru/explore/*
Disallow: /ru/event/*/ical
Disallow: /biennaleluanda/2021/ru/search
Disallow: /biennaleluanda/2021/ru/search/*
Disallow: /biennaleluanda/2021/ru/explore/*
Disallow: /biennaleluanda/2021/ru/event/*/ical
Disallow: /ar/search
Disallow: /ar/search/*
Disallow: /ar/explore/*
Disallow: /ar/event/*/ical
Disallow: /biennaleluanda/2021/ar/search
Disallow: /biennaleluanda/2021/ar/search/*
Disallow: /biennaleluanda/2021/ar/explore/*
Disallow: /biennaleluanda/2021/ar/event/*/ical
Disallow: /zh/search
Disallow: /zh/search/*
Disallow: /zh/explore/*
Disallow: /zh/event/*/ical
Disallow: /biennaleluanda/2021/zh/search
Disallow: /biennaleluanda/2021/zh/search/*
Disallow: /biennaleluanda/2021/zh/explore/*
Disallow: /biennaleluanda/2021/zh/event/*/ical
Disallow: /pt/search
Disallow: /pt/search/*
Disallow: /pt/explore/*
Disallow: /pt/event/*/ical
Disallow: /biennaleluanda/2021/pt/search
Disallow: /biennaleluanda/2021/pt/search/*
Disallow: /biennaleluanda/2021/pt/explore/*
Disallow: /biennaleluanda/2021/pt/event/*/ical
Disallow: /ja/search
Disallow: /ja/search/*
Disallow: /ja/explore/*
Disallow: /ja/event/*/ical
Disallow: /biennaleluanda/2021/ja/search
Disallow: /biennaleluanda/2021/ja/search/*
Disallow: /biennaleluanda/2021/ja/explore/*
Disallow: /biennaleluanda/2021/ja/event/*/ical
Disallow: /biodiversity/portal/*?f[*
Disallow: /biodiversity/portal/*&f[*
Disallow: /biodiversity/portal/*?f%5B0*
Disallow: /biodiversity/portal/*&f%5B1*
Disallow: /biodiversity/portal/search
Disallow: /biodiversity/portal/en/search
Disallow: /biodiversity/portal/en/search/*
Disallow: /biodiversity/portal/en/explore/*
Disallow: /biodiversity/portal/event/*/ical
Disallow: /biodiversity/portal/en/event/*/ical
Disallow: /biodiversity/portal/fr/search
Disallow: /biodiversity/portal/fr/search/*
Disallow: /biodiversity/portal/fr/explore/*
Disallow: /biodiversity/portal/fr/event/*/ical
Disallow: /biodiversity/portal/es/search
Disallow: /biodiversity/portal/es/search/*
Disallow: /biodiversity/portal/es/explore/*
Disallow: /biodiversity/portal/es/event/*/ical
Disallow: /biodiversity/portal/ru/search
Disallow: /biodiversity/portal/ru/search/*
Disallow: /biodiversity/portal/ru/explore/*
Disallow: /biodiversity/portal/ru/event/*/ical
Disallow: /biodiversity/portal/ar/search
Disallow: /biodiversity/portal/ar/search/*
Disallow: /biodiversity/portal/ar/explore/*
Disallow: /biodiversity/portal/ar/event/*/ical
Disallow: /biodiversity/portal/zh/search
Disallow: /biodiversity/portal/zh/search/*
Disallow: /biodiversity/portal/zh/explore/*
Disallow: /biodiversity/portal/zh/event/*/ical
Disallow: /biodiversity/portal/pt/search
Disallow: /biodiversity/portal/pt/search/*
Disallow: /biodiversity/portal/pt/explore/*
Disallow: /biodiversity/portal/pt/event/*/ical
Disallow: /biodiversity/portal/ja/search
Disallow: /biodiversity/portal/ja/search/*
Disallow: /biodiversity/portal/ja/explore/*
Disallow: /biodiversity/portal/ja/event/*/ical
Disallow: /creativity/*?f[*
Disallow: /creativity/*&f[*
Disallow: /creativity/*?f%5B0*
Disallow: /creativity/*&f%5B1*
Disallow: /creativity/search
Disallow: /creativity/en/search
Disallow: /creativity/en/search/*
Disallow: /creativity/en/explore/*
Disallow: /creativity/event/*/ical
Disallow: /creativity/en/event/*/ical
Disallow: /creativity/fr/search
Disallow: /creativity/fr/search/*
Disallow: /creativity/fr/explore/*
Disallow: /creativity/fr/event/*/ical
Disallow: /creativity/es/search
Disallow: /creativity/es/search/*
Disallow: /creativity/es/explore/*
Disallow: /creativity/es/event/*/ical
Disallow: /creativity/ru/search
Disallow: /creativity/ru/search/*
Disallow: /creativity/ru/explore/*
Disallow: /creativity/ru/event/*/ical
Disallow: /creativity/ar/search
Disallow: /creativity/ar/search/*
Disallow: /creativity/ar/explore/*
Disallow: /creativity/ar/event/*/ical
Disallow: /creativity/zh/search
Disallow: /creativity/zh/search/*
Disallow: /creativity/zh/explore/*
Disallow: /creativity/zh/event/*/ical
Disallow: /creativity/pt/search
Disallow: /creativity/pt/search/*
Disallow: /creativity/pt/explore/*
Disallow: /creativity/pt/event/*/ical
Disallow: /creativity/ja/search
Disallow: /creativity/ja/search/*
Disallow: /creativity/ja/explore/*
Disallow: /creativity/ja/event/*/ical
Disallow: /interculturaldialogu/*?f[*
Disallow: /interculturaldialogu/*&f[*
Disallow: /interculturaldialogu/*?f%5B0*
Disallow: /interculturaldialogu/*&f%5B1*
Disallow: /interculturaldialogu/search
Disallow: /interculturaldialogu/en/search
Disallow: /interculturaldialogu/en/search/*
Disallow: /interculturaldialogu/en/explore/*
Disallow: /interculturaldialogu/event/*/ical
Disallow: /interculturaldialogu/en/event/*/ical
Disallow: /interculturaldialogu/fr/search
Disallow: /interculturaldialogu/fr/search/*
Disallow: /interculturaldialogu/fr/explore/*
Disallow: /interculturaldialogu/fr/event/*/ical
Disallow: /interculturaldialogu/es/search
Disallow: /interculturaldialogu/es/search/*
Disallow: /interculturaldialogu/es/explore/*
Disallow: /interculturaldialogu/es/event/*/ical
Disallow: /interculturaldialogu/ru/search
Disallow: /interculturaldialogu/ru/search/*
Disallow: /interculturaldialogu/ru/explore/*
Disallow: /interculturaldialogu/ru/event/*/ical
Disallow: /interculturaldialogu/ar/search
Disallow: /interculturaldialogu/ar/search/*
Disallow: /interculturaldialogu/ar/explore/*
Disallow: /interculturaldialogu/ar/event/*/ical
Disallow: /interculturaldialogu/zh/search
Disallow: /interculturaldialogu/zh/search/*
Disallow: /interculturaldialogu/zh/explore/*
Disallow: /interculturaldialogu/zh/event/*/ical
Disallow: /interculturaldialogu/pt/search
Disallow: /interculturaldialogu/pt/search/*
Disallow: /interculturaldialogu/pt/explore/*
Disallow: /interculturaldialogu/pt/event/*/ical
Disallow: /interculturaldialogu/ja/search
Disallow: /interculturaldialogu/ja/search/*
Disallow: /interculturaldialogu/ja/explore/*
Disallow: /interculturaldialogu/ja/event/*/ical
Disallow: /reports/gem-report/*?f[*
Disallow: /reports/gem-report/*&f[*
Disallow: /reports/gem-report/*?f%5B0*
Disallow: /reports/gem-report/*&f%5B1*
Disallow: /reports/gem-report/search
Disallow: /reports/gem-report/en/search
Disallow: /reports/gem-report/en/search/*
Disallow: /reports/gem-report/en/explore/*
Disallow: /reports/gem-report/event/*/ical
Disallow: /reports/gem-report/en/event/*/ical
Disallow: /reports/gem-report/fr/search
Disallow: /reports/gem-report/fr/search/*
Disallow: /reports/gem-report/fr/explore/*
Disallow: /reports/gem-report/fr/event/*/ical
Disallow: /reports/gem-report/es/search
Disallow: /reports/gem-report/es/search/*
Disallow: /reports/gem-report/es/explore/*
Disallow: /reports/gem-report/es/event/*/ical
Disallow: /reports/gem-report/ru/search
Disallow: /reports/gem-report/ru/search/*
Disallow: /reports/gem-report/ru/explore/*
Disallow: /reports/gem-report/ru/event/*/ical
Disallow: /reports/gem-report/ar/search
Disallow: /reports/gem-report/ar/search/*
Disallow: /reports/gem-report/ar/explore/*
Disallow: /reports/gem-report/ar/event/*/ical
Disallow: /reports/gem-report/zh/search
Disallow: /reports/gem-report/zh/search/*
Disallow: /reports/gem-report/zh/explore/*
Disallow: /reports/gem-report/zh/event/*/ical
Disallow: /reports/gem-report/pt/search
Disallow: /reports/gem-report/pt/search/*
Disallow: /reports/gem-report/pt/explore/*
Disallow: /reports/gem-report/pt/event/*/ical
Disallow: /reports/gem-report/ja/search
Disallow: /reports/gem-report/ja/search/*
Disallow: /reports/gem-report/ja/explore/*
Disallow: /reports/gem-report/ja/event/*/ical
Disallow: /sdg4education2030/*?f[*
Disallow: /sdg4education2030/*&f[*
Disallow: /sdg4education2030/*?f%5B0*
Disallow: /sdg4education2030/*&f%5B1*
Disallow: /sdg4education2030/search
Disallow: /sdg4education2030/en/search
Disallow: /sdg4education2030/en/search/*
Disallow: /sdg4education2030/en/explore/*
Disallow: /sdg4education2030/event/*/ical
Disallow: /sdg4education2030/en/event/*/ical
Disallow: /sdg4education2030/fr/search
Disallow: /sdg4education2030/fr/search/*
Disallow: /sdg4education2030/fr/explore/*
Disallow: /sdg4education2030/fr/event/*/ical
Disallow: /sdg4education2030/es/search
Disallow: /sdg4education2030/es/search/*
Disallow: /sdg4education2030/es/explore/*
Disallow: /sdg4education2030/es/event/*/ical
Disallow: /sdg4education2030/ru/search
Disallow: /sdg4education2030/ru/search/*
Disallow: /sdg4education2030/ru/explore/*
Disallow: /sdg4education2030/ru/event/*/ical
Disallow: /sdg4education2030/ar/search
Disallow: /sdg4education2030/ar/search/*
Disallow: /sdg4education2030/ar/explore/*
Disallow: /sdg4education2030/ar/event/*/ical
Disallow: /sdg4education2030/zh/search
Disallow: /sdg4education2030/zh/search/*
Disallow: /sdg4education2030/zh/explore/*
Disallow: /sdg4education2030/zh/event/*/ical
Disallow: /sdg4education2030/pt/search
Disallow: /sdg4education2030/pt/search/*
Disallow: /sdg4education2030/pt/explore/*
Disallow: /sdg4education2030/pt/event/*/ical
Disallow: /sdg4education2030/ja/search
Disallow: /sdg4education2030/ja/search/*
Disallow: /sdg4education2030/ja/explore/*
Disallow: /sdg4education2030/ja/event/*/ical
Disallow: /gem-report/*?f[*
Disallow: /gem-report/*&f[*
Disallow: /gem-report/*?f%5B0*
Disallow: /gem-report/*&f%5B1*
Disallow: /gem-report/search
Disallow: /gem-report/en/search
Disallow: /gem-report/en/search/*
Disallow: /gem-report/en/explore/*
Disallow: /gem-report/event/*/ical
Disallow: /gem-report/en/event/*/ical
Disallow: /gem-report/fr/search
Disallow: /gem-report/fr/search/*
Disallow: /gem-report/fr/explore/*
Disallow: /gem-report/fr/event/*/ical
Disallow: /gem-report/es/search
Disallow: /gem-report/es/search/*
Disallow: /gem-report/es/explore/*
Disallow: /gem-report/es/event/*/ical
Disallow: /gem-report/ru/search
Disallow: /gem-report/ru/search/*
Disallow: /gem-report/ru/explore/*
Disallow: /gem-report/ru/event/*/ical
Disallow: /gem-report/ar/search
Disallow: /gem-report/ar/search/*
Disallow: /gem-report/ar/explore/*
Disallow: /gem-report/ar/event/*/ical
Disallow: /gem-report/zh/search
Disallow: /gem-report/zh/search/*
Disallow: /gem-report/zh/explore/*
Disallow: /gem-report/zh/event/*/ical
Disallow: /gem-report/pt/search
Disallow: /gem-report/pt/search/*
Disallow: /gem-report/pt/explore/*
Disallow: /gem-report/pt/event/*/ical
Disallow: /gem-report/ja/search
Disallow: /gem-report/ja/search/*
Disallow: /gem-report/ja/explore/*
Disallow: /gem-report/ja/event/*/ical
Disallow: /mab/50anniversary/*?f[*
Disallow: /mab/50anniversary/*&f[*
Disallow: /mab/50anniversary/*?f%5B0*
Disallow: /mab/50anniversary/*&f%5B1*
Disallow: /mab/50anniversary/search
Disallow: /mab/50anniversary/en/search
Disallow: /mab/50anniversary/en/search/*
Disallow: /mab/50anniversary/en/explore/*
Disallow: /mab/50anniversary/event/*/ical
Disallow: /mab/50anniversary/en/event/*/ical
Disallow: /mab/50anniversary/fr/search
Disallow: /mab/50anniversary/fr/search/*
Disallow: /mab/50anniversary/fr/explore/*
Disallow: /mab/50anniversary/fr/event/*/ical
Disallow: /mab/50anniversary/es/search
Disallow: /mab/50anniversary/es/search/*
Disallow: /mab/50anniversary/es/explore/*
Disallow: /mab/50anniversary/es/event/*/ical
Disallow: /mab/50anniversary/ru/search
Disallow: /mab/50anniversary/ru/search/*
Disallow: /mab/50anniversary/ru/explore/*
Disallow: /mab/50anniversary/ru/event/*/ical
Disallow: /mab/50anniversary/ar/search
Disallow: /mab/50anniversary/ar/search/*
Disallow: /mab/50anniversary/ar/explore/*
Disallow: /mab/50anniversary/ar/event/*/ical
Disallow: /mab/50anniversary/zh/search
Disallow: /mab/50anniversary/zh/search/*
Disallow: /mab/50anniversary/zh/explore/*
Disallow: /mab/50anniversary/zh/event/*/ical
Disallow: /mab/50anniversary/pt/search
Disallow: /mab/50anniversary/pt/search/*
Disallow: /mab/50anniversary/pt/explore/*
Disallow: /mab/50anniversary/pt/event/*/ical
Disallow: /mab/50anniversary/ja/search
Disallow: /mab/50anniversary/ja/search/*
Disallow: /mab/50anniversary/ja/explore/*
Disallow: /mab/50anniversary/ja/event/*/ical
Disallow: /mil4teachers/*?f[*
Disallow: /mil4teachers/*&f[*
Disallow: /mil4teachers/*?f%5B0*
Disallow: /mil4teachers/*&f%5B1*
Disallow: /mil4teachers/search
Disallow: /mil4teachers/en/search
Disallow: /mil4teachers/en/search/*
Disallow: /mil4teachers/en/explore/*
Disallow: /mil4teachers/event/*/ical
Disallow: /mil4teachers/en/event/*/ical
Disallow: /mil4teachers/fr/search
Disallow: /mil4teachers/fr/search/*
Disallow: /mil4teachers/fr/explore/*
Disallow: /mil4teachers/fr/event/*/ical
Disallow: /mil4teachers/es/search
Disallow: /mil4teachers/es/search/*
Disallow: /mil4teachers/es/explore/*
Disallow: /mil4teachers/es/event/*/ical
Disallow: /mil4teachers/ru/search
Disallow: /mil4teachers/ru/search/*
Disallow: /mil4teachers/ru/explore/*
Disallow: /mil4teachers/ru/event/*/ical