robots.txt
See which AI crawlers other sites block →# As a condition of accessing this website, you agree to abide by the
# following content signals:
# (a) If a content-signal = yes, you may collect content for the
# corresponding use.
# (b) If a content-signal = no, you may not collect content for the
# corresponding use.
# (c) If the website operator does not include a content signal for a
# corresponding use, the website operator neither grants nor restricts
# permission via content signal with respect to the corresponding use.
# The content signals and their meanings are:
# search: building a search index and providing search results (e.g., returning
# hyperlinks and short excerpts from your website's contents). Search
# does not include providing AI-generated search summaries.
# ai-input: inputting content into one or more AI models (e.g., retrieval
# augmented generation, grounding, or other real-time taking of
# content for generative AI search answers).
# ai-train: training or fine-tuning AI models.
# ANY RESTRICTIONS EXPRESSED VIA CONTENT SIGNALS ARE EXPRESS RESERVATIONS OF
# RIGHTS UNDER ARTICLE 4 OF THE EUROPEAN UNION DIRECTIVE 2019/790 ON COPYRIGHT
# AND RELATED RIGHTS IN THE DIGITAL SINGLE MARKET.
# All AI Crawler Rules
User-agent: *
Content-signal: search=yes, ai-train=no, ai-input=no
Allow: /
User-agent: Amazonbot
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: Claude-User
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Claude-SearchBot
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: Google-CloudVertexBot
Disallow: /
User-agent: GoogleOther
Disallow: /
User-agent: GPTBot
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: cohere-training-data-crawler
Disallow: /
User-agent: meta-externalagent
Disallow: /
User-agent: Meta-ExternalFetcher
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Perplexity-User
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: omgili
Disallow: /
User-agent: omgilibot
Disallow: /
User-agent: AI2Bot
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: Webzio-Extended
Disallow: /
User-agent: VelenPublicWebCrawler
Disallow: /
User-agent: ICC-Crawler
Disallow: /
User-agent: PanguBot
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: img2dataset
Disallow: /
User-agent: Crawlspace
Disallow: /
User-agent: FriendlyCrawler
Disallow: /
User-agent: Kangaroo Bot
Disallow: /
User-agent: iaskspider/2.0
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Sidetrade indexer bot
Disallow: /
User-agent: DuckAssistBot
Disallow: /
User-agent: PhindBot
Disallow: /
User-agent: MistralAI-User
Disallow: /
User-agent: Amzn-SearchBot
Disallow: /
User-agent: Amzn-User
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
# All crawler rules
# Questions: webmaster@dagbladet.no
User-agent: *
Disallow: *.cgi
Disallow: /2009/06/18/nyheter/iran/valg/innenriks/politikk/6784480/
Disallow: /2009/09/21/nyheter/innenriks/giftdrap/8225100/
Disallow: /2011/09/01/tema/barn/familie/oppvekst/foreldre/17916046/
Disallow: /2011/12/05/kultur/debatt/debattinnlegg/anders_behring_breivik/ideologi/19288301/
Disallow: /2013/07/12/nyheter/innenriks/giftmorder/terje_wiik/28160672/
Disallow: /2013/09/29/nyheter/utenriks/terror/kenya/westgate/29515561/
Disallow: /tag/saera_khan
Disallow: /tag/terje_wiik
Allow: /eavis/_next/
Disallow: /eavis/
Disallow: /youtube-player/
Disallow: /video/playlist/
Disallow: /video/popular/
Disallow: /video/
Disallow: /emne/terje%20erlid
Allow: /app/sanity-cdn/images/
Allow: /app/sanity-next-app/
Allow: /app/raven-frontend/
Allow: /app/showcasr/
Allow: /app/cyclops-router-api/api/v1/
Allow: /app/jw-api-proxy/sitemap/index/dagbladet.xml
Disallow: /app/
Allow: /video/*/*
Disallow: /*.pdf
# All sitemaps
Sitemap: https://www.dagbladet.no/sitemapindex.xml
Sitemap: https://www.dagbladet.no/app/jw-api-proxy/sitemap/index/dagbladet.xml
ads.txt
OWNERDOMAIN=aller.no, DIRECT
google.com, pub-1599987413866196, DIRECT, f08c47fec0942fa0
google.com, pub-3166474279256625, DIRECT, f08c47fec0942fa0
google.com, pub-9374486642976031, DIRECT, f08c47fec0942fa0
adform.com, 34, DIRECT
adform.com, 2544, RESELLER
rubiconproject.com, 15770, DIRECT, 0bfd66d529a55807
rubiconproject.com, 20284, DIRECT, 0bfd66d529a55807
netricsales.com, 15770, DIRECT
appnexus.com, 11187, RESELLER
google.com, pub-5002094515318070, RESELLER, f08c47fec0942fa0
rubiconproject.com, 16474, RESELLER, 0bfd66d529a55807
improvedigital.com, 1752, DIRECT
omnicommediagroup.com, A09, RESELLER
ipgmediabrands.dk, N008, RESELLER
pubmatic.com, 166453, DIRECT, 5d62403b186f2ace
app.readpeak.com, 1703, DIRECT
indexexchange.com, 214087, DIRECT
indexexchange.com, 214890, DIRECT, 50b1c356f2c5c8fc
amxrtb.com, 105199957, DIRECT
adform.com, 2865, RESELLER