robots.txt
See which AI crawlers other sites block →#
# robots.txt for setlist.fm
# adapted from for http://www.wikipedia.org/ and friends
#
# Please note: There are a lot of pages on this site, and there are
# some misbehaved spiders out there that go _way_ too fast. If you're
# irresponsible, your access to the site may be blocked.
#
# advertising-related bots:
# User-agent: Mediapartners-Google*
# Disallow: /
# Crawlers that are kind enough to obey, but which we'd rather not have
# unless they're feeding search engines.
User-agent: UbiCrawler
Disallow: /
User-agent: DOC
Disallow: /
User-agent: Zao
Disallow: /
# Some bots are known to be trouble, particularly those designed to copy
# entire sites. Please obey robots.txt.
User-agent: sitecheck.internetseer.com
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: MSIECrawler
Disallow: /
User-agent: SiteSnagger
Disallow: /
User-agent: WebStripper
Disallow: /
User-agent: WebCopier
Disallow: /
User-agent: Fetch
Disallow: /
User-agent: Offline Explorer
Disallow: /
User-agent: Teleport
Disallow: /
User-agent: TeleportPro
Disallow: /
User-agent: WebZIP
Disallow: /
User-agent: linko
Disallow: /
User-agent: HTTrack
Disallow: /
User-agent: Microsoft.URL.Control
Disallow: /
User-agent: Xenu
Disallow: /
User-agent: larbin
Disallow: /
User-agent: libwww
Disallow: /
User-agent: ZyBORG
Disallow: /
User-agent: Download Ninja
Disallow: /
#
# Sorry, wget in its recursive mode is a frequent problem.
# Please read the man page and use it properly; there is a
# --wait option you can use to set the delay between hits,
# for instance.
#
User-agent: wget
Disallow: /
#
# The 'grub' distributed client has been *very* poorly behaved.
#
User-agent: grub-client
Disallow: /
#
# Doesn't follow robots.txt anyway, but...
#
User-agent: k2spider
Disallow: /
#
# Hits many times per second, not acceptable
# http://www.nameprotect.com/botinfo.html
User-agent: NPBot
Disallow: /
# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /
# grapeshot often comes with 30+ crawlers at once
# User-agent: grapeshot
# Disallow: /
# msnbot should crawl a bit slower
# User-agent: msnbot
# Crawl-delay: 2
#
# Friendly, low-speed bots are welcome viewing article pages, but not
# dynamically-generated pages please.
#
# Inktomi's "Slurp" can read a minimum delay between hits; if your
# bot supports such a thing using the 'Crawl-delay' or another
# instruction, please let us know.
#
User-agent: *
Disallow: /error/
Disallow: /auth/
Disallow: /edit/
Disallow: /setlist/history/
Disallow: /api/
Disallow: /web/
Sitemap: https://sitemap.cdn.setlistfm.com/sitemap/Sitemap.xml
ads.txt
#ads.txt file for setlist.fm
google.com, pub-2565655743967084, DIRECT, f08c47fec0942fa0
indexexchange.com, 185683, DIRECT, 50b1c356f2c5c8fc
google.com, pub-1884777957953415, DIRECT, f08c47fec0942fa0
openx.com, 539478444, DIRECT, 6a698e2ec38604c6
spotxchange.com, 251064, DIRECT, 7842df1d2fe2db34
spotx.tv, 251064, DIRECT, 7842df1d2fe2db34
jwplayer.com, 6DSs7Ul1, DIRECT, 99ca9a2050fa3047
themediagrid.com, 7COYL2, DIRECT, 35d5010d7789b49d
advertising.com, 28814, RESELLER
appnexus.com, 12564, RESELLER, f5ab79cb980f11d1
pubmatic.com, 159247, RESELLER, 5d62403b186f2ace
google.com, pub-5760410923284845, RESELLER, f08c47fec0942fa0
advertising.com, 28816, RESELLER
indexexchange.com, 193162, RESELLER
pubmatic.com, 160127, RESELLER, 5d62403b186f2ace
indexexchange.com, 199932, RESELLER, 50b1c356f2c5c8fc
onetag.com, 7ac2dc69493a27c, DIRECT
lijit.com, 418348, DIRECT, fafdf38b16bf6b2b #SOVRN