# http://www.tbray.org's robots policy:
# 
# It has to be a win-win with a reasonable cost-benefit trade-off.
# That is to say, you can crawl me at my expense if you offer something
# of roughly comparable value in return, as for example well-run search 
# engines do.
#
# If you're looking for stolen term papers or brand-theft, sorry, I don't
# do that kind of thing and I don't need you spending my money to prove it.
# 
# If you're a search engine but stupidly-implemented, you'll get locked out.

User-agent: DataForSeoBot
Disallow: /

# just randomized dynamic decoration, following it apt to confuse your bot
User-agent: *
Disallow: /ongoing/goto-potd/

User-agent: Google-Extended
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: Barkrowler/0.9
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: The Knowledge AI
Disallow: /

User-agent: MJ13bot
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: bcknd.net
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: larbin_2.6.3
Disallow: /

User-agent: MS Search 4.0 Robot
Disallow: /

User-agent: QuepasaCreep
Disallow: /

User-agent: QuepasaCreep v0.9.14
Disallow: /

User-agent: QPCreep Test Rig ( We are not indexing, just testing )
Disallow: /

User-agent: QPCreep
Disallow: /

User-agent: Dattatec.com-hosting-Econonico-Menos-de-10-Dolares
Disallow: /

User-agent: Dattatec.com-hosting-Econonico-Menos-de-10-Dolares (security_at_dattatec_dot_com)
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-Agent: sitecheck.internetseer.com
Disallow: /

User-Agent: HTTrack
Disallow: /

User-Agent: MSIECrawler
Disallow: /

User-Agent: aipbot
Disallow: /

User-Agent: aipbot/1.0
Disallow: /

User-agent: Hatena Antenna
Disallow: /

User-agent: noxtrumbot/1.0
Disallow: /

User-agent: noxtrumbot/1.0 (crawler@noxtrum.com)
Disallow: /

User-agent: MegaIndex.ru/2.0
Disallow: /

User-agent: SemrushBot
Disallow: /