# no caching
User-agent: ia_archiver
Disallow: /

User-agent: R6_FeedFetcher
Disallow: /
User-agent: R6_CommentReader
Disallow: /

User-agent: Yandex/*
Disallow: /
User-agent: Yandex
Disallow: /

# don't crawl pages of pages
User-Agent: *
Disallow: /tmp
Disallow: /links
Disallow: /photos
Disallow: /pages
Disallow: /search
Disallow: /tags
Disallow: /2001
Disallow: /2002
Disallow: /2003
Disallow: /2004
Disallow: /2005
Disallow: /2006
Disallow: /2007
Disallow: /2008
Disallow: /2009
Disallow: /2010
Disallow: /2011