# no caching
User-agent: ia_archiver
Disallow: /

User-agent: R6_FeedFetcher
Disallow: /

User-agent: R6_CommentReader
Disallow: /

User-agent: Yandex/*
Disallow: /

User-agent: Yandex
Disallow: /

# your spidering frequency sucks
User-agent: Baiduspider
Disallow: /

# this old stuff is mostly useless and has bad titles
User-agent: *
Disallow: /notaweblog/2006/
Disallow: /notaweblog/2005/
Disallow: /notaweblog/2004/
Disallow: /notaweblog/2003/
Disallow: /notaweblog/2002/
Disallow: /notaweblog/2001/
# don't crawl pages of pages
Disallow: /tmp/
Disallow: /links/
Disallow: /photos/
Disallow: /pages/
Disallow: /search/
Disallow: /tags/
Disallow: /2001/
Disallow: /2002/
Disallow: /2003/
Disallow: /2004/
Disallow: /2005/
Disallow: /2006/
Disallow: /2007/
Disallow: /2008/
Disallow: /2009/
Disallow: /2010/
Disallow: /2011/