User-agent: *
# Contains an infinite number of "pages":
Disallow: /calendar/
# These are obsolete:
Disallow: /text/netexpres-2.4.7.html
Disallow: /text/netexpres-2.4.0-test4.html
Disallow: /text/netexpres-2.3.39.html
# These have moved:
Disallow: /cam/
Disallow: /log/
Disallow: /staff/
# My darcs repos have been deprecated in favour of git:
Disallow: /darcs/
Disallow: /darcsweb
# No need to advertise my locally built packages:
Disallow: /debian/
# Nor my local git repo:
Disallow: /git/
Disallow: /gitweb
#
Disallow: /text/handel/
#
User-agent: TurnitinBot
Disallow: /
#
User-agent: Girafabot
Disallow: /
#
User-agent: NPBot
Disallow: /
#
User-agent: HostHarvest
Disallow: /
#
# I don't want aggregators to advertise next to/around my content:
User-agent: Bloglines
Disallow: /log.rss
#
User-agent: Feedster Crawler
Disallow: /log.rss
#
User-agent: Syndic8
Disallow: /log.rss
#
User-agent: everyfeed-spider
Disallow: /log.rss