User-agent: W3C-checklink
Disallow: /

User-agent: peerbot
Disallow: /favicon.ico

User-agent: MSIECrawler
Disallow: /

User-agent: Googlebot-Image
Disallow: /

User-agent: psbot
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: MondoSearch
Disallow: /

User-agent: Spiderline
Disallow: /

User-agent: fusionbot
Disallow: /

User-agent: WebReaper
Disallow: /

User-agent: NPBot
Disallow: /

User-agent: *
Disallow: /download/
Disallow: /etc/
Disallow: /ftp/
Disallow: /tmp/
Disallow: /users/
Disallow: /var/
Disallow: /www/htdocs/images/
Disallow: /www/cgi-bin/
Disallow: /www/logs/
Disallow: /www/shopsite-data/
Disallow: /www/urchin/
Disallow: /www/urchin5/
Disallow: /wholesale/
Disallow: /htdocs/images/
Disallow: /htdocs/wholesale/
Disallow: /htdocs/cgi-bin/
Disallow: /htdocs/ufm/
Disallow: /styles/
Disallow: /include/
Disallow: /cgi-bin/
Disallow: /feedback.php
Disallow: /favicon.ico

# too many repeated hits, too quick
User-agent: litefinder
Disallow: /

# Yahoo. too many repeated hits, too quick
User-agent: Slurp
Disallow: /

# too many repeated hits, too quick
User-agent: Baidu
Disallow: /

Disallow: /download/
Disallow: /etc/
Disallow: /ftp/
Disallow: /tmp/
Disallow: /users/
Disallow: /var/
Disallow: /www/htdocs/images/
Disallow: /www/cgi-bin/
Disallow: /www/logs/
Disallow: /www/shopsite-data/
Disallow: /www/urchin/
Disallow: /www/urchin5/
Disallow: /wholesale/
Disallow: /htdocs/images/
Disallow: /htdocs/wholesale/
Disallow: /htdocs/cgi-bin/
Disallow: /htdocs/ufm/
Disallow: /styles/
Disallow: /include/
Disallow: /cgi-bin/
Disallow: /feedback.php
Disallow: /favicon.ico