# Robots.txt file from http://www.searchengineworld.com
#  
# Built from text file 
http://info.webcrawler.com/mak/projects/robots/active/all.txt
# 
# This restricts access to only known and registered robots. 
#

User-agent: *
Disallow: /cgi-bin
Disallow: /cp
Disallow: /modlogan
Disallow: /mone
Disallow: /webalizer
Disallow: /wp-admin

User-agent: heritrix
Disallow: /

User-agent: Twiceler
Disallow: /

User-agent: powerset
Disallow: /