# Robots.txt file from http://www.searchengineworld.com # # Bans from cgi-bin and images # # Disallow /cgi-bin/ # The following allows all robots to visit all files because # the wildcard "*" specifies all robots. # User-agent: * # Disallow: # # This one keeps all robots out. # User-agent: * # Disallow: / # # The next one bars all robots from the cgi-bin and images directories: # User-agent: * # Disallow: /cgi-bin/ # Disallow: /images/ # # This one bans Roverdog from all files on the server: # User-agent: Roverdog # Disallow: / # # This one bans keeps googlebot from getting at the cheese.htm file: # User-agent: googlebot # Disallow: cheese.htm User-agent: * Disallow: /cgi-bin/ Disallow: /images/