# robots.txt for http://www.catcaresociety.org/ # more information at http://www.robotstxt.org/wc/robots.html User-agent: * Disallow: /volunteers/ # do not permitting searching in the volunteers directory # Disallow: # the previous allows all robots or spiders to index the entire site # example of not allowing logging of a file # Disallow: /foo.html # example of allowing a particular robot entry to the entire site # User-agent: googlebot # Disallow: # example of wildcard which would prevent indexing of any file or directory # Disallow: /bob # prevents indexing of anything that starts with /bob, like /bob.html or /bob/index.html