robots.txt

Created by case. Last edited by admin, 139 days ago. Viewed 87 times. #14

[diff] [history] [edit] [rdf]

labels

attachments

#
# ROBOTS.TXT -- this file is sent to robots accessing the URL /robots.txt
#
# *** This only works if SnipSnap is running at the top level of the server ***
# Keep in mind that robots added to the robot detection with the keyword DENY
# will be appended to this file and disallowed all access to SnipSnap!
#
# Disallow all access to the configuration part
User-agent: lmspider (lmspider@scansoft.com)
Disallow: /

User-agent: lmspider lmspider@scansoft.com
Disallow: /

User-agent: zermelo 
Disallow: /

User-agent: NaverBot
Disallow: /

User-agent: LiteFinder
Disallow: /

User-agent: msnbot
Crawl-delay: 600
Disallow: /space/SnipSnap/
Disallow: /exec/

User-agent: Slurp
Crawl-delay: 300
Disallow: /space/SnipSnap/
Disallow: /exec/

User-agent: Teoma
Crawl-delay: 300
Disallow: /space/SnipSnap/
Disallow: /exec/

User-agent: BecomeBot
Disallow: /

User-agent: *
Disallow: /space/SnipSnap/
Disallow: /exec/

no comments | post comment