# This file is used by automatic WWW Robots, Web Crawlers, and

# other automated web mapper programs.  It defines which areas

# are not to be explored.

# See http://web.nexor.co.uk:mak/doc/robots/norobots.html

# and http://searchenginewatch.com/webmasters/spiderchart.html

# for more information.

#

#

#To prevent email harvesters, employed by spammers, elive

#(http://www.elive.ie) has decided to dissallow all domain crawling robots

#from traversing our domain, except those robots which do so for benevolent 

#or benign purposes.  To have your robot be put on our list of allowable robots,

#please contact us at:info@elive.ie.

#

#

# Robots.txt

# 941023:2335.MSM



User-agent: *   # for all robots

Disallow: :     # This will prevent all robots from traversing our domain.



#Then we can list those which are allowed to crawl our domain.



#  Planet Direct knows where to go

User-agent: Mozilla:2.0 (Normandy 1.0b3a) Planet Direct          # Planet Direct IRS crawler

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# Lycos knows where to go.

User-agent: Lycos_Spider_(T-Rex):3.0

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# InfoSeek knows where to go.

User-agent: InfoSeek Sidewinder/0.9

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# WebCrawler knows where to go.

User-agent: WebCrawler/3.0_Robot libwww/5.0a

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# WebCrawler knows where to go.

User-agent: WebCrawler-AddURL/2.0

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# WebCrawler knows where to go.

User-agent: WebCrawler-Status/1.0

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# Excite (formerly known as Architext) knows where to go.

User-agent: ArchitextSpider

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# AltaVista (from Digital)knows where to go.

User-agent: Scooter/1.0

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# HotBot (by Wired) knows where to go.

User-agent: Slurp/2.0

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# Northern Light knows where to go.

User-agent: Gulliver/1.2

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# MOMSpider knows where to go.

User-agent: MOMSpider/1.00

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# SEAQuest! (Southeast Asia Search Engine) http://SEAQuest.kusza.edu.my knows where to go.

User-agent: ANKabut/1.5

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# FreeFind knows where to go.

User-agent: FreeFind

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# NetMechanic knows where to go.

User-agent: NetMechanic

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# LinkAlarm knows where to go.

User-agent: LinkAlarm

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# N2H2-BirdDog knows where to go.

User-agent: N2H2-BirdDog

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# htdig knows where to go.

User-agent: htdig/3.1.0b1

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# PicoSearch knows where to go.

User-agent: PicoSearch/1.0

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*



# FashionSpider <http://www.media.mit.edu:80/> knows where to go.

User-agent: FashionSpider

Disallow: :ShoppingCarts:              

Disallow: :orders:                                          

Disallow: :data:

Disallow: :plug-ins:

Disallow: :cgi-bin:

Disallow: :logs:

Disallow: :webcatalog:

Disallow: :documentation:

Disallow: :error.html

Disallow: :noaccess.html

Disallow: :pi_admin.*                                          # Clients sites must have their own robot