{\rtf1\ansi\ansicpg1252\deff0\deflang1033{\fonttbl{\f0\fswiss\fcharset0 Arial;}}
{\*\generator Msftedit 5.41.15.1507;}\viewkind4\uc1\pard\f0\fs20 # Robots.txt file from http://www.searchengineworld.com\par
#  \par
# Built from text file \par
http://info.webcrawler.com/mak/projects/robots/active/all.txt\par
# \par
# This restricts access to only known and registered robots. \par
#\par
\par
User-agent: Mozilla/3.0 (compatible;miner;mailto:miner@miner.com.br)\par
Disallow:\par
\par
User-agent: WebFerret\par
Disallow:\par
\par
User-agent: Due to a deficiency in Java it's not currently possible \par
to set the User-agent. \par
Disallow: \par
\par
User-agent: no \par
Disallow: \par
\par
User-agent: 'Ahoy! The Homepage Finder' \par
Disallow: \par
\par
User-agent: Arachnophilia \par
Disallow: \par
\par
User-agent: ArchitextSpider \par
Disallow: \par
\par
User-agent: ASpider/0.09 \par
Disallow: \par
\par
User-agent: AURESYS/1.0 \par
Disallow: \par
\par
User-agent: BackRub/*.* \par
Disallow: \par
\par
User-agent: Big Brother \par
Disallow: \par
\par
User-agent: BlackWidow \par
Disallow: \par
\par
User-agent: BSpider/1.0 libwww-perl/0.40 \par
Disallow: \par
\par
User-agent: CACTVS Chemistry Spider \par
Disallow: \par
\par
User-agent: Digimarc CGIReader/1.0 \par
Disallow: \par
\par
User-agent: Checkbot/x.xx LWP/5.x \par
Disallow: \par
\par
User-agent: CMC/0.01 \par
Disallow: \par
\par
User-agent: combine/0.0 \par
Disallow: \par
\par
User-agent: conceptbot/0.3 \par
Disallow: \par
\par
User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0\par
Disallow: \par
\par
User-agent: root/0.1 \par
Disallow: \par
\par
User-agent: CS-HKUST-IndexServer/1.0 \par
Disallow: \par
\par
User-agent: CyberSpyder/2.1 \par
Disallow: \par
\par
User-agent: Deweb/1.01 \par
Disallow: \par
\par
User-agent: DragonBot/1.0 libwww/5.0 \par
Disallow: \par
\par
User-agent: EIT-Link-Verifier-Robot/0.2 \par
Disallow: \par
\par
User-agent: Emacs-w3/v[0-9\\.]+ \par
Disallow: \par
\par
User-agent: EmailSiphon\par
Disallow:\par
\par
User-agent: EMC Spider \par
Disallow: \par
\par
User-agent: explorersearch \par
Disallow: \par
\par
User-agent: Explorer\par
Disallow: \par
\par
User-agent: ExtractorPro\par
Disallow: \par
\par
User-agent: FelixIDE/1.0 \par
Disallow: \par
\par
User-agent: Hazel's Ferret Web hopper, \par
Disallow: \par
\par
User-agent: ESIRover v1.0 \par
Disallow: \par
\par
User-agent: fido/0.9 Harvest/1.4.pl2 \par
Disallow: \par
\par
User-agent: H\'e4m\'e4h\'e4kki/0.2 \par
Disallow: \par
\par
User-agent: KIT-Fireball/2.0 libwww/5.0a \par
Disallow: \par
\par
User-agent: Fish-Search-Robot \par
Disallow: \par
\par
User-agent: Mozilla/2.0 (compatible fouineur v2.0; \par
fouineur.9bit.qc.ca) \par
Disallow: \par
\par
User-agent: Robot du CRIM 1.0a \par
Disallow: \par
\par
User-agent: Freecrawl \par
Disallow: \par
\par
User-agent: FunnelWeb-1.0 \par
Disallow: \par
\par
User-agent: gcreep/1.0 \par
Disallow: \par
\par
User-agent: ??? \par
Disallow: \par
\par
User-agent: GetURL.rexx v1.05 \par
Disallow: \par
\par
User-agent: Golem/1.1 \par
Disallow: \par
\par
User-agent: Gromit/1.0 \par
Disallow: \par
\par
User-agent: Gulliver/1.1 \par
Disallow: \par
\par
User-agent: yes \par
Disallow: \par
\par
User-agent: AITCSRobot/1.1 \par
Disallow: \par
\par
User-agent: wired-digital-newsbot/1.5 \par
Disallow: \par
\par
User-agent: htdig/3.0b3 \par
Disallow: \par
\par
User-agent: HTMLgobble v2.2 \par
Disallow: \par
\par
User-agent: no \par
Disallow: \par
\par
User-agent: IBM_Planetwide, \par
Disallow: \par
\par
User-agent: gestaltIconoclast/1.0 libwww-FM/2.17 \par
Disallow: \par
\par
User-agent: INGRID/0.1 \par
Disallow: \par
\par
User-agent: IncyWincy/1.0b1 \par
Disallow: \par
\par
User-agent: Informant \par
Disallow: \par
\par
User-agent: InfoSeek Robot 1.0 \par
Disallow: \par
\par
User-agent: Infoseek Sidewinder \par
Disallow: \par
\par
User-agent: InfoSpiders/0.1 \par
Disallow: \par
\par
User-agent: inspectorwww/1.0 \par
http://www.greenpac.com/inspectorwww.html \par
Disallow: \par
\par
User-agent: 'IAGENT/1.0' \par
Disallow: \par
\par
User-agent: IsraeliSearch/1.0 \par
Disallow: \par
\par
User-agent: JCrawler/0.2 \par
Disallow: \par
\par
User-agent: Jeeves v0.05alpha (PERL, LWP, lglb@doc.ic.ac.uk) \par
Disallow: \par
\par
User-agent: Jobot/0.1alpha libwww-perl/4.0 \par
Disallow: \par
\par
User-agent: JoeBot, \par
Disallow: \par
\par
User-agent: JubiiRobot\par
Disallow: \par
\par
User-agent: jumpstation \par
Disallow: \par
\par
User-agent: Katipo/1.0 \par
Disallow: \par
\par
User-agent: KDD-Explorer/0.1 \par
Disallow: \par
\par
User-agent: KO_Yappo_Robot/1.0.4(http://yappo.com/info/robot.html) \par
Disallow: \par
\par
User-agent: LabelGrab/1.1 \par
Disallow: \par
\par
User-agent: LinkWalker \par
Disallow: \par
\par
User-agent: logo.gif crawler \par
Disallow: \par
\par
User-agent: Lycos/x.x \par
Disallow: \par
\par
User-agent: Lycos_Spider_(T-Rex)\par
Disallow: \par
\par
User-agent: Magpie/1.0 \par
Disallow: \par
\par
User-agent: MediaFox/x.y \par
Disallow: \par
\par
User-agent: MerzScope \par
Disallow: \par
\par
User-agent: NEC-MeshExplorer \par
Disallow: \par
\par
User-agent: MOMspider/1.00 libwww-perl/0.40 \par
Disallow: \par
\par
User-agent: Monster/vX.X.X -$TYPE ($OSTYPE) \par
Disallow: \par
\par
User-agent: Motor/0.2 \par
Disallow: \par
\par
User-agent: MuscatFerret \par
Disallow: \par
\par
User-agent: MwdSearch/0.1 \par
Disallow: \par
\par
User-agent: NetCarta CyberPilot Pro \par
Disallow: \par
\par
User-agent: NetMechanic \par
Disallow: \par
\par
User-agent: NetScoop/1.0 libwww/5.0a \par
Disallow: \par
\par
User-agent: NHSEWalker/3.0 \par
Disallow: \par
\par
User-agent: Nomad-V2.x \par
Disallow: \par
\par
User-agent: NorthStar \par
Disallow: \par
\par
User-agent: Occam/1.0 \par
Disallow: \par
\par
User-agent: HKU WWW Robot, \par
Disallow: \par
\par
User-agent: Orbsearch/1.0 \par
Disallow: \par
\par
User-agent: PackRat/1.0 \par
Disallow: \par
\par
User-agent: Patric/0.01a \par
Disallow: \par
\par
User-agent: Peregrinator-Mathematics/0.7 \par
Disallow: \par
\par
User-agent: Duppies \par
Disallow: \par
\par
User-agent: Pioneer \par
Disallow: \par
\par
User-agent: PGP-KA/1.2 \par
Disallow: \par
\par
User-agent: Resume Robot \par
Disallow: \par
\par
User-agent: Road Runner: ImageScape Robot (lim@cs.leidenuniv.nl) \par
Disallow: \par
\par
User-agent: Robbie/0.1 \par
Disallow: \par
\par
User-agent: ComputingSite Robi/1.0 (robi@computingsite.com) \par
Disallow: \par
\par
User-agent: Roverbot \par
Disallow: \par
\par
User-agent: SafetyNet Robot 0.1, \par
Disallow: \par
\par
User-agent: Scooter/1.0 \par
Disallow: \par
\par
User-agent: not available \par
Disallow: \par
\par
User-agent: Senrigan/xxxxxx \par
Disallow: \par
\par
User-agent: SG-Scout \par
Disallow: \par
\par
User-agent: Shai'Hulud \par
Disallow: \par
\par
User-agent: SimBot/1.0 \par
Disallow: \par
\par
User-agent: Open Text Site Crawler V1.0 \par
Disallow: \par
\par
User-agent: SiteTech-Rover \par
Disallow: \par
\par
User-agent: Slurp/2.0 \par
Disallow: \par
\par
User-agent: ESISmartSpider/2.0 \par
Disallow: \par
\par
User-agent: Snooper/b97_01 \par
Disallow: \par
\par
User-agent: Solbot/1.0 LWP/5.07 \par
Disallow: \par
\par
User-agent: Spanner/1.0 (Linux 2.0.27 i586) \par
Disallow: \par
\par
User-agent: no \par
Disallow: \par
\par
User-agent: Mozilla/3.0 (Black Widow v1.1.0; Linux 2.0.27; Dec 31 \par
1997 12:25:00 \par
Disallow: \par
\par
User-agent: Tarantula/1.0 \par
Disallow: \par
\par
User-agent: tarspider \par
Disallow: \par
\par
User-agent: dlw3robot/x.y (in TclX by http://hplyot.obspm.fr/~dl/) \par
Disallow: \par
\par
User-agent: Templeton/ \par
Disallow: \par
\par
User-agent: TitIn/0.2 \par
Disallow: \par
\par
User-agent: TITAN/0.1 \par
Disallow: \par
\par
User-agent: UCSD-Crawler \par
Disallow: \par
\par
User-agent: urlck/1.2.3 \par
Disallow: \par
\par
User-agent: Valkyrie/1.0 libwww-perl/0.40 \par
Disallow: \par
\par
User-agent: Victoria/1.0 \par
Disallow: \par
\par
User-agent: vision-search/3.0' \par
Disallow: \par
\par
User-agent: VWbot_K/4.2 \par
Disallow: \par
\par
User-agent: w3index \par
Disallow: \par
\par
User-agent: W3M2/x.xxx \par
Disallow: \par
\par
User-agent: WWWWanderer v3.0 \par
Disallow: \par
\par
User-agent: WebCopy/\par
Disallow: \par
\par
User-agent: WebCrawler/3.0 Robot libwww/5.0a \par
Disallow: \par
\par
User-agent: WebFetcher/0.8, \par
Disallow: \par
\par
User-agent: weblayers/0.0 \par
Disallow: \par
\par
User-agent: WebLinker/0.0 libwww-perl/0.1 \par
Disallow: \par
\par
User-agent: no \par
Disallow: \par
\par
User-agent: WebMoose/0.0.0000 \par
Disallow: \par
\par
User-agent: Digimarc WebReader/1.2 \par
Disallow: \par
\par
User-agent: webs@recruit.co.jp \par
Disallow: \par
\par
User-agent: webvac/1.0 \par
Disallow: \par
\par
User-agent: webwalk \par
Disallow: \par
\par
User-agent: WebWalker/1.10 \par
Disallow: \par
\par
User-agent: WebWatch \par
Disallow: \par
\par
User-agent: Wget/1.4.0 \par
Disallow: \par
\par
User-agent: w3mir \par
Disallow: \par
\par
User-agent: no \par
Disallow: \par
\par
User-agent: WWWC/0.25 (Win95) \par
Disallow: \par
\par
User-agent: none \par
Disallow: \par
\par
User-agent: XGET/0.7 \par
Disallow: \par
\par
User-agent: Nederland.zoek \par
Disallow: \par
\par
User-agent: BizBot04 kirk.overleaf.com \par
Disallow: \par
\par
User-agent: HappyBot (gserver.kw.net) \par
Disallow: \par
\par
User-agent: CaliforniaBrownSpider \par
Disallow: \par
\par
User-agent: EI*Net/0.1 libwww/0.1 \par
Disallow: \par
\par
User-agent: Ibot/1.0 libwww-perl/0.40 \par
Disallow: \par
\par
User-agent: Merritt/1.0 \par
Disallow: \par
\par
User-agent: StatFetcher/1.0 \par
Disallow: \par
\par
User-agent: TeacherSoft/1.0 libwww/2.17 \par
Disallow: \par
\par
User-agent: WWW Collector \par
Disallow: \par
\par
User-agent: processor/0.0ALPHA libwww-perl/0.20 \par
Disallow: \par
\par
User-agent: wobot/1.0 from 206.214.202.45 \par
Disallow: \par
\par
User-agent: Libertech-Rover www.libertech.com? \par
Disallow: \par
\par
User-agent: WhoWhere Robot \par
Disallow: \par
\par
User-agent: ITI Spider \par
Disallow: \par
\par
User-agent: w3index \par
Disallow: \par
\par
User-agent: MyCNNSpider \par
Disallow: \par
\par
User-agent: SummyCrawler \par
Disallow: \par
\par
User-agent: OGspider \par
Disallow: \par
\par
User-agent: linklooker \par
Disallow: \par
\par
User-agent: CyberSpyder (amant@www.cyberspyder.com) \par
Disallow: \par
\par
User-agent: SlowBot \par
Disallow: \par
\par
User-agent: heraSpider \par
Disallow: \par
\par
User-agent: Surfbot \par
Disallow: \par
\par
User-agent: Bizbot003 \par
Disallow: \par
\par
User-agent: WebWalker \par
Disallow: \par
\par
User-agent: SandBot \par
Disallow: \par
\par
User-agent: EnigmaBot \par
Disallow: \par
\par
User-agent: spyder3.microsys.com \par
Disallow: \par
\par
User-agent: www.freeloader.com. \par
Disallow: \par
\par
User-agent: Googlebot\par
Disallow: \par
\par
User-agent: METAGOPHER\par
Disallow: \par
\par
User-agent: *\par
Disallow: /\par
\par
}