# #robots file # # # More SEARCHENGINE commands can be found at # http://www.analog.cx/helpers/#conffiles SEARCHENGINE http://*altavista.*/* q SEARCHENGINE http://*yahoo.*/* p SEARCHENGINE http://*google.*/* q SEARCHENGINE http://*lycos.*/* query SEARCHENGINE http://*aol.*/* query SEARCHENGINE http://*excite.*/* search SEARCHENGINE http://*go2net.*/* general SEARCHENGINE http://*metacrawler.*/* general SEARCHENGINE http://*msn.*/* MT SEARCHENGINE http://*hotbot.com/* MT SEARCHENGINE http://*netscape.*/* search SEARCHENGINE http://*looksmart.*/* key SEARCHENGINE http://*infoseek.*/* qt SEARCHENGINE http://*webcrawler.*/* search,searchText SEARCHENGINE http://*goto.*/* Keywords SEARCHENGINE http://*snap.*/* keyword SEARCHENGINE http://*dogpile.*/* q SEARCHENGINE http://*askjeeves.*/* ask SEARCHENGINE http://*ask.*/* ask SEARCHENGINE http://*aj.*/* ask SEARCHENGINE http://*directhit.*/* qry SEARCHENGINE http://*alltheweb.*/* query SEARCHENGINE http://*northernlight.*/* qr SEARCHENGINE http://*nlsearch.*/* qr SEARCHENGINE http://*dmoz.*/* search SEARCHENGINE http://*newhoo.*/* search SEARCHENGINE http://*netfind.*/* query,search,s SEARCHENGINE http://*/netfind* query SEARCHENGINE http://*/pursuit query ROBOTINCLUDE "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)" BROWEXCLUDE "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)" ROBOTINCLUDE "emyuu_bot*" BROWEXCLUDE "emyuu_bot*" ROBOTINCLUDE "FeedChecker/*" BROWEXCLUDE "FeedChecker/*" ROBOTINCLUDE "RYUKOGO-LINKER-BOT*" BROWEXCLUDE "RYUKOGO-LINKER-BOT*" ROBOTINCLUDE "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" BROWEXCLUDE "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" ROBOTINCLUDE "blogoonbot*" BROWEXCLUDE "blogoonbot*" ROBOTINCLUDE "HyperEstraier*" BROWEXCLUDE "HyperEstraier*" ROBOTINCLUDE "livedoorCheckers*" BROWEXCLUDE "livedoorCheckers*" ROBOTINCLUDE "MJ12bot/*" BROWEXCLUDE "MJ12bot/*" ROBOTINCLUDE "Snapbot/*" BROWEXCLUDE "Snapbot/*" ROBOTINCLUDE "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iOpus-I-M)" BROWEXCLUDE "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; iOpus-I-M)" ROBOTINCLUDE "Mozilla/4.0 (compatible; MSIE 5.01; Windows 98)" BROWEXCLUDE "Mozilla/4.0 (compatible; MSIE 5.01; Windows 98)" ROBOTINCLUDE "Mozilla/5.0 (compatible; BecomeJPBot*" BROWEXCLUDE "Mozilla/5.0 (compatible; BecomeJPBot*" ROBOTINCLUDE "Website Explorer/*" BROWEXCLUDE "Website Explorer/*" ROBOTINCLUDE "Mozilla/4.0 (compatible; MSIE 6.0b; Windows NT 5.0)" BROWEXCLUDE "Mozilla/4.0 (compatible; MSIE 6.0b; Windows NT 5.0)" ROBOTINCLUDE "Mozilla/4.0 (compatible; MSIE 4.5; Windows 98; )" BROWEXCLUDE "Mozilla/4.0 (compatible; MSIE 4.5; Windows 98; )" ROBOTINCLUDE "WWW-Mechanize*" BROWEXCLUDE "WWW-Mechanize*" ROBOTINCLUDE "Image Collector*" BROWEXCLUDE "Image Collector*" ROBOTINCLUDE "(compatible; MSIE 5.0; Windows NT)" BROWEXCLUDE "(compatible; MSIE 5.0; Windows NT)" ROBOTINCLUDE "1.0" BROWEXCLUDE "1.0" ROBOTINCLUDE "Exabot*" BROWEXCLUDE "Exabot*" ROBOTINCLUDE "ejupiter*" BROWEXCLUDE "ejupiter*" ROBOTINCLUDE "*J-BSC*" BROWEXCLUDE "*J-BSC*" ROBOTINCLUDE "*waseda*" BROWEXCLUDE "*waseda*" ROBOTINCLUDE "Mozilla/5.0 (compatible; BecomeBot*" BROWEXCLUDE "Mozilla/5.0 (compatible; BecomeBot*" ROBOTINCLUDE "Drecombot*" BROWEXCLUDE "Drecombot*" ROBOTINCLUDE "Mozilla/4.0 (compatible; MSIE 5.00; Windows 98" BROWEXCLUDE "Mozilla/4.0 (compatible; MSIE 5.00; Windows 98" ROBOTINCLUDE "Mozilla/4.0" BROWEXCLUDE "Mozilla/4.0" ROBOTINCLUDE "Gigabot*" BROWEXCLUDE "Gigabot*" ROBOTINCLUDE "AppleSyndication*" BROWEXCLUDE "AppleSyndication*" ROBOTINCLUDE "Jakarta Commons-HttpClient*" BROWEXCLUDE "Jakarta Commons-HttpClient*" ROBOTINCLUDE "appie*" BROWEXCLUDE "appie*" ROBOTINCLUDE "OmniExplorer_Bot*" BROWEXCLUDE "OmniExplorer_Bot* ROBOTINCLUDE "Hatena Antenna*" BROWEXCLUDE "Hatena Antenna*" ROBOTINCLUDE "Livedoor SF*" BROWEXCLUDE "Livedoor SF*" ROBOTINCLUDE "Plucker*" BROWEXCLUDE "Plucker*" ROBOTINCLUDE "DownloadSession" BROWEXCLUDE "DownloadSession" ROBOTINCLUDE "lwp-trivial*" BROWEXCLUDE "lwp-trivial*" ROBOTINCLUDE "Microsoft Data Access*" BROWEXCLUDE "Microsoft Data Access*" ROBOTINCLUDE "PEAR HTTP_Request class*" BROWEXCLUDE "PEAR HTTP_Request class*" ROBOTINCLUDE "Technoratibot*" BROWEXCLUDE "Technoratibot*" ROBOTINCLUDE "InternetLinkAgent*" BROWEXCLUDE "InternetLinkAgent*" ROBOTINCLUDE "Bookmark Renewal*" BROWEXCLUDE "Bookmark Renewal*" ROBOTINCLUDE "SAO Japone MAP URL CHECK*" BROWEXCLUDE "SAO Japone MAP URL CHECK*" ROBOTINCLUDE "libwww-perl*" BROWEXCLUDE "libwww-perl*" ROBOTINCLUDE "NG/*" BROWEXCLUDE "NG/*" ROBOTINCLUDE "NutchCVS/*" BROWEXCLUDE "NutchCVS/*" ROBOTINCLUDE "aipbot/*" BROWEXCLUDE "aipbot/*" ROBOTINCLUDE BlogRanking* BROWEXCLUDE BlogRanking* BROWEXCLUDE ia_archiver ROBOTINCLUDE ia_archiver ROBOTINCLUDE ichiro* BROWEXCLUDE ichiro* ROBOTINCLUDE Mediapartners-Google* BROWEXCLUDE Mediapartners-Google* ROBOTINCLUDE "Mozilla/2.0 (compatible; Ask Jeeves/Teoma*" BROWEXCLUDE "Mozilla/2.0 (compatible; Ask Jeeves/Teoma*" ROBOTINCLUDE "Mozilla/3.0 (compatible; PerMan*" BROWEXCLUDE "Mozilla/3.0 (compatible; PerMan*" ROBOTINCLUDE "Mozilla/4.0 compatible ZyBorg*" BROWEXCLUDE "Mozilla/4.0 compatible ZyBorg*" ROBOTINCLUDE "Pockey-GetHTML/*" BROWEXCLUDE "Pockey-GetHTML/*" ROBOTINCLUDE REGEXPI:robot ROBOTINCLUDE REGEXPI:spider ROBOTINCLUDE REGEXPI:crawler ROBOTINCLUDE "Mozilla/5.0 (compatible; Yahoo! Slurp*" ROBOTINCLUDE "Mozilla/5.0 (compatible; Googlebot/2.1*" ROBOTINCLUDE *Googlebot/* ROBOTINCLUDE Infoseek* ROBOTINCLUDE Scooter* ROBOTINCLUDE Slurp* ROBOTINCLUDE Ultraseek* ROBOTINCLUDE $1$1$1* ROBOTINCLUDE AITCSRobot/* ROBOTINCLUDE ATN_Worldwide* ROBOTINCLUDE AURESYS/* ROBOTINCLUDE AlkalEXeBOT* ROBOTINCLUDE AnthillV* ROBOTINCLUDE Arachnophilia* ROBOTINCLUDE Araneo/* ROBOTINCLUDE AraybOt/* ROBOTINCLUDE Atomz/* ROBOTINCLUDE BackRub/$1.$1* ROBOTINCLUDE Bjaaland/* ROBOTINCLUDE BlackWidow* ROBOTINCLUDE CMC/* ROBOTINCLUDE Calif/* ROBOTINCLUDE CoolBot* ROBOTINCLUDE Cusco/* ROBOTINCLUDE CyberSpyder/* ROBOTINCLUDE DIIbot* ROBOTINCLUDE DNAbot/* ROBOTINCLUDE DWCP/* ROBOTINCLUDE DesertRealm.com;* ROBOTINCLUDE Deweb/* ROBOTINCLUDE Digger/* ROBOTINCLUDE DragonBot/* ROBOTINCLUDE Duppies* ROBOTINCLUDE EIT-LEXk-Verifier-Robot/* ROBOTINCLUDE EbEXess/* ROBOTINCLUDE Emacs-w3/v[0-9\.]+* ROBOTINCLUDE FelixIDE/* ROBOTINCLUDE Fish-Search-Robot* ROBOTINCLUDE Freecrawl* ROBOTINCLUDE FunnelWeb-* ROBOTINCLUDE Golem/* ROBOTINCLUDE Gromit/* ROBOTINCLUDE Gulliver/* ROBOTINCLUDE H???ki/* ROBOTINCLUDE IAGENT/* ROBOTINCLUDE IBM_Planetwide* ROBOTINCLUDE EXGRID/* ROBOTINCLUDE EXcyWEXcy/* ROBOTINCLUDE EXformant* ROBOTINCLUDE Iron33/* ROBOTINCLUDE IsraeliSearch/* ROBOTINCLUDE JavaBee* ROBOTINCLUDE Jobot/* ROBOTINCLUDE JoeBot/x.x* ROBOTINCLUDE JubiiRobot/version#* ROBOTINCLUDE KDD-Explorer/* ROBOTINCLUDE KIT-Fireball/* ROBOTINCLUDE KO_Yappo_Robot/* ROBOTINCLUDE Katipo/* ROBOTINCLUDE LWP* ROBOTINCLUDE LabelGrab/* ROBOTINCLUDE LEXkWalker* ROBOTINCLUDE LEXkidator/* ROBOTINCLUDE Lockon/xxxxx* ROBOTINCLUDE Lycos/x.x* ROBOTINCLUDE M/* ROBOTINCLUDE MSNBOT/* ROBOTINCLUDE Magpie/* ROBOTINCLUDE MediaFox/x.y* ROBOTINCLUDE MerzScope* ROBOTINCLUDE Motor/* ROBOTINCLUDE MunEXn/* ROBOTINCLUDE MuscatFerret/* ROBOTINCLUDE MwdSearch/* ROBOTINCLUDE NEC-MeshExplorer* ROBOTINCLUDE NHSEWalker/* ROBOTINCLUDE Naver* ROBOTINCLUDE Nederland.zoek* ROBOTINCLUDE NetMechanic* ROBOTINCLUDE NetScoop/* ROBOTINCLUDE Nomad-V2.x* ROBOTINCLUDE None* ROBOTINCLUDE NorthStar* ROBOTINCLUDE ObjectsSearch/* ROBOTINCLUDE Occam/* ROBOTINCLUDE Orbsearch/* ROBOTINCLUDE PGP-KA/* ROBOTINCLUDE PackRat/* ROBOTINCLUDE PageBoy/* ROBOTINCLUDE ParaSite/* ROBOTINCLUDE Patric/* ROBOTINCLUDE PeregrEXator-Mathematics/* ROBOTINCLUDE PiltdownMan/* ROBOTINCLUDE Pioneer* ROBOTINCLUDE PlumtreeWebAccessor/* ROBOTINCLUDE Poppi/* ROBOTINCLUDE PortalJuice.com/* ROBOTINCLUDE RHCS/* ROBOTINCLUDE Raven-v2* ROBOTINCLUDE Robbie/* ROBOTINCLUDE Robozilla/ ROBOTINCLUDE Roverbot* ROBOTINCLUDE RuLeS/* ROBOTINCLUDE SG-Scout* ROBOTINCLUDE Senrigan/xxxxxx* ROBOTINCLUDE Shai'Hulud* ROBOTINCLUDE SimBot/* ROBOTINCLUDE SiteTech-Rover* ROBOTINCLUDE Solbot/* ROBOTINCLUDE TITAN/* ROBOTINCLUDE Tarantula/* ROBOTINCLUDE TechBOT* ROBOTINCLUDE TitEX/* ROBOTINCLUDE UdmSearch* ROBOTINCLUDE VWbot_K/* ROBOTINCLUDE Valkyrie/* ROBOTINCLUDE Verticrawl* ROBOTINCLUDE Victoria/* ROBOTINCLUDE Voyager/* ROBOTINCLUDE W3M2/x.xxx* ROBOTINCLUDE WOLP/* ROBOTINCLUDE WWWC/* ROBOTINCLUDE WebBandit/* ROBOTINCLUDE WebCatcher/* ROBOTINCLUDE WebCopy/(version)* ROBOTINCLUDE WebFetcher/* ROBOTINCLUDE WebLEXker/* ROBOTINCLUDE WebMoose/* ROBOTINCLUDE WebQuest/* ROBOTINCLUDE WebWalker/* ROBOTINCLUDE WebWatch* ROBOTINCLUDE Wget/* ROBOTINCLUDE XGET/* ROBOTINCLUDE aWapClient* ROBOTINCLUDE appie/* ROBOTINCLUDE arks/* ROBOTINCLUDE bbot/* ROBOTINCLUDE borg-bot/* ROBOTINCLUDE combEXe/* ROBOTINCLUDE conceptbot/* ROBOTINCLUDE cosmos/* ROBOTINCLUDE elfEXbot* ROBOTINCLUDE esculapio/* ROBOTINCLUDE esther* ROBOTINCLUDE explorersearch* ROBOTINCLUDE fido/* ROBOTINCLUDE gazz/* ROBOTINCLUDE gcreep/* ROBOTINCLUDE grabber* ROBOTINCLUDE griffon/* ROBOTINCLUDE havEXdex/X.xx[bxx]* ROBOTINCLUDE htdig/* ROBOTINCLUDE http://www.sygol.com* ROBOTINCLUDE iajaBot/* ROBOTINCLUDE image.kapsi.net/* ROBOTINCLUDE EXspectorwww/* ROBOTINCLUDE jumpstation* ROBOTINCLUDE legs* ROBOTINCLUDE libwww-perl-* ROBOTINCLUDE msnbot* ROBOTINCLUDE moget/* ROBOTINCLUDE mouse.house/* ROBOTINCLUDE newscan-onlEXe/* ROBOTINCLUDE no* ROBOTINCLUDE phpdig/x.x.x* ROBOTINCLUDE psbot/* ROBOTINCLUDE root/* ROBOTINCLUDE searchprocess/* ROBOTINCLUDE ssearcher100* ROBOTINCLUDE suke/$1.$1* ROBOTINCLUDE suntek/* ROBOTINCLUDE uptimebot* ROBOTINCLUDE urlck/* ROBOTINCLUDE vision-search/* ROBOTINCLUDE void-bot/* ROBOTINCLUDE w3EXdex* ROBOTINCLUDE w3mir* ROBOTINCLUDE weblayers/* ROBOTINCLUDE webs@recruit.co.jp* ROBOTINCLUDE webvac/* ROBOTINCLUDE webwalk* ROBOTINCLUDE wired-digital-newsbot/* ROBOTINCLUDE wlm-* ROBOTINCLUDE yes* BROWEXCLUDE *Yahoo* BROWEXCLUDE "Mozilla/5.0 (compatible; Googlebot/2.1*" BROWEXCLUDE REGEXPI:robot BROWEXCLUDE REGEXPI:spider BROWEXCLUDE REGEXPI:crawler BROWEXCLUDE Googlebot* BROWEXCLUDE Infoseek* BROWEXCLUDE Scooter* BROWEXCLUDE Slurp* BROWEXCLUDE Ultraseek* BROWEXCLUDE AITCSRobot/* BROWEXCLUDE ATN_Worldwide* BROWEXCLUDE AURESYS/* BROWEXCLUDE AlkalEXeBOT* BROWEXCLUDE AnthillV* BROWEXCLUDE Arachnophilia* BROWEXCLUDE Araneo/* BROWEXCLUDE AraybOt/* BROWEXCLUDE Atomz/* BROWEXCLUDE BackRub/$1.$1* BROWEXCLUDE Bjaaland/* BROWEXCLUDE BlackWidow* BROWEXCLUDE CMC/* BROWEXCLUDE Calif/* BROWEXCLUDE CoolBot* BROWEXCLUDE Cusco/* BROWEXCLUDE CyberSpyder/* BROWEXCLUDE DIIbot* BROWEXCLUDE DNAbot/* BROWEXCLUDE DWCP/* BROWEXCLUDE DesertRealm.com;* BROWEXCLUDE Deweb/* BROWEXCLUDE Digger/* BROWEXCLUDE DragonBot/* BROWEXCLUDE Duppies* BROWEXCLUDE EIT-LEXk-Verifier-Robot/* BROWEXCLUDE EbEXess/* BROWEXCLUDE Emacs-w3/v[0-9\.]+* BROWEXCLUDE FelixIDE/* BROWEXCLUDE Fish-Search-Robot* BROWEXCLUDE Freecrawl* BROWEXCLUDE FunnelWeb-* BROWEXCLUDE Golem/* BROWEXCLUDE Gromit/* BROWEXCLUDE Gulliver/* BROWEXCLUDE H???ki/* BROWEXCLUDE IAGENT/* BROWEXCLUDE IBM_Planetwide* BROWEXCLUDE EXGRID/* BROWEXCLUDE EXcyWEXcy/* BROWEXCLUDE EXformant* BROWEXCLUDE Iron33/* BROWEXCLUDE IsraeliSearch/* BROWEXCLUDE JavaBee* BROWEXCLUDE Jobot/* BROWEXCLUDE JoeBot/x.x* BROWEXCLUDE JubiiRobot/version#* BROWEXCLUDE KDD-Explorer/* BROWEXCLUDE KIT-Fireball/* BROWEXCLUDE KO_Yappo_Robot/* BROWEXCLUDE Katipo/* BROWEXCLUDE LWP* BROWEXCLUDE LabelGrab/* BROWEXCLUDE LEXkWalker* BROWEXCLUDE LEXkidator/* BROWEXCLUDE Lockon/xxxxx* BROWEXCLUDE Lycos/x.x* BROWEXCLUDE M/* BROWEXCLUDE MSNBOT/* BROWEXCLUDE Magpie/* BROWEXCLUDE MediaFox/x.y* BROWEXCLUDE MerzScope* BROWEXCLUDE Motor/* BROWEXCLUDE MunEXn/* BROWEXCLUDE MuscatFerret/* BROWEXCLUDE MwdSearch/* BROWEXCLUDE Naver* BROWEXCLUDE NEC-MeshExplorer* BROWEXCLUDE NHSEWalker/* BROWEXCLUDE Nederland.zoek* BROWEXCLUDE NetMechanic* BROWEXCLUDE NetScoop/* BROWEXCLUDE Nomad-V2.x* BROWEXCLUDE None* BROWEXCLUDE NorthStar* BROWEXCLUDE ObjectsSearch/* BROWEXCLUDE Occam/* BROWEXCLUDE Orbsearch/* BROWEXCLUDE PGP-KA/* BROWEXCLUDE PackRat/* BROWEXCLUDE PageBoy/* BROWEXCLUDE ParaSite/* BROWEXCLUDE Patric/* BROWEXCLUDE PeregrEXator-Mathematics/* BROWEXCLUDE PiltdownMan/* BROWEXCLUDE Pioneer* BROWEXCLUDE PlumtreeWebAccessor/* BROWEXCLUDE Poppi/* BROWEXCLUDE PortalJuice.com/* BROWEXCLUDE RHCS/* BROWEXCLUDE Raven-v2* BROWEXCLUDE Robbie/* BROWEXCLUDE Robozilla/* BROWEXCLUDE Roverbot* BROWEXCLUDE RuLeS/* BROWEXCLUDE SG-Scout* BROWEXCLUDE Senrigan/xxxxxx* BROWEXCLUDE Shai'Hulud* BROWEXCLUDE SimBot/* BROWEXCLUDE SiteTech-Rover* BROWEXCLUDE Solbot/* BROWEXCLUDE TITAN/* BROWEXCLUDE Tarantula/* BROWEXCLUDE TechBOT* BROWEXCLUDE TitEX/* BROWEXCLUDE UdmSearch* BROWEXCLUDE VWbot_K/* BROWEXCLUDE Valkyrie/* BROWEXCLUDE Verticrawl* BROWEXCLUDE Victoria/* BROWEXCLUDE Voyager/* BROWEXCLUDE W3M2/x.xxx* BROWEXCLUDE WOLP/* BROWEXCLUDE WWWC/* BROWEXCLUDE WebBandit/* BROWEXCLUDE WebCatcher/* BROWEXCLUDE WebCopy/(version)* BROWEXCLUDE WebFetcher/* BROWEXCLUDE WebLEXker/* BROWEXCLUDE WebMoose/* BROWEXCLUDE WebQuest/* BROWEXCLUDE WebWalker/* BROWEXCLUDE WebWatch* BROWEXCLUDE Wget/* BROWEXCLUDE XGET/* BROWEXCLUDE aWapClient* BROWEXCLUDE appie/* BROWEXCLUDE arks/* BROWEXCLUDE bbot/* BROWEXCLUDE borg-bot/* BROWEXCLUDE combEXe/* BROWEXCLUDE conceptbot/* BROWEXCLUDE cosmos/* BROWEXCLUDE elfEXbot* BROWEXCLUDE esculapio/* BROWEXCLUDE esther* BROWEXCLUDE explorersearch* BROWEXCLUDE fido/* BROWEXCLUDE gazz/* BROWEXCLUDE gcreep/* BROWEXCLUDE grabber* BROWEXCLUDE griffon/* BROWEXCLUDE havEXdex/X.xx[bxx]* BROWEXCLUDE htdig/* BROWEXCLUDE http://www.sygol.com* BROWEXCLUDE iajaBot/* BROWEXCLUDE image.kapsi.net/* BROWEXCLUDE EXspectorwww/* BROWEXCLUDE jumpstation* BROWEXCLUDE legs* BROWEXCLUDE libwww-perl-* BROWEXCLUDE moget/* BROWEXCLUDE mouse.house/* BROWEXCLUDE msnbot* BROWEXCLUDE newscan-onlEXe/* BROWEXCLUDE no* BROWEXCLUDE phpdig/x.x.x* BROWEXCLUDE psbot/* BROWEXCLUDE root/* BROWEXCLUDE searchprocess/* BROWEXCLUDE ssearcher100* BROWEXCLUDE suke/$1.$1* BROWEXCLUDE suntek/* BROWEXCLUDE uptimebot* BROWEXCLUDE urlck/* BROWEXCLUDE vision-search/* BROWEXCLUDE void-bot/* BROWEXCLUDE w3EXdex* BROWEXCLUDE w3mir* BROWEXCLUDE weblayers/* BROWEXCLUDE webs@recruit.co.jp* BROWEXCLUDE webvac/* BROWEXCLUDE webwalk* BROWEXCLUDE wired-digital-newsbot/* BROWEXCLUDE wlm-* BROWEXCLUDE yes*