Боты и их жизнь среде интернет.

rewritecond %{http_user_agent} ^QRVA [OR]

rewritecond %{http_user_agent} ^QuePasaCreep [OR]

rewritecond %{http_user_agent} ^QueryN.Metasearch [OR]

rewritecond %{http_user_agent} ^Reaper [OR]

rewritecond %{http_user_agent} ^Recorder [OR]

rewritecond %{http_user_agent} ^ReGet [OR]

rewritecond %{http_user_agent} ^RepoMonkey [OR]

rewritecond %{http_user_agent} ^RMA [OR]

rewritecond %{http_user_agent} ^Scooter [OR]

rewritecond %{http_user_agent} ^Seeker [OR]

rewritecond %{http_user_agent} ^Siphon [OR]

rewritecond %{http_user_agent} ^sitecheck.internetseer.com [OR]

rewritecond %{http_user_agent} ^SiteSnagger [OR]

rewritecond %{http_user_agent} ^SlySearch [OR]

rewritecond %{http_user_agent} ^SmartDownload [OR]

rewritecond %{http_user_agent} ^Snake [OR]

rewritecond %{http_user_agent} ^SpaceBison [OR]

rewritecond %{http_user_agent} ^SpankBot [OR]

rewritecond %{http_user_agent} ^spanner [OR]

rewritecond %{http_user_agent} ^spider [OR]

rewritecond %{http_user_agent} ^Stripper [OR]

rewritecond %{http_user_agent} ^Sucker [OR]

rewritecond %{http_user_agent} ^SuperHTTP [OR]

rewritecond %{http_user_agent} ^suzuran [OR]

rewritecond %{http_user_agent} ^Szukacz [OR]

rewritecond %{http_user_agent} ^tAkeOut [OR]

rewritecond %{http_user_agent} ^Teleport\ Pro [OR]

rewritecond %{http_user_agent} ^Telesoft [OR]

rewritecond %{http_user_agent} ^The.Intraformant [OR]

rewritecond %{http_user_agent} ^TheNomad [OR]

rewritecond %{http_user_agent} ^Titan [OR]

rewritecond %{http_user_agent} ^toCrawl/UrlDispatcher [OR]

rewritecond %{http_user_agent} ^True_Robot [OR]

rewritecond %{http_user_agent} ^turingos [OR]

rewritecond %{http_user_agent} ^TurnitinBot [OR]

rewritecond %{http_user_agent} ^URLy.Warning [OR]

rewritecond %{http_user_agent} ^Vacuum [OR]

rewritecond %{http_user_agent} ^VCI [OR]

rewritecond %{http_user_agent} ^VoidEYE [OR]

rewritecond %{http_user_agent} ^Web.*Image.*Collector [OR]

rewritecond %{http_user_agent} ^Web\ Sucker [OR]

rewritecond %{http_user_agent} ^WebAuto [OR]

rewritecond %{http_user_agent} ^WebBandit [OR]

rewritecond %{http_user_agent} ^WebEMailExtrac [OR]

rewritecond %{http_user_agent} ^WebEnhancer [OR]

rewritecond %{http_user_agent} ^WebFetch [OR]

rewritecond %{http_user_agent} ^WebGo\ IS [OR]

rewritecond %{http_user_agent} ^WebHook [OR]

rewritecond %{http_user_agent} ^WebLeacher [OR]

rewritecond %{http_user_agent} ^WebmasterWorldForumBot [OR]

rewritecond %{http_user_agent} ^WebMiner [OR]

rewritecond %{http_user_agent} ^WebMirror [OR]

rewritecond %{http_user_agent} ^WebReaper [OR]

rewritecond %{http_user_agent} ^WebSauger [OR]

rewritecond %{http_user_agent} ^Website [OR]

rewritecond %{http_user_agent} ^Website.*Quester [OR]

rewritecond %{http_user_agent} ^Website\ eXtractor [OR]

rewritecond %{http_user_agent} ^Webster [OR]

rewritecond %{http_user_agent} ^WebStripper [OR]

rewritecond %{http_user_agent} ^WebWhacker [OR]

rewritecond %{http_user_agent} ^Whacker [OR]

rewritecond %{http_user_agent} ^Widow [OR]

rewritecond %{http_user_agent} ^Widow [OR]

rewritecond %{http_user_agent} ^WWW-Collector-E [OR]

rewritecond %{http_user_agent} ^WWWOFFLE [OR]

rewritecond %{http_user_agent} ^Xaldon\ WebSpider [OR]

rewritecond %{http_user_agent} ^Xenu [OR]

rewritecond %{http_user_agent} ^x-Tractor [OR]

rewritecond %{http_user_agent} ^Zeus [OR]

RewriteCond %{HTTP_USER_AGENT} ^InternetSeer.com [OR]

RewriteCond %{HTTP_USER_AGENT} ^ZyBorg [OR]

RewriteCond %{HTTP_USER_AGENT} ^(autoemailspider|ExtractorPro) [OR]

RewriteCond %{HTTP_USER_AGENT} ^E?Mail.?(Collect|Harvest|Magnet|Reaper|Siphon|Sweeper|Wolf) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Alligator|DA.?[0-9]|DC\-Sakura|Download.?(Demon|Express|Master|Wonder)|FileHound) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Flash|Leech)Get [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Fresh|Lightning|Mass|Real|Smart|Speed|Star).?Download(er)? [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Gamespy|Go!Zilla|iGetter|JetCar|Net(Ants|Pumper)|SiteSnagger|Teleport.?Pro|WebReaper) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(My)?GetRight [OR]

RewriteCond %{HTTP_USER_AGENT} ^(AcoiRobot|FlickBot|webcollage) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Express|Mister|Web).?(Web|Pix|Image).?(Pictures|Collector)? [OR]

RewriteCond %{HTTP_USER_AGENT} ^Image.?(fetch|Stripper|Sucker) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Atomz|BlackWidow|BlogBot|EasyDL|Marketwave|Sqworm|SurveyBot|Webclipping\.com) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(eCatch|(Get|Super)Bot|Kapere|HTTrack|JOC|Offline|UtilMind|Xaldon) [OR]

RewriteCond %{HTTP_USER_AGENT} ^Web.?(Auto|Cop|dup|Fetch|Filter|Gather|Go|Leach|Mine|Mirror|Pix|QL|RACE|Sauger) [OR]

RewriteCond %{HTTP_USER_AGENT} ^Web.?(site.?(eXtractor|Quester)|Snake|ster|Strip|Suck|vac|walk|Whacker|ZIP) [OR]

RewriteCond %{HTTP_USER_AGENT} ^WebCapture [OR]

RewriteCond %{HTTP_USER_AGENT} ^DISCo\ Pump [OR]

RewriteCond %{HTTP_USER_AGENT} ^EirGrabber [OR]

RewriteCond %{HTTP_USER_AGENT} ^Net\ Vampire [OR]

RewriteCond %{HTTP_USER_AGENT} ^NetZIP [OR]

RewriteCond %{HTTP_USER_AGENT} ^(curl|Dart.?Communications|Enfish|htdig|Java|larbin) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(libwww|lwp|PHP|Python|www\.thatrobotsite\.com|webbandit|Wget|Zeus) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Crawl_Application|Lachesis|Nutscrape) [OR]

RewriteCond %{HTTP_USER_AGENT} ^[CDEFPRS](Browse|Eval|Surf) [OR]

RewriteCond %{HTTP_USER_AGENT} ^(Demo|Full.?Web|Lite|Production|Franklin|Missauga|Missigua).?(Bot|Locat) [OR]

RewriteCond %{HTTP_USER_AGENT} ^TurnitinBot [OR]

RewriteCond %{HTTP_USER_AGENT} ^Mihov_Picture_Downloader [OR]

RewriteCond %{HTTP_USER_AGENT} ^WebPix_1.0_(www.netwu.com)[OR]

RewriteCond %{HTTP_USER_AGENT} ^None_of_your_business_WebWasher_3.4 [OR]

RewriteCond %{HTTP_USER_AGENT} ^YahooVideoSearchhttp://www.yahoo.com/ [OR]

RewriteCond %{HTTP_USER_AGENT} ^Fuck_U! [OR]

RewriteCond %{HTTP_USER_AGENT} ^MisterPrivacy_Anonymous_Surfing_Services [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_2.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^FDM_2.x [OR]

RewriteCond %{HTTP_USER_AGENT} ^Squid-Prefetch [OR]

RewriteCond %{HTTP_USER_AGENT} ^DirtySexSearch[OR]

RewriteCond %{HTTP_USER_AGENT} ^Snoopy_v1.2.3 [OR]

RewriteCond %{HTTP_USER_AGENT} ^eBoard_Session [OR]

RewriteCond %{HTTP_USER_AGENT} ^Ineta_6.1_(http://ineta.de)[OR]

RewriteCond %{HTTP_USER_AGENT} ^abot[OR]

RewriteCond %{HTTP_USER_AGENT} ^aipbot[OR]

RewriteCond %{HTTP_USER_AGENT} ^asterias[OR]

RewriteCond %{HTTP_USER_AGENT} ^EI[OR]

RewriteCond %{HTTP_USER_AGENT} ^libwww-perl[OR]

RewriteCond %{HTTP_USER_AGENT} ^LWP[OR]

RewriteCond %{HTTP_USER_AGENT} ^lwp[OR]

RewriteCond %{HTTP_USER_AGENT} ^nameprotect[OR]

RewriteCond %{HTTP_USER_AGENT} ^PlantyNet_WebRobot[OR]

RewriteCond %{HTTP_USER_AGENT} ^UCmore[OR]

RewriteCond %{HTTP_USER_AGENT} ^Alligator[OR]

RewriteCond %{HTTP_USER_AGENT} ^AllSubmitter[OR]

RewriteCond %{HTTP_USER_AGENT} ^Asterias[OR]

RewriteCond %{HTTP_USER_AGENT} ^autoemailspider[OR]

RewriteCond %{HTTP_USER_AGENT} ^Badass[OR]

RewriteCond %{HTTP_USER_AGENT} ^Baiduspider[OR]

RewriteCond %{HTTP_USER_AGENT} ^BecomeBot[OR]

RewriteCond %{HTTP_USER_AGENT} ^Bitacle[OR]

RewriteCond %{HTTP_USER_AGENT} ^bladder\ fusion[OR]

RewriteCond %{HTTP_USER_AGENT} ^Blogshares\ Spiders[OR]

RewriteCond %{HTTP_USER_AGENT} ^Board\ Bot[OR]

RewriteCond %{HTTP_USER_AGENT} ^Board\ Bot[OR]

RewriteCond %{HTTP_USER_AGENT} ^Convera[OR]

RewriteCond %{HTTP_USER_AGENT} ^ConveraMultiMediaCrawler[OR]

RewriteCond %{HTTP_USER_AGENT} ^c-spider[OR]

RewriteCond %{HTTP_USER_AGENT} ^DA[OR]

RewriteCond %{HTTP_USER_AGENT} ^DnloadMage[OR]

RewriteCond %{HTTP_USER_AGENT} ^dragonfly[OR]

RewriteCond %{HTTP_USER_AGENT} ^DreamPassport[OR]

RewriteCond %{HTTP_USER_AGENT} ^FSurf[OR]

RewriteCond %{HTTP_USER_AGENT} ^Gaisbot[OR]

RewriteCond %{HTTP_USER_AGENT} ^Gamespy_Arcade[OR]

RewriteCond %{HTTP_USER_AGENT} ^genieBot[OR]

RewriteCond %{HTTP_USER_AGENT} ^GetBot[OR]

RewriteCond %{HTTP_USER_AGENT} ^Gigabot[OR]

RewriteCond %{HTTP_USER_AGENT} ^GOFORITBOT[OR]

RewriteCond %{HTTP_USER_AGENT} ^heritrix[OR]

RewriteCond %{HTTP_USER_AGENT} ^HLoader[OR]

RewriteCond %{HTTP_USER_AGENT} ^HooWWWer[OR]

RewriteCond %{HTTP_USER_AGENT} ^HTTrack[OR]

RewriteCond %{HTTP_USER_AGENT} ^iCCrawler[OR]

RewriteCond %{HTTP_USER_AGENT} ^ichiro[OR]

RewriteCond %{HTTP_USER_AGENT} ^iGetter[OR]

RewriteCond %{HTTP_USER_AGENT} ^imds_monitor[OR]

RewriteCond %{HTTP_USER_AGENT} ^IRLbot[OR]

RewriteCond %{HTTP_USER_AGENT} ^Jakarta[OR]

RewriteCond %{HTTP_USER_AGENT} ^jeteye[OR]

RewriteCond %{HTTP_USER_AGENT} ^jeteyebot[OR]

RewriteCond %{HTTP_USER_AGENT} ^JoBo[OR]

RewriteCond %{HTTP_USER_AGENT} ^Kapere[OR]

RewriteCond %{HTTP_USER_AGENT} ^Larbin[OR]

RewriteCond %{HTTP_USER_AGENT} ^LeechGet[OR]

RewriteCond %{HTTP_USER_AGENT} ^LightningDownload[OR]

RewriteCond %{HTTP_USER_AGENT} ^Linkie[OR]

RewriteCond %{HTTP_USER_AGENT} ^MyFamilyBot[OR]

RewriteCond %{HTTP_USER_AGENT} ^NASA\ Search[OR]

RewriteCond %{HTTP_USER_AGENT} ^Naver[OR]

RewriteCond %{HTTP_USER_AGENT} ^NaverBot[OR]

RewriteCond %{HTTP_USER_AGENT} ^NetAnts[OR]

RewriteCond %{HTTP_USER_AGENT} ^NetResearchServer[OR]

RewriteCond %{HTTP_USER_AGENT} ^NEWT\ ActiveX[OR]

RewriteCond %{HTTP_USER_AGENT} ^Nextopia[OR]

RewriteCond %{HTTP_USER_AGENT} ^NICErsPRO[OR]

RewriteCond %{HTTP_USER_AGENT} ^NimbleCrawler[OR]

RewriteCond %{HTTP_USER_AGENT} ^Nitro\ Downloader[OR]

RewriteCond %{HTTP_USER_AGENT} ^Nutch[OR]

RewriteCond %{HTTP_USER_AGENT} ^OmniExplorer[OR]

RewriteCond %{HTTP_USER_AGENT} ^OutfoxBot[OR]

RewriteCond %{HTTP_USER_AGENT} ^P3P[OR]

RewriteCond %{HTTP_USER_AGENT} ^PagmIEDownload[OR]

RewriteCond %{HTTP_USER_AGENT} ^pavuk[OR]

RewriteCond %{HTTP_USER_AGENT} ^playstarmusic[OR]

RewriteCond %{HTTP_USER_AGENT} ^psycheclone[OR]

RewriteCond %{HTTP_USER_AGENT} ^puf[OR]

RewriteCond %{HTTP_USER_AGENT} ^PussyCat[OR]

RewriteCond %{HTTP_USER_AGENT} ^PuxaRapido[OR]

RewriteCond %{HTTP_USER_AGENT} ^Python-urllib[OR]

RewriteCond %{HTTP_USER_AGENT} ^RealDownload[OR]

RewriteCond %{HTTP_USER_AGENT} ^RedKernel[OR]

RewriteCond %{HTTP_USER_AGENT} ^relevantnoise[OR]

RewriteCond %{HTTP_USER_AGENT} ^SBIder[OR]

RewriteCond %{HTTP_USER_AGENT} ^Seekbot[OR]

RewriteCond %{HTTP_USER_AGENT} ^SiteSnagger[OR]

RewriteCond %{HTTP_USER_AGENT} ^SmartDownload[OR]

RewriteCond %{HTTP_USER_AGENT} ^sna-[OR]

RewriteCond %{HTTP_USER_AGENT} ^Snap\ bot[OR]

RewriteCond %{HTTP_USER_AGENT} ^SpeedDownload[OR]

RewriteCond %{HTTP_USER_AGENT} ^Sphere[OR]

RewriteCond %{HTTP_USER_AGENT} ^sproose[OR]

RewriteCond %{HTTP_USER_AGENT} ^Stamina[OR]

RewriteCond %{HTTP_USER_AGENT} ^Teleport[OR]

RewriteCond %{HTTP_USER_AGENT} ^TurnitinBot[OR]

RewriteCond %{HTTP_USER_AGENT} ^UdmSearch[OR]

RewriteCond %{HTTP_USER_AGENT} ^WebAuto[OR]

RewriteCond %{HTTP_USER_AGENT} ^WebCapture[OR]

RewriteCond %{HTTP_USER_AGENT} ^webcollage[OR]

RewriteCond %{HTTP_USER_AGENT} ^WebFilter[OR]

RewriteCond %{HTTP_USER_AGENT} ^WebReaper[OR]

RewriteCond %{HTTP_USER_AGENT} ^WebStripper[OR]

RewriteCond %{HTTP_USER_AGENT} ^Wells\ Search[OR]

RewriteCond %{HTTP_USER_AGENT} ^WinHttpRequest[OR]

RewriteCond %{HTTP_USER_AGENT} ^WWWOFFLE[OR]

RewriteCond %{HTTP_USER_AGENT} ^Y!TunnelPro[OR]

RewriteCond %{HTTP_USER_AGENT} ^YahooYSMcm[OR]

RewriteCond %{HTTP_USER_AGENT} ^http://Anonymouse.org/_(Unix)[OR]

RewriteCond %{HTTP_USER_AGENT} ^TMhtload/14.10.1023[OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet_Ninja_5.0[OR]

RewriteCond %{HTTP_USER_AGENT} ^www.adressendeutschland.de [OR]

RewriteCond %{HTTP_USER_AGENT} ^Offline_Downloader_3.5[OR]

RewriteCond %{HTTP_USER_AGENT} ^CFNetwork/129.16 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Mister_Pix_II_2.15 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Master [OR]

RewriteCond %{HTTP_USER_AGENT} ^Irvine/1.1.2 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_1.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_2.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_3.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_4.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_5.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_6.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Ninja_7.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet_Ninja_1.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet_Ninja_2.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet_Ninja_3.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet_Ninja_4.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet_Ninja_5.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet_Ninja_6.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Mister_Pix_II_2.03 [OR]

RewriteCond %{HTTP_USER_AGENT} ^BMLauncher [OR]

RewriteCond %{HTTP_USER_AGENT} ^Z-Add_Link_Checker_(http://w3.z-add.co.uk/linkcheck/) [OR]

RewriteCond %{HTTP_USER_AGENT} ^WebRipper [OR]

RewriteCond %{HTTP_USER_AGENT} ^mysmutsearch/5.0__evaluating_xmlhttp://www.mysmutsearch.com/faq.shtml_ [OR]

RewriteCond %{HTTP_USER_AGENT} ^Downes/Referrers [OR]

RewriteCond %{HTTP_USER_AGENT} ^CFNetwork/129.16 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Jakarta_Commons-HttpClient/3.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^WebSauger_1.20b [OR]

RewriteCond %{HTTP_USER_AGENT} ^Xenu_Link_Sleuth_1.2h [OR]

RewriteCond %{HTTP_USER_AGENT} ^Download_Druid_2.2 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Der_gro\xdfe_BilderSauger_2.00u [OR]

RewriteCond %{HTTP_USER_AGENT} ^DA_7.0 [OR]

RewriteCond %{HTTP_USER_AGENT} ^R\xc3\xa9initialiser [OR]

RewriteCond %{HTTP_USER_AGENT} ^Missigua_Locator_1.9 [OR]

RewriteCond %{HTTP_USER_AGENT} ^NetAnts/1.25 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Webdup/0.9 [OR]

RewriteCond %{HTTP_USER_AGENT} ^WebStripper/2.66 [OR]

RewriteCond %{HTTP_USER_AGENT} ^WebSauger_1.20i[OR]

RewriteCond %{HTTP_USER_AGENT} ^CoralWebPrx/0.1.17_(Seehttp://coralcdn.org/) [OR]

RewriteCond %{HTTP_USER_AGENT} ^Web_Downloader/4.9 [OR]

RewriteCond %{HTTP_USER_AGENT} ^Mister_Pix_II_2.00b [OR]

RewriteCond %{HTTP_USER_AGENT} ^Moozilla [OR]

RewriteCond %{HTTP_USER_AGENT} ^Fuck_off [OR]

RewriteCond %{HTTP_USER_AGENT} ^Teleport [OR]

RewriteCond %{HTTP_USER_AGENT} ^Internet\ Ninja [OR]

RewriteCond %{HTTP_USER_AGENT} ^Acul

========================================

список собирался под конкретные нужды. потому советую проверить нет ли чего лишнего в нем, чтобы кого-нибудь случайно нужного не прикрыть :)

MoMM

cпасибо огромное полестаю.

MoMM, А что всякие там дерьмо граберы уже не умеют маскироваться под ИЕ и Мозиллу ?

Моё имхо ботов можно давить по странам, если сайты на сервере не на азиатских языках, то задавить всю азию и всё что вместе с ней, к примеру. Также погасить число потоков до Максимум 300 (кто в здравом уме будет браузером открывать страницу в 300 потоков), в среднем значения в браузерах не более 20-30. Также можно подгружать листы анонимных прокси и дырявых сетей из спец списков в режиме реального времени и банить по ним.

Боюсь, что парсинг такого списочка при каждом запросе создаст нагрузку намного большую, чем несколько несчастных ботов.

Беда не в парсинге списка, беда в OR и особенно в RewriteCond.

Да я одним словом назвал, чтобы не вдаваться в подробности. Просто сталкивался с аппетитами mod_rewrite.

Ну да. Особенно, если почитать как там механизм преобразований работает. 😮

Дозированно надо подходить к реврайту.

MoMM, скажу по секрету - все нормальные реальные парсеры и граберы ходят под именем гугла или браузера, да и жабаскрипты и т.п. поддерживают, так что этот километровый список не нужен, он создаст значительно большую нагрузку :)