分享一份 Nginx配置文件,对付爬虫蜘蛛

被蜘蛛爬死了。机子内存爆上去。我都无语。nginx.conf配置一下。放置位置

下面设置就是在Nginx 上禁止他们访问。爬虫那么多。要越过去好会有很多办法的。避免不了,能避免就避免一些。

刚才看好多ip访问,同时几百个。然后机子这几天经常邮件报警。Linode还是很不错的。

if ($http_user_agent ~* "^Mozilla.*Indy"){return 403;}
if ($http_user_agent ~* "^Mozilla.*NEWT"){return 403;}
if ($http_user_agent ~* "^$"){return 403;}
if ($http_user_agent ~* "^Maxthon$"){return 403;}
if ($http_user_agent ~* "^SeaMonkey$"){return 403;}
if ($http_user_agent ~* "^Acunetix"){return 403;}
if ($http_user_agent ~* "^binlar"){return 403;}
if ($http_user_agent ~* "^BlackWidow"){return 403;}
if ($http_user_agent ~* "^Bolt 0"){return 403;}
if ($http_user_agent ~* "^BOT for JCE"){return 403;}
if ($http_user_agent ~* "^Bot mailto:craftbot@yahoo.com"){return 403;}
if ($http_user_agent ~* "^casper"){return 403;}
if ($http_user_agent ~* "^checkprivacy"){return 403;}
if ($http_user_agent ~* "^ChinaClaw"){return 403;}
if ($http_user_agent ~* "^clshttp"){return 403;}
if ($http_user_agent ~* "^cmsworldmap"){return 403;}
if ($http_user_agent ~* "^comodo"){return 403;}
if ($http_user_agent ~* "^Custo"){return 403;}
if ($http_user_agent ~* "^Default Browser 0"){return 403;}
if ($http_user_agent ~* "^diavol"){return 403;}
if ($http_user_agent ~* "^DIIbot"){return 403;}
if ($http_user_agent ~* "^DISCo"){return 403;}
if ($http_user_agent ~* "^dotbot"){return 403;}
if ($http_user_agent ~* "^Download Demon"){return 403;}
if ($http_user_agent ~* "^eCatch"){return 403;}
if ($http_user_agent ~* "^EirGrabber"){return 403;}
if ($http_user_agent ~* "^EmailCollector"){return 403;}
if ($http_user_agent ~* "^EmailSiphon"){return 403;}
if ($http_user_agent ~* "^EmailWolf"){return 403;}
if ($http_user_agent ~* "^Express WebPictures"){return 403;}
if ($http_user_agent ~* "^extract"){return 403;}
if ($http_user_agent ~* "^ExtractorPro"){return 403;}
if ($http_user_agent ~* "^EyeNetIE"){return 403;}
if ($http_user_agent ~* "^feedfinder"){return 403;}
if ($http_user_agent ~* "^FHscan"){return 403;}
if ($http_user_agent ~* "^FlashGet"){return 403;}
if ($http_user_agent ~* "^flicky"){return 403;}
if ($http_user_agent ~* "^g00g1e"){return 403;}
if ($http_user_agent ~* "^GetRight"){return 403;}
if ($http_user_agent ~* "^GetWeb!"){return 403;}
if ($http_user_agent ~* "^Go!Zilla"){return 403;}
if ($http_user_agent ~* "^Go-Ahead-Got-It"){return 403;}
if ($http_user_agent ~* "^grab"){return 403;}
if ($http_user_agent ~* "^GrabNet"){return 403;}
if ($http_user_agent ~* "^Grafula"){return 403;}
if ($http_user_agent ~* "^harvest"){return 403;}
if ($http_user_agent ~* "^HMView"){return 403;}
if ($http_user_agent ~* "^Image Stripper"){return 403;}
if ($http_user_agent ~* "^Image Sucker"){return 403;}
if ($http_user_agent ~* "^InterGET"){return 403;}
if ($http_user_agent ~* "^Internet Ninja"){return 403;}
if ($http_user_agent ~* "^InternetSeer.com"){return 403;}
if ($http_user_agent ~* "^jakarta"){return 403;}
if ($http_user_agent ~* "^Java"){return 403;}
if ($http_user_agent ~* "^JetCar"){return 403;}
if ($http_user_agent ~* "^JOC Web Spider"){return 403;}
if ($http_user_agent ~* "^kanagawa"){return 403;}
if ($http_user_agent ~* "^kmccrew"){return 403;}
if ($http_user_agent ~* "^larbin"){return 403;}
if ($http_user_agent ~* "^LeechFTP"){return 403;}
if ($http_user_agent ~* "^libwww"){return 403;}
if ($http_user_agent ~* "^Mass Downloader"){return 403;}
if ($http_user_agent ~* "^microsoft.url"){return 403;}
if ($http_user_agent ~* "^MIDown tool"){return 403;}
if ($http_user_agent ~* "^miner"){return 403;}
if ($http_user_agent ~* "^Mister PiX"){return 403;}
if ($http_user_agent ~* "^MSFrontPage"){return 403;}
if ($http_user_agent ~* "^Navroad"){return 403;}
if ($http_user_agent ~* "^NearSite"){return 403;}
if ($http_user_agent ~* "^Net Vampire"){return 403;}
if ($http_user_agent ~* "^NetAnts"){return 403;}
if ($http_user_agent ~* "^NetSpider"){return 403;}
if ($http_user_agent ~* "^NetZIP"){return 403;}
if ($http_user_agent ~* "^nutch"){return 403;}
if ($http_user_agent ~* "^Octopus"){return 403;}
if ($http_user_agent ~* "^Offline Explorer"){return 403;}
if ($http_user_agent ~* "^Offline Navigator"){return 403;}
if ($http_user_agent ~* "^PageGrabber"){return 403;}
if ($http_user_agent ~* "^Papa Foto"){return 403;}
if ($http_user_agent ~* "^pavuk"){return 403;}
if ($http_user_agent ~* "^pcBrowser"){return 403;}
if ($http_user_agent ~* "^PeoplePal"){return 403;}
if ($http_user_agent ~* "^planetwork"){return 403;}
if ($http_user_agent ~* "^psbot"){return 403;}
if ($http_user_agent ~* "^purebot"){return 403;}
if ($http_user_agent ~* "^pycurl"){return 403;}
if ($http_user_agent ~* "^RealDownload"){return 403;}
if ($http_user_agent ~* "^ReGet"){return 403;}
if ($http_user_agent ~* "^Rippers 0"){return 403;}
if ($http_user_agent ~* "^sitecheck.internetseer.com"){return 403;}
if ($http_user_agent ~* "^SiteSnagger"){return 403;}
if ($http_user_agent ~* "^skygrid"){return 403;}
if ($http_user_agent ~* "^SmartDownload"){return 403;}
if ($http_user_agent ~* "^sucker"){return 403;}
if ($http_user_agent ~* "^SuperBot"){return 403;}
if ($http_user_agent ~* "^SuperHTTP"){return 403;}
if ($http_user_agent ~* "^Surfbot"){return 403;}
if ($http_user_agent ~* "^tAkeOut"){return 403;}
if ($http_user_agent ~* "^Teleport Pro"){return 403;}
if ($http_user_agent ~* "^Toata dragostea mea pentru diavola"){return 403;}
if ($http_user_agent ~* "^turnit"){return 403;}
if ($http_user_agent ~* "^vikspider"){return 403;}
if ($http_user_agent ~* "^VoidEYE"){return 403;}
if ($http_user_agent ~* "^Web Image Collector"){return 403;}
if ($http_user_agent ~* "^WebAuto"){return 403;}
if ($http_user_agent ~* "^WebBandit"){return 403;}
if ($http_user_agent ~* "^WebCopier"){return 403;}
if ($http_user_agent ~* "^WebFetch"){return 403;}
if ($http_user_agent ~* "^WebGo IS"){return 403;}
if ($http_user_agent ~* "^WebLeacher"){return 403;}
if ($http_user_agent ~* "^WebReaper"){return 403;}
if ($http_user_agent ~* "^WebSauger"){return 403;}
if ($http_user_agent ~* "^Website eXtractor"){return 403;}
if ($http_user_agent ~* "^Website Quester"){return 403;}
if ($http_user_agent ~* "^WebStripper"){return 403;}
if ($http_user_agent ~* "^WebWhacker"){return 403;}
if ($http_user_agent ~* "^WebZIP"){return 403;}
if ($http_user_agent ~* "^Widow"){return 403;}
if ($http_user_agent ~* "^WPScan"){return 403;}
if ($http_user_agent ~* "^WWW-Mechanize"){return 403;}
if ($http_user_agent ~* "^WWWOFFLE"){return 403;}
if ($http_user_agent ~* "^Xaldon WebSpider"){return 403;}
if ($http_user_agent ~* "^Zeus"){return 403;}
if ($http_user_agent ~* "^zmeu"){return 403;}
if ($http_user_agent ~* "360Spider"){return 403;}
if ($http_user_agent ~* "AhrefsBot"){return 403;}
if ($http_user_agent ~* "CazoodleBot"){return 403;}
if ($http_user_agent ~* "discobot"){return 403;}
if ($http_user_agent ~* "EasouSpider"){return 403;}
if ($http_user_agent ~* "ecxi"){return 403;}
if ($http_user_agent ~* "GT::WWW"){return 403;}
if ($http_user_agent ~* "heritrix"){return 403;}
if ($http_user_agent ~* "HTTP::Lite"){return 403;}
if ($http_user_agent ~* "HTTrack"){return 403;}
if ($http_user_agent ~* "ia_archiver"){return 403;}
if ($http_user_agent ~* "id-search"){return 403;}
if ($http_user_agent ~* "IDBot"){return 403;}
if ($http_user_agent ~* "Indy Library"){return 403;}
if ($http_user_agent ~* "IRLbot"){return 403;}
if ($http_user_agent ~* "ISC Systems iRc Search 2.1"){return 403;}
if ($http_user_agent ~* "LinksCrawler"){return 403;}
if ($http_user_agent ~* "LinksManager.com_bot"){return 403;}
if ($http_user_agent ~* "linkwalker"){return 403;}
if ($http_user_agent ~* "lwp-trivial"){return 403;}
if ($http_user_agent ~* "MFC_Tear_Sample"){return 403;}
if ($http_user_agent ~* "Microsoft URL Control"){return 403;}
if ($http_user_agent ~* "Missigua Locator"){return 403;}
if ($http_user_agent ~* "MJ12bot"){return 403;}
if ($http_user_agent ~* "panscient.com"){return 403;}
if ($http_user_agent ~* "PECL::HTTP"){return 403;}
if ($http_user_agent ~* "PHPCrawl"){return 403;}
if ($http_user_agent ~* "PleaseCrawl"){return 403;}
if ($http_user_agent ~* "SBIder"){return 403;}
if ($http_user_agent ~* "SearchmetricsBot"){return 403;}
if ($http_user_agent ~* "SeznamBot"){return 403;}
if ($http_user_agent ~* "Snoopy"){return 403;}
if ($http_user_agent ~* "Steeler"){return 403;}
if ($http_user_agent ~* "URI::Fetch"){return 403;}
if ($http_user_agent ~* "urllib"){return 403;}
if ($http_user_agent ~* "Web Sucker"){return 403;}
if ($http_user_agent ~* "webalta"){return 403;}
if ($http_user_agent ~* "WebCollage"){return 403;}
if ($http_user_agent ~* "Wells Search II"){return 403;}
if ($http_user_agent ~* "WEP Search"){return 403;}
if ($http_user_agent ~* "XoviBot"){return 403;}
if ($http_user_agent ~* "YisouSpider"){return 403;}
if ($http_user_agent ~* "zermelo"){return 403;}
if ($http_user_agent ~* "ZyBorg"){return 403;}

deny 115.239.212.0/24; #屏蔽IP段

评论1

发表评论

电子邮件地址不会被公开。 必填项已用*标注

*