nginx禁止机器人访问网站

追影

1. #禁止Scrapy等工具的抓取
   
2.    if ($http_user_agent ~* (Scrapy|Curl|HttpClient)) {
   
3.      return 403;
   
4.     }
   
5. 
   
6.   #禁止指定UA及UA为空的访问
   
7.    if ($http_user_agent ~* "YandexBot|Bytespider|FeedDemon|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|CoolpadWebkit|Java|Feedly|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|HttpClient|MJ12bot|heritrix|Ezooms|^[        DISCUZ_CODE_0        ]quot; ) {
   
8.      return 403;
   
9.    }
   
10.    #禁止非GET|HEAD|POST方式的抓取
    
11.     if ($request_method !~ ^(GET|HEAD|POST)$) {
    
12.       return 403;
    
13.     }

复制代码

APACHE用rewrite实现：

1. RewriteEngine On
   
2. <blockquote>RewriteCond %{HTTP_USER_AGENT} ^.*(spider|bot|slurp).*$ [NC]

复制代码