1.随机请求头

# -*- coding: utf-8 -*-
"""
所有请求头的USER_AGENTS网址
http://www.useragentstring.com/pages/useragentstring.php?name=All
"""
import json
import random
import requestsUSER_AGENTS = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36','Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36','Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML like Gecko) Chrome/44.0.2403.155 Safari/537.36','Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36','Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36 Chrome 41.0.2227.0','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36','Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36','Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36',
]#从列表中随机选择一个
**User_Agent = random.choice(USER_AGENTS)**headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8','Accept-Language': 'en','User-Agent': User_Agent
}def testUserAgent():resp = requests.get("http://httpbin.org/user-agent", headers=headers)user_agent = json.loads(resp.text)['user-agent']#将返回的字符串loads为字典print(user_agent)def testProxy():proxies = {"https": "223.199.25.183:9999"}resp = requests.get("http://httpbin.org/ip", proxies=proxies)origin = json.loads(resp.text)['origin']print(origin)# https://www.xicidaili.com/nn/ 西刺代理
if __name__ == "__main__":testUserAgent()testProxy()

在middleware.py文件进行修改，

#将多个请求头放在一个列表中
User_Agent = ['请求头1'，'请求头2'，'请求头3'，...]
#需要使用的时候从列表中随机抽取一个请求头，代码如下
```python
class user_agentdownloadmiddleware(Object):def process_request(self,request,spider):User_Agent = random.chioce(self.User_Agent)request.headers['User_Agent'] = User_Agent

在setting.py中进行修改

将配置文件中的downloadmiddleware取消注销即可

取消链接去重

def parse():...#此处的dont_filter即为去重的函数，将其设置为True则表示不要去重复的链接进行去重，即可以对同一个链接进行反复地请求并且获得数据yield scrapy.Request(self.start_urls[0],dont_filter = True)...