在Scrapy爬虫中使用代理IP和反爬虫策略

DOWNLOADER_MIDDLEWARES = {    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 1,    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,    'your_project.middlewares.RandomUserAgentMiddleware': 400,    'your_project.middlewares.RandomProxyMiddleware': 410,}

import requestsimport randomclass RandomProxyMiddleware(object):    def __init__(self, proxy_list_path):        with open(proxy_list_path, 'r') as f:            self.proxy_list = f.readlines()    @classmethod    def from_crawler(cls, crawler):        settings = crawler.settings        return cls(settings.get('PROXY_LIST_PATH'))    def process_request(self, request, spider):        proxy = random.choice(self.proxy_list).strip()        request.meta['proxy'] = "http://" + proxy

PROXY_LIST_PATH = 'path/to/your/proxy/list'

import randomfrom scrapy.downloadermiddlewares.useragent import UserAgentMiddlewareclass RandomUserAgentMiddleware(UserAgentMiddleware):    def __init__(self, user_agent):        self.user_agent = user_agent    @classmethod    def from_crawler(cls, crawler):        s = cls(crawler.settings.get('user_agent', 'Scrapy'))        crawler.signals.connect(s.spider_closed, signal=signals.spider_closed)        return s    def process_request(self, request, spider):        ua = random.choice(self.user_agent_list)        if ua:            request.headers.setdefault('User-Agent', ua)

USER_AGENT_LIST = ['Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36']

COOKIES_ENABLED = TrueCOOKIES_DEBUG = True

from scrapy.exceptions import IgnoreRequestclass CookieMiddleware(object):    def __init__(self, cookies):        self.cookies = cookies    @classmethod    def from_crawler(cls, crawler):        return cls(            cookies=crawler.settings.getdict('COOKIES')        )    def process_request(self, request, spider):        request.cookies.update(self.cookies)

COOKIES = {    'cookie1': 'value1',    'cookie2': 'value2',    ...}