scrapy爬虫库入门教程

pip install scrapy

scrapy startproject your_project

from scrapy.spiders import Spiderfrom scrapy.http import Requestfrom scrapy.selector import Selectorclass MySpider(Spider):    name = "myspider"    allowed_domains = ["example.com"]    start_urls = ["https://example.com"]    def parse(self, response):        selector = Selector(response)        item = {            'title':  selector.xpath('//title/text()').extract(),            'content': selector.xpath('//p/text()').extract(),        }        yield item

scrapy crawl myspider