使用 Beautiful Soup 和 Scrapy 进行网页抓取：高效、负责任地提取数据

pip install beautifulsoup4 requests

import requestsfrom bs4 import BeautifulSoupurl = 'https://example-blog.com'response = requests.get(url)if response.status_code == 200:    soup = BeautifulSoup(response.text, 'html.parser')    titles = soup.find_all('h1', class_='entry-title')    if titles:        for title in titles:            print(title.get_text(strip=True))    else:        print("未找到标题，请检查HTML结构和选择器。")else:    print(f"页面获取失败，状态码：{response.status_code}")

pip install scrapy

scrapy startproject quotes_scrapercd quotes_scraper

import scrapyclass QuotesSpider(scrapy.Spider):    name = 'quotes'    start_urls = ['http://quotes.toscrape.com']    def parse(self, response):        for quote in response.css('div.quote'):            yield {                'text': quote.css('span.text::text').get(),                'author': quote.css('small.author::text').get(),            }        next_page = response.css('li.next a::attr(href)').get()        if next_page:            yield response.follow(next_page, callback=self.parse)

scrapy crawl quotes -o quotes.json