使用“加载更多”按钮抓取无限滚动页面：分步指南

pip install requests beautifulsoup4 selenium

from selenium import webdriverdriver = webdriver.Chrome()  # 确保ChromeDriver已安装并位于路径中driver.get("https://www.scrapingcourse.com/button-click")print(driver.title)driver.quit()

python scraper.py

Load More Button Challenge to Learn Web Scraping - ScrapingCourse.com

import requests# 演示页面的URL，包含产品信息url = "https://www.scrapingcourse.com/button-click"# 发送GET请求到URLresponse = requests.get(url)# 检查请求是否成功if response.status_code == 200:    html_content = response.text    print(html_content)  # 可选：预览HTMLelse:    print(f"未能检索内容：{response.status_code}")

from selenium import webdriverfrom selenium.webdriver.common.by import Byimport time# 设置WebDriver（确保您已安装相应的驱动程序，例如ChromeDriver）driver = webdriver.Chrome()# 打开页面driver.get("https://www.scrapingcourse.com/button-click")# 循环点击“加载更多”按钮，直到没有更多产品while True:    try:        # 通过其ID查找“加载更多”按钮并点击它        load_more_button = driver.find_element(By.ID, "load-more-btn")        load_more_button.click()        # 等待内容加载（根据需要调整时间）        time.sleep(2)    except Exception as e:        # 如果找不到“加载更多”按钮（产品结束），则跳出循环        print("没有更多产品可加载。")        break# 获取加载所有产品后的更新页面内容html_content = driver.page_source# 关闭浏览器窗口driver.quit()

from selenium import webdriverfrom selenium.webdriver.common.by import Byimport time# 实例化一个ChromeOptions对象options = webdriver.ChromeOptions()# 将选项设置为headless模式options.add_argument("--headless=new")# 以headless模式初始化Chrome驱动程序实例driver = webdriver.Chrome(options=options)...

from bs4 import BeautifulSoup# 使用Beautiful Soup解析页面内容soup = BeautifulSoup(html_content, 'html.parser')# 提取产品详细信息products = []# 查找网格中的所有产品项目product_items = soup.find_all('div', class_='product-item')for product in product_items:    # 提取产品名称    name = product.find('span', class_='product-name').get_text(strip=True)    # 提取产品价格    price = product.find('span', class_='product-price').get_text(strip=True)    # 提取产品链接    link = product.find('a')['href']    # 提取图片URL    image_url = product.find('img')['src']    # 创建一个包含产品详细信息的字典    products.append({        'name': name,        'price': price,        'link': link,        'image_url': image_url    })# 打印提取的产品详细信息for product in products[:2]:    print(f"名称：{product['name']}")    print(f"价格：{product['price']}")    print(f"链接：{product['link']}")    print(f"图片URL：{product['image_url']}")    print('-' * 30)

名称：Chaz Kangeroo Hoodie价格：$52链接：https://scrapingcourse.com/ecommerce/product/chaz-kangeroo-hoodie图片URL：https://scrapingcourse.com/ecommerce/wp-content/uploads/2024/03/mh01-gray_main.jpg------------------------------名称：Teton Pullover Hoodie价格：$70链接：https://scrapingcourse.com/ecommerce/product/teton-pullover-hoodie图片URL：https://scrapingcourse.com/ecommerce/wp-content/uploads/2024/03/mh02-black_main.jpg------------------------------…

import csv# 将产品信息写入CSV文件with open("products.csv", mode="w", newline="") as file:    writer = csv.DictWriter(file, fieldnames=["name", "image_url", "price", "link"])    writer.writeheader()    for product in products:        writer.writerow(product)

# 按价格降序排列产品sorted_products = sorted(products, key=lambda x: float(x['price'].replace('$', '')), reverse=True)# 抓取前5个产品的额外详细信息driver = webdriver.Chrome()for product in sorted_products[:5]:    driver.get(product['link'])    time.sleep(3)    soup = BeautifulSoup(driver.page_source, 'html.parser')    description = soup.find('div', class_='product-description')    product['description'] = description.get_text(strip=True) if description else "无描述"    sku = soup.find('span', class_='sku')    product['sku'] = sku.get_text(strip=True) if sku else "无SKU"driver.quit()# 将包含额外数据的产品信息写入CSV文件with open("products.csv", mode="w", newline="") as file:    writer = csv.DictWriter(file, fieldnames=["name", "image_url", "price", "link", "description", "sku"])    writer.writeheader()    for product in products:        writer.writerow({            'name': product['name'],            'image_url': product['image_url'],            'price': product['price'],            'link': product['link'],            'description': product.get('description', ''),            'sku': product.get('sku', '')        })