用户 → 请求 → 服务器 → 响应 → 解析 → 存储

pip install requests beautifulsoup4 lxml

import requests
from bs4 import BeautifulSoup

# 1. 发送请求
url = "https://example.com"
response = requests.get(url)

# 2. 解析内容
soup = BeautifulSoup(response.text, 'lxml')

# 3. 提取数据
titles = soup.find_all('h2')
for title in titles:
    print(title.text)

import requests
from bs4 import BeautifulSoup
import time

def get_news(page=1):
    url = f"https://news.example.com?page={page}"

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }

    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'lxml')

    news_list = []
    articles = soup.find_all('article')

    for article in articles:
        title = article.find('h2').text
        link = article.find('a')['href']
        news_list.append({
            'title': title,
            'link': link
        })

    return news_list

# 抓取前 5 页
all_news = []
for i in range(1, 6):
    all_news.extend(get_news(i))
    time.sleep(1)  # 礼貌爬取

print(f"共抓取 {len(all_news)} 条新闻")