用 Python 进行高效数据抓取的七招

import requests  

# 发送GET请求  
response = requests.get('https://api.example.com/data')  

# 检查请求是否成功  
if response.status_code == 200:  
    print("请求成功！")  
    data = response.json()  # 将响应内容解析为JSON  
    print(data)  
else:  
    print(f"请求失败，状态码：{response.status_code}")  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.

from bs4 import BeautifulSoup  
import requests  

url = 'https://example.com'  
response = requests.get(url)  
soup = BeautifulSoup(response.content, 'html.parser')  

# 查找所有标题  
titles = soup.find_all('h1')  
for title in titles:  
    print(title.get_text())  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

import requests  

base_url = 'https://example.com/page/'  

# 假设共有10页  
for page in range(1, 11):  
    url = f"{base_url}{page}"  
    response = requests.get(url)  
    if response.status_code == 200:  
        print(f"抓取第{page}页数据")  
        # 处理数据...  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

# scrapy.cfg  
# [settings]  
# default = myproject.settings  

# myproject/settings.py  
# BOT_NAME = 'myproject'  
# SPIDER_MODULES = ['myproject.spiders']  
# NEWSPIDER_MODULE = 'myproject.spiders'  

# myproject/spiders/example_spider.py  
import scrapy  

class ExampleSpider(scrapy.Spider):  
    name = 'example'  
    start_urls = ['https://example.com']  

    def parse(self, response):  
        for item in response.css('div.item'):  
            yield {  
                'title': item.css('h2::text').get(),  
                'link': item.css('a::attr(href)').get(),  
            }  

        # 处理分页  
        next_page = response.css('a.next::attr(href)').get()  
        if next_page:  
            yield response.follow(next_page, self.parse)  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.

from selenium import webdriver  
from selenium.webdriver.common.by import By  

driver = webdriver.Chrome()  
driver.get('https://example.com')  

# 等待页面加载  
element = driver.find_element(By.ID, 'some-element-id')  
print(element.text)  

driver.quit()  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

import pandas as pd  

# 假设我们有一个CSV文件  
df = pd.read_csv('data.csv')  

# 查看前几行数据  
print(df.head())  

# 数据清洗，例如删除缺失值  
df_cleaned = df.dropna()  

# 数据分析，例如计算平均值  
print(df_cleaned['column_name'].mean())  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.

import requests  

url = 'https://example.com/robots.txt'  
response = requests.get(url)  

if response.status_code == 200:  
    robots_txt = response.text  
    print(robots_txt)  
    # 解析robots.txt，判断是否可以抓取  
else:  
    print("无法获取robots.txt文件")  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.

import requests  
from bs4 import BeautifulSoup  
import pandas as pd  

base_url = 'https://example.com/page/'  
all_data = []  

# 抓取所有页面数据  
for page in range(1, 11):  # 假设共10页  
    url = f"{base_url}{page}"  
    response = requests.get(url)  
    if response.status_code == 200:  
        soup = BeautifulSoup(response.content, 'html.parser')  
        items = soup.find_all('div', class_='item')  
        for item in items:  
            title = item.find('h2').get_text()  
            price = item.find('span', class_='price').get_text()  
            link = item.find('a')['href']  
            all_data.append({  
                'title': title,  
                'price': price,  
                'link': link  
            })  

# 将数据转换为DataFrame  
df = pd.DataFrame(all_data)  

# 保存为CSV文件  
df.to_csv('products.csv', index=False)  

# 查看前几行数据  
print(df.head())  
1.
2.
3.
4.
5.
6.
7.
8.
9.
10.
11.
12.
13.
14.
15.
16.
17.
18.
19.
20.
21.
22.
23.
24.
25.
26.
27.
28.
29.
30.
31.
32.