如何使用 Pandas 进行数据清洗？如何保证数据清洗的效果？

import pandas as pd
import numpy as np

# 读取数据
df = pd.read_csv('data.csv')
print(df.head())

# 查看基本信息
print(df.info())
# 查看描述性统计信息
print(df.describe())
# 检查缺失值
print(df.isnull().sum())

# 删除含有缺失值的行
df = df.dropna()
# 删除含有缺失值的列
df = df.dropna(axis=1)
填充缺失值
# 用均值填充数值列的缺失值
df['Age'] = df['Age'].fillna(df['Age'].mean())
# 用众数填充分类列的缺失值
df['Gender'] = df['Gender'].fillna(df['Gender'].mode()[0])
# 用特定值填充
df['Income'] = df['Income'].fillna(0)
# 使用前一个值填充
df['Salary'] = df['Salary'].fillna(method='ffill')
# 使用后一个值填充
df['Salary'] = df['Salary'].fillna(method='bfill')
# 使用插值方法填充
df['Temperature'] = df['Temperature'].interpolate()

# 移除年龄大于100岁的记录
df = df[df['Age'] <= 100]
使用 Z-score 方法
from scipy import stats
# 计算 Z-score
z_scores = np.abs(stats.zscore(df.select_dtypes(include=[np.number])))
# 移除 Z-score 大于 3 的记录
df = df[(z_scores < 3).all(axis=1)]
使用 IQR 方法
Q1 = df.quantile(0.25)
Q3 = df.quantile(0.75)
IQR = Q3 - Q1
# 移除 IQR 范围外的记录
df = df[~((df < (Q1 - 1.5 * IQR)) | (df > (Q3 + 1.5 * IQR))).any(axis=1)]

# 检查并删除重复记录
df = df.drop_duplicates()

# 将字符串转换为日期
df['Date'] = pd.to_datetime(df['Date'])
# 将对象类型转换为数值类型
df['Age'] = pd.to_numeric(df['Age'], errors='coerce')
# 将数值类型转换为类别类型
df['Category'] = df['Category'].astype('category')

# 将所有文本转换为小写
df['Name'] = df['Name'].str.lower()
# 去除空格
df['Name'] = df['Name'].str.strip()
替换特定值
# 替换特定值
df['City'] = df['City'].replace({'New York City': 'New York', 'LA': 'Los Angeles'})

# 去除特殊字符
df['Comment'] = df['Comment'].str.replace('[^\w\s]', '', regex=True)

# 提取年份、月份、日
df['Year'] = df['Date'].dt.year
df['Month'] = df['Date'].dt.month
df['Day'] = df['Date'].dt.day

# 保存清洗后的数据
df.to_csv('cleaned_data.csv', index=False)

# 查看基本信息
print(df.info())
# 查看描述性统计信息
print(df.describe())
# 检查缺失值
print(df.isnull().sum())
# 可视化数据分布
import matplotlib.pyplot as plt
df['Age'].hist(bins=20)
plt.show()

# 记录每一步操作
with open('data_cleaning_log.txt', 'w') as f:
    f.write("Data Cleaning Log:\n")
    f.write(f"Initial shape: {df.shape}\n")
    # 示例：处理缺失值
    df = df.dropna()
    f.write(f"After dropping missing values: {df.shape}\n")
    # 示例：处理异常值
    df = df[df['Age'] <= 100]
    f.write(f"After removing outliers: {df.shape}\n")
    # 其他步骤...

# 第一阶段：处理缺失值
df = df.dropna()
# 验证结果
print(df.isnull().sum())
# 第二阶段：处理异常值
df = df[df['Age'] <= 100]
# 验证结果
print(df['Age'].describe())

# 断言
assert df.isnull().sum().sum() == 0, "There are still missing values in the dataset"
assert (df['Age'] > 100).sum() == 0, "There are still age values greater than 100"
# 单元测试
import unittest
class TestDataCleaning(unittest.TestCase):
    def test_missing_values(self):
        self.assertEqual(df.isnull().sum().sum(), 0)
    def test_outliers(self):
        self.assertEqual((df['Age'] > 100).sum(), 0)
if __name__ == '__main__':
    unittest.main(argv=['first-arg-is-ignored'], exit=False)

# 定期复查数据
def check_data_quality(df):
    print("Checking data quality...")
    print("Missing values:", df.isnull().sum())
    print("Outliers in Age:", (df['Age'] > 100).sum())
check_data_quality(df)

# 初始化 Git 仓库
git init
# 添加文件
git add data_cleaning_script.py
git add data_cleaning_log.txt
# 提交更改
git commit -m "Initial data cleaning script and log"