Python - AIT Notebook

工具：Anaconda3 + Python

Part01：读取文件

1️⃣ 导入Pandas库并设置别名

import pandas as pd

2️⃣ 数据加载到DataFrame变量

import pandas as pd

# test_1.csv
data1 = pd.read_csv('test_1.csv')

# test_2.xlsx
data2 = pd.read_excel('test_2.xlsx')

3️⃣ 读取数据的基本操作

import pandas as pd

data = pd.read_csv('test.csv')
# 读取文件前五行
print(data.head())
# 读取表结构的基本信息
print(data.info())
# 查看文件行数
print(data.shape[0])

Part02：数据清洗

1️⃣ 缺失值

# 检查缺失值
missing_sum = df.isnull().sum()
missing_percent = (df.isnull().sum() / len(df)) * 100
missing_info = pd.DataFrame({
    '缺失数量': missing_sum,
    '缺失比例%': missing_percent
})
print(missing_info[missing_info['缺失数量'] > 0])


# 可视化缺失值
import matplotlib.pyplot as plt
import seaborn as sns
plt.figure(figsize=(10, 6))
sns.heatmap(df.isnull(), cbar=False, cmap='viridis')
plt.title('缺失值热图')
plt.show()

# 缺失值处理策略
def handle_missing_values(df):
    """
    处理缺失值的多种策略
    """
    df_clean = df.copy()
    
    # 1. 删除缺失值过多的列
    threshold = 0.7  # 缺失比例超过70%的列删除
    cols_to_drop = missing_percent[missing_percent > threshold].index
    df_clean.drop(columns=cols_to_drop, inplace=True)
    
    # 2. 数值型列填充
    numeric_cols = df_clean.select_dtypes(include=['number']).columns
    for col in numeric_cols:
        if df_clean[col].isnull().any():
            # 根据数据分布选择填充方法
            if df_clean[col].skew() > 1:  # 偏态分布用中位数
                df_clean[col].fillna(df_clean[col].median(), inplace=True)
            else:  # 对称分布用均值
                df_clean[col].fillna(df_clean[col].mean(), inplace=True)
    
    # 3. 分类列填充
    categorical_cols = df_clean.select_dtypes(include=['object', 'category']).columns
    for col in categorical_cols:
        if df_clean[col].isnull().any():
            df_clean[col].fillna(df_clean[col].mode()[0], inplace=True)
    
    # 4. 时间序列填充
    datetime_cols = df_clean.select_dtypes(include=['datetime']).columns
    for col in datetime_cols:
        if df_clean[col].isnull().any():
            df_clean[col].fillna(method='ffill', inplace=True)
    
    # 5. 删除剩余的行缺失
    df_clean.dropna(inplace=True)
    
    return df_clean

df_clean = handle_missing_values(df)

2️⃣ 重复值

# 检查重复值
duplicate_rows = df.duplicated().sum()
print(f"完全重复的行数: {duplicate_rows}")

# 基于关键列的重复检查
key_columns = ['user_id', 'timestamp']
duplicate_key = df.duplicated(subset=key_columns).sum()
print(f"基于关键列 {key_columns} 的重复: {duplicate_key}")

# 删除重复值
df_dedup = df.drop_duplicates(
    subset=key_columns,  # 基于哪些列检查重复
    keep='first',        # 保留第一个出现的数据
    inplace=False
)

# 高级重复处理：考虑时间窗口
def remove_temporal_duplicates(df, id_col, time_col, window='1H'):
    """
    在时间窗口内去除重复记录
    """
    df = df.sort_values([id_col, time_col])
    df['time_diff'] = df.groupby(id_col)[time_col].diff()
    # 删除时间差小于窗口的重复记录
    df = df[~(df['time_diff'] < pd.Timedelta(window))]
    return df.drop(columns=['time_diff'])

3️⃣ 数据类型转换

# Age数据类型转换为int
data['Age'] = data['Age'].astype(int)

4️⃣ 列名修改

data = data.rename(columns:{'病人ID':'患者ID'},inplace=True)

未完待续...