pandas 数据处理 10 个高效技巧，让分析速度提升 5 倍

import pandas as pd
import numpy as np

df = pd.DataFrame({'value': np.random.randn(1_000_000)})

# 慢：逐行 apply，约 1.2s
df['flag_apply'] = df['value'].apply(lambda x: 1 if x > 0 else 0)

# 快：向量化，约 8ms
df['flag_vec'] = np.where(df['value'] > 0, 1, 0)

# 可读性好且略快
result = df.query('value > 0 and category == "A"')

df['city'] = df['city'].astype('category')
# 100 万行字符串列，内存从 64MB 降到 1MB
print(df.memory_usage(deep=True))

chunks = []
for chunk in pd.read_csv('big.csv', chunksize=100_000):
    chunk = chunk[chunk['amount'] > 0]
    chunks.append(chunk.groupby('user_id')['amount'].sum())
result = pd.concat(chunks).groupby(level=0).sum()

df = pd.read_csv('data.csv', dtype={
    'user_id': 'int32',
    'amount': 'float32',
    'status': 'category'
})

mapping = {1: '待付款', 2: '已付款', 3: '已发货'}
df['status_name'] = df['status'].map(mapping)

summary = df.groupby('category').agg(
    total=('amount', 'sum'),
    avg=('amount', 'mean'),
    cnt=('order_id', 'count')
).reset_index()

# 先设置索引再 join，比 merge 快
df1 = df1.set_index('user_id')
df2 = df2.set_index('user_id')
result = df1.join(df2, how='left')

# 错误：可能触发 SettingWithCopyWarning
df[df.age > 30]['score'] = 100

# 正确
df.loc[df.age > 30, 'score'] = 100

# 大表上比普通写法快 2 倍
df.eval('total = price * quantity * (1 - discount)', inplace=True)