Pandas数据清洗

1、处理缺失数据

import pandas as pd
log_data = pd.read_csv('log.csv')
log_data
判断数据缺失
print(log_data.isnull())
print(log_data['paused'].isnull())
# 取出volume不为空的数据
print(log_data[log_data['volume'].notnull()])
log_data.set_index(['time', 'user'], inplace=True)
log_data.sort_index(inplace=True)
print(log_data)
print(log_data.fillna(0))
print(log_data.dropna())
print(log_data.ffill())
print(log_data.bfill())

2、数据变形

处理重复数据
data = pd.DataFrame({'k1': ['one', 'two'] * 3 + ['two'],
                     'k2': [1, 1, 2, 3, 3, 4, 4]})
print(data)
# 判断数据是否重复
data.duplicated()
# 去除重复数据
data.drop_duplicates()
data['v1'] = range(7)
print(data)
# 去除指定列的重复数据
data.drop_duplicates(['k1'])
data.drop_duplicates(['k1', 'k2'], keep='last')

使用函数或map转化数据
data = pd.DataFrame({'food': ['bacon', 'pulled pork', 'bacon', 'Pastrami', 'corned beef', 'Bacon', 'pastrami', 'honey ham', 'nova lox'],
                     'ounces': [4, 3, 12, 6, 7.5, 8, 3, 5, 6]})
print(data)
# 添加一列，用于指定食物的来源
meat_to_animal = {
    'bacon': 'pig',
    'pulled pork': 'pig',
    'pastrami': 'cow',
    'corned beef': 'cow',
    'honey ham': 'pig',
    'nova lox': 'salmon'
}
# 使用map()
lowercased = data['food'].str.lower()
data['animal'] = lowercased.map(meat_to_animal)
print(data)
# 使用方法
data['animal2'] = data['food'].map(lambda x: meat_to_animal[x.lower()])
print(data)

替换值
data = pd.Series([1., -999., 2., -999., -1000., 3.])
print(data)
import numpy as np
# 将-999替换为空值
data.replace(-999, np.nan)
# 将-999，-1000都替换为空值
data.replace([-999, -1000], np.nan)
# 将-999，-1000分别替换为空值和0
data.replace([-999, -1000], [np.nan, 0])
data.replace({-999: np.nan, -1000: 0})

离散化和分箱操作
# 年龄数据
ages = [20, 22, 25, 27, 21, 23, 37, 31, 61, 45, 41, 32]
# 分箱的边界
bins = [18, 25, 35, 60, 100]
cats = pd.cut(ages, bins)
print(type(cats))
# Categorical对象
print(cats)
# 获取分箱编码
print(cats.codes)
# 返回分箱边界索引
print(cats.categories)
# 统计箱中元素的个数
pd.value_counts(cats)
# 带标签的分箱
group_names = ['Youth', 'YoungAdult', 'MiddleAged', 'Senior']
cats = pd.cut(ages, bins, labels=group_names)
print(cats.get_values())

哑变量操作
df = pd.DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'], 
                   'data1': range(6)})
print(df)
pd.get_dummies(df['key'])

向量化字符串操作
data = {'Dave': '[email protected]', 'Steve': '[email protected]', 'Rob': '[email protected]', 'Wes': np.nan}
data = pd.Series(data)
print(data)
print(data.str.contains('gmail'))
print(data.str[:5])
split_df = data.str.split('@', expand=True)
print(split_df)
split_df[0].str.cat(split_df[1], sep='@')
print(split_df)