【Python - 100天从新手到大师】——pandas数据清洗之空数据

原文链接：https://github.com/jackfrued/Python-100-Days

【Python - 100天从新手到大师】

import numpy as np

import pandas as pd

from pandas import Series,DataFrame

df = DataFrame(np.random.randint(0,150,size = (100,5)),index = np.arange(100,200),
								columns=['Python','En','Math','Physic','Chem'])
df

# 判断DataFrame是否存在空数据
df.isnull().any()

df.notnull().all()

100*5

500

for i in range(50):
    # 行索引
    index = np.random.randint(100,200,size =1)[0]

    cols = df.columns

    # 列索引
    col = np.random.choice(cols)

    df.loc[index,col] = None

for i in range(20):
    # 行索引
    index = np.random.randint(100,200,size =1)[0]

    cols = df.columns

    # 列索引
    col = np.random.choice(cols)

	# not a number 不是一个数
    df.loc[index,col] = np.NAN

df

df.isnull().any()

df.isnull().sum()

df2 = df.copy()

df2.isnull().sum()

# 固定值填充
df2.fillna(value=100)

df2.mean()

# 均值
df3 = df2.fillna(value=df2.mean())
df3.astype(np.int16)

nd = np.random.randint(0,20,size = 10)
nd

nd.sort()
nd

(13 + 16)/2

14.5

np.median(nd)

14.5

# 中位数填充
df2.median()
df4 = df2.fillna(df2.median())
df4

# 众数填充，数量最多的那个数
df2

df = DataFrame(np.random.randint(0,150,size = (2000,5)),index = np.arange(100,2100),columns=['Python','En','Math','Physic','Chem'])
df

for i in range(1000):
    # 行索引
    index = np.random.randint(100,2100,size =1)[0]

    cols = df.columns

    # 列索引
    col = np.random.choice(cols)

    df.loc[index,col] = None

df.isnull().sum()

df.head()

df.tail()

# 去重之后的数据
df['Python'].unique()

df['Python'].value_counts()

en = df['En'].value_counts()
en

en.index[0]

s = df.median()
print(s,type(s))

zhongshu = []
for col in df.columns:
    zhongshu.append(df[col].value_counts().index[0])

s = Series(zhongshu,index = df.columns)
s

df2 = df.fillna(s)
df2

df2.isnull().sum()

df.isnull().sum()

df3 = df.iloc[:20]
df3

'''method : {'backfill', 'bfill', 'pad', 'ffill', None}, default None
    Method to use for filling holes in reindexed Series
    pad / ffill: propagate last valid observation forward to next valid
    backfill / bfill: use NEXT valid observation to fill gap'''
df3.fillna(method='bfill',axis = 1)

#数据量足够大，空数据比较少，直接删除
df.shape