【数据挖掘】二手车交易价格预测大赛-探索性数据分析（EDA）

2.【数据挖掘】二手车交易价格预测大赛-探索性数据分析（EDA）

2.1 什么是探索性数据分析

探索性数据分析(EDA ，Exploratory Data Analysis)，是指对已有的数据（特别是调查或观察得来的原始数据）在尽量少的先验假定下进行探索，通过作图、制表、方程拟合、计算特征量等手段探索数据的结构和规律的一种数据分析方法。

2.2 本案例的探索性分析流程

实验工具：Jupyter Notebook

（1）导入各种数据科学库以及可视化库

#coding:utf-8
import warnings
warnings.filterwarnings('ignore') # 导入warnings包，利用过滤器来实现忽略警告语句

import numpy as np
import pandas as pd 
import matplotlib.pyplot as plt
import seaborn as sns
import missingno as msno

（2）载入数据

Train_data = pd.read_csv('data/train.csv', sep=' ')
Test_data = pd.read_csv('data/testA.csv', sep=' ')

（3）查看数据基本情况

Train_data.head().append(Train_data.tail())  # 简要查看前五行和后五行；
Train_data.shape 

Test_data.head().append(Test_data.tail())
Test_data.shape

Train_data.describe()
Test_data.describe()

Train_data.info()  # 查看数据的基本信息
Test_data.info()

（4）判断数据异常情况

1. 缺失值总数统计

Train_data.isnull().sum()  # 缺失值总数统计
Test_data.isnull().sum()

2. nan可视化

missing = Train_data.isnull().sum()  
missing = missing[missing > 0]
missing.sort_values(inplace=True)
missing.plot.bar()

msno.matrix(Train_data.sample(250))
msno.bar(Train_data.sample(1000))

msno.matrix(Test_data.sample(250))
msno.bar(Test_data.sample(1000))

3. 查看异常值

   通过以上返回结果知notRepairedDamage列需要纠正。

Train_data['notRepairedDamage'].value_counts()  # 对该列的值进行统计
Train_data['notRepairedDamage'].replace('-', np.nan, inplace=True)  # 对未知值’-‘用nan表示
Train_data['notRepairedDamage'].value_counts()  # 查看修改后结果

Test_data['notRepairedDamage'].value_counts()
Test_data['notRepairedDamage'].replace('-', np.nan, inplace=True)

4. 清理严重倾斜的数据

Test_data['seller'].value_counts()
Train_data['offerType'].value_counts()

del Train_data['seller']
del Train_data['offerType']

del Test_data['seller']
del Test_data['offerType']

（5）了解预测值分布情况

Train_data['price']  # 数据price列
Train_data['price'].value_counts()

1. 总体分布情况

import scipy.stats as st

y = Train_data['price']
plt.figure(1); plt.title('Johnson SU')
sns.distplot(y, kde=False, fit=st.johnsonsu)  #无界约翰逊分布
plt.figure(2); plt.title('Normal')
sns.distplot(y, kde=False, fit=st.norm)
plt.figure(3); plt.title('Log Normal')
sns.distplot(y, kde=False, fit=st.lognorm)

2. 查看skewness and kurtosis,即偏度与峰值

sns.distplot(Train_data['price']);
print("Skewness: %f" % Train_data['price'].skew())
print("Kurtosis: %f" % Train_data['price'].kurt())

Train_data.skew(), Train_data.kurt()
sns.distplot(Train_data.skew(),color='blue',axlabel ='Skewness')
sns.distplot(Train_data.kurt(),color='blue',axlabel ='Skewness')

3. 查看预测值具体频数

plt.hist(Train_data['price'], orientation = 'vertical',histtype = 'bar', color ='red')
plt.show()

（6）特征分析

特征分为类别特征和数字特征

Y_train = Train_data['price']

numeric_features = ['power', 'kilometer', 'v_0', 'v_1', 'v_2', 'v_3', 
                    'v_4', 'v_5', 'v_6', 'v_7', 'v_8', 'v_9', 'v_10', 'v_11', 'v_12', 'v_13','v_14' ]
categorical_features = ['name', 'model', 'brand', 'bodyType', 'fuelType',
                        'gearbox', 'notRepairedDamage', 'regionCode',]
                        
                        
for cat_fea in categorical_features:
    print(cat_fea + "的特征分布如下：")
    print("{}特征有个{}不同的值".format(cat_fea, Train_data[cat_fea].nunique()))
    print(Train_data[cat_fea].value_counts())

1）数字特征分析

numeric_features.append('price')
Train_data.head()

1. 相关性分析

price_numeric = Train_data[numeric_features]
correlation = price_numeric.corr()
print(correlation['price'].sort_values(ascending = False),'\n')

f , ax = plt.subplots(figsize = (7, 7))
plt.title('Correlation of Numeric Features with Price',y=1,size=16)
sns.heatmap(correlation,square = True,  vmax=0.8)

del price_numeric['price']

2. 查看几个特征的偏度与峰值

for col in numeric_features:
    print('{:15}'.format(col), 
          'Skewness: {:05.2f}'.format(Train_data[col].skew()) , 
          '   ' ,
          'Kurtosis: {:06.2f}'.format(Train_data[col].kurt())  
         )

3. 每个数字特征分布可视化

f = pd.melt(Train_data, value_vars=numeric_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False)
g = g.map(sns.distplot, "value")

4. 数字特征相互之间的关系可视化

sns.set()
columns = ['price', 'v_12', 'v_8' , 'v_0', 'power', 'v_5',  'v_2', 'v_6', 'v_1', 'v_14']
sns.pairplot(Train_data[columns],size = 2 ,kind ='scatter',diag_kind='kde')
plt.show()

5. 多变量互相回归关系可视化

fig, ((ax1, ax2), (ax3, ax4), (ax5, ax6), (ax7, ax8), (ax9, ax10))= plt.subplots(nrows=5, ncols=2, figsize=(24, 20))

v_12_scatter_plot = pd.concat([Y_train,Train_data['v_12']],axis = 1)
sns.regplot(x='v_12',y = 'price', data = v_12_scatter_plot,scatter= True, fit_reg=True, ax=ax1)

v_8_scatter_plot = pd.concat([Y_train,Train_data['v_8']],axis = 1)
sns.regplot(x='v_8',y = 'price',data = v_8_scatter_plot,scatter= True, fit_reg=True, ax=ax2)

v_0_scatter_plot = pd.concat([Y_train,Train_data['v_0']],axis = 1)
sns.regplot(x='v_0',y = 'price',data = v_0_scatter_plot,scatter= True, fit_reg=True, ax=ax3)

power_scatter_plot = pd.concat([Y_train,Train_data['power']],axis = 1)
sns.regplot(x='power',y = 'price',data = power_scatter_plot,scatter= True, fit_reg=True, ax=ax4)

v_5_scatter_plot = pd.concat([Y_train,Train_data['v_5']],axis = 1)
sns.regplot(x='v_5',y = 'price',data = v_5_scatter_plot,scatter= True, fit_reg=True, ax=ax5)

v_2_scatter_plot = pd.concat([Y_train,Train_data['v_2']],axis = 1)
sns.regplot(x='v_2',y = 'price',data = v_2_scatter_plot,scatter= True, fit_reg=True, ax=ax6)

v_6_scatter_plot = pd.concat([Y_train,Train_data['v_6']],axis = 1)
sns.regplot(x='v_6',y = 'price',data = v_6_scatter_plot,scatter= True, fit_reg=True, ax=ax7)

v_1_scatter_plot = pd.concat([Y_train,Train_data['v_1']],axis = 1)
sns.regplot(x='v_1',y = 'price',data = v_1_scatter_plot,scatter= True, fit_reg=True, ax=ax8)

v_14_scatter_plot = pd.concat([Y_train,Train_data['v_14']],axis = 1)
sns.regplot(x='v_14',y = 'price',data = v_14_scatter_plot,scatter= True, fit_reg=True, ax=ax9)

v_13_scatter_plot = pd.concat([Y_train,Train_data['v_13']],axis = 1)
sns.regplot(x='v_13',y = 'price',data = v_13_scatter_plot,scatter= True, fit_reg=True, ax=ax10)

2）类别特征分析

1. unique分布

for fea in categorical_features:
    print(Train_data[fea].nunique())

2. 类型特征箱图可视化

categorical_features = ['model',
 'brand',
 'bodyType',
 'fuelType',
 'gearbox',
 'notRepairedDamage']
for c in categorical_features:
    Train_data[c] = Train_data[c].astype('category')
    if Train_data[c].isnull().any():
        Train_data[c] = Train_data[c].cat.add_categories(['MISSING'])
        Train_data[c] = Train_data[c].fillna('MISSING')

def boxplot(x, y, **kwargs):
    sns.boxplot(x=x, y=y)
    x=plt.xticks(rotation=90)

f = pd.melt(Train_data, id_vars=['price'], value_vars=categorical_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False, size=5)
g = g.map(boxplot, "value", "price")

3. 类别特征的小提琴图可视化

catg_list = categorical_features
target = 'price'
for catg in catg_list :
    sns.violinplot(x=catg, y=target, data=Train_data)
    plt.show()

4. 类别特征的柱形图可视化

def bar_plot(x, y, **kwargs):
    sns.barplot(x=x, y=y)
    x=plt.xticks(rotation=90)

f = pd.melt(Train_data, id_vars=['price'], value_vars=categorical_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False, size=5)
g = g.map(bar_plot, "value", "price")

5. 类别特征的每个类别频数可视化

def count_plot(x,  **kwargs):
    sns.countplot(x=x)
    x=plt.xticks(rotation=90)

f = pd.melt(Train_data,  value_vars=categorical_features)
g = sns.FacetGrid(f, col="variable",  col_wrap=2, sharex=False, sharey=False, size=5)
g = g.map(count_plot, "value")

（6）生成数据报告

import pandas_profiling

pfr = pandas_profiling.ProfileReport(Train_data)
pfr.to_file("./example.html")

2.3 附录

遇到的问题以及解决方法：

Q1:

安装pandas-profiling库

A1:

• 终端输入pip install pandas-profiling[notebook,html]

参考链接：pandas-profiling

---

Q2:

返回错误：ERROR: Cannot uninstall 'wrapt'. It is a distutils installed project and thus we cannot accurately determine which files belong to it which would lead to only a partial uninstall.

A2:

• 终端输入 pip install -U --ignore-installed wrapt enum34 simplejson netaddr

---

Q3:

返回错误：ImportError: Numpy version 1.16.0 or later must be installed to use Astropy.

[外链图片转存失败,源站可能有防盗链机制,建议将图片保存下来直接上传(img-pyXftmV9-1585043230056)(image/image-20200324161048300.png)]

A3:

• 输入numpy.version.version查看中numpy的版本；
• （解决后更新）

​