数据预处理2——数据集成

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/Vinsuan1993/article/details/80040628

一、介绍

    数据挖掘需要的数据往往分布在不同的数据集中，数据集成就是将多个数据集合并放在一个一致的数据存储中的过程。

对于DataFrame来说，它的连接有时候是通过索引的。

三、代码示例

# coding: utf-8

# In[2]:


from pandas import DataFrame  
import pandas as pd
import numpy as np
###dataframe合并
#1
df1 = DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'a', 'b'],'data1': range(7)})
df2 = DataFrame({'key': ['a', 'b', 'd'],'data2': range(3)})
df1


# In[8]:


df2


# In[9]:


pd.merge(df1,df2)


# In[10]:


#默认重叠列名当作键，如果有多个重叠的列名，就会将多个列名当作键；
#如果不希望把多个列名当作键，可以用on指定。
pd.merge(df1,df2,on='key')


# In[12]:


df3 = DataFrame({'lkey': ['b', 'b', 'a', 'c', 'a', 'a', 'b'],
                 'data1': range(7)})
df4 = DataFrame({'rkey': ['a', 'b', 'd'],
                 'data2': range(3)})
df3


# In[13]:


df4


# In[14]:


#默认内连接，取交集
pd.merge(df3, df4, left_on='lkey', right_on='rkey')


# In[15]:


pd.merge(df1,df2,how='outer')


# In[16]:


df1 = DataFrame({'key': ['b', 'b', 'a', 'c', 'a', 'b'],
                 'data1': range(6)})
df2 = DataFrame({'key': ['a', 'b', 'a', 'b', 'd'],
                 'data2': range(5)})
df1


# In[17]:


df2


# In[18]:


pd.merge(df1, df2, on='key', how='left')


# In[19]:


pd.merge(df1, df2, on='key', how='right')


# In[20]:


pd.merge(df1, df2, how='inner')


# In[22]:


#多个键的连接
left = DataFrame({'key1': ['foo', 'foo', 'bar'],
                  'key2': ['one', 'two', 'one'],
                  'lval': [1, 2, 3]})
right = DataFrame({'key1': ['foo', 'foo', 'bar', 'bar'],
                   'key2': ['one', 'one', 'one', 'two'],
                   'rval': [4, 5, 6, 7]})
pd.merge(left, right, on=['key1', 'key2'], how='outer')


# In[27]:


pd.merge(left, right, on='key1',copy='True')


# In[24]:


pd.merge(left, right, on='key1', suffixes=('_left', '_right'))


# In[28]:


###索引上的合并
#1
left1 = DataFrame({'key': ['a', 'b', 'a', 'a', 'b', 'c'],'value': range(6)})
right1 = DataFrame({'group_val': [3.5, 7]}, index=['a', 'b'])
left1


# In[29]:


right1


# In[31]:


pd.merge(left1,right1,left_on='key',right_index=True)


# In[32]:


pd.merge(left1, right1, left_on='key', right_index=True, how='outer')


# In[34]:


#2
import numpy as np
lefth = DataFrame({'key1': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada'],
                   'key2': [2000, 2001, 2002, 2001, 2002],
                   'data': np.arange(5.)})
righth = DataFrame(np.arange(12).reshape((6, 2)),
                   index=[['Nevada', 'Nevada', 'Ohio', 'Ohio', 'Ohio', 'Ohio'],
                          [2001, 2000, 2000, 2000, 2001, 2002]],
                   columns=['event1', 'event2'])
lefth


# In[35]:


righth


# In[36]:


pd.merge(lefth, righth, left_on=['key1', 'key2'], right_index=True)


# In[37]:


left2 = DataFrame([[1., 2.], [3., 4.], [5., 6.]], index=['a', 'c', 'e'],
                 columns=['Ohio', 'Nevada'])
left2


# In[38]:


right2 = DataFrame([[7., 8.], [9., 10.], [11., 12.], [13, 14]],
                   index=['b', 'c', 'd', 'e'], columns=['Missouri', 'Alabama'])
right2


# In[39]:


pd.merge(left2, right2, how='outer', left_index=True, right_index=True)


# In[40]:


left2.join(right2, how='outer')


# In[41]:


left1.join(right1, on='key')


# In[42]:


another = DataFrame([[7., 8.], [9., 10.], [11., 12.], [16., 17.]],
                    index=['a', 'c', 'e', 'f'], columns=['New York', 'Oregon'])
left2.join([right2, another])


# In[43]:


left2.join([right2,another],how='outer')


# In[3]:


###轴向连接
#1
arr = np.arange(12).reshape((3, 4))
arr


# In[4]:


np.concatenate([arr,arr],axis=1)


# In[8]:


#2
from pandas import Series
s1 = Series([0, 1], index=['a', 'b'])
s2 = Series([2, 3, 4], index=['c', 'd', 'e'])
s3 = Series([5, 6], index=['f', 'g'])


# In[9]:


pd.concat([s1,s2,s3])


# In[10]:


pd.concat([s1, s2, s3], axis=1)


# In[11]:


s4 = pd.concat([s1 * 5, s3])
s4


# In[12]:


pd.concat([s1, s4], axis=1)


# In[13]:


pd.concat([s1, s4], axis=1, join='inner')


# In[14]:


pd.concat([s1, s4], axis=1, join_axes=[['a', 'c', 'b', 'e']])


# In[15]:


result = pd.concat([s1, s1, s3], keys=['one', 'two', 'three'])
result


# In[16]:


result.unstack()


# In[17]:


#4
pd.concat([s1, s2, s3], axis=1, keys=['one', 'two', 'three'])


# In[18]:


df1 = DataFrame(np.arange(6).reshape(3, 2), index=['a', 'b', 'c'],
                columns=['one', 'two'])
df2 = DataFrame(5 + np.arange(4).reshape(2, 2), index=['a', 'c'],
                columns=['three', 'four'])


# In[19]:


pd.concat([df1, df2], axis=1, keys=['level1', 'level2'])


# In[20]:


pd.concat({'level1': df1, 'level2': df2}, axis=1)


# In[21]:


pd.concat([df1, df2], axis=1, keys=['level1', 'level2'],
          names=['upper', 'lower'])


# In[22]:


#5
df1 = DataFrame(np.random.randn(3, 4), columns=['a', 'b', 'c', 'd'])
df2 = DataFrame(np.random.randn(2, 3), columns=['b', 'd', 'a'])


# In[23]:


df1


# In[24]:


df2


# In[25]:


pd.concat([df1, df2], ignore_index=True)


# In[26]:


###合并重叠数据
#1
a = Series([np.nan, 2.5, np.nan, 3.5, 4.5, np.nan],
           index=['f', 'e', 'd', 'c', 'b', 'a'])
b = Series(np.arange(len(a), dtype=np.float64),
           index=['f', 'e', 'd', 'c', 'b', 'a'])
b[-1] = np.nan

a


# In[27]:


b


# In[29]:


#如果a里面是NaN，那么就用b里面替代，否则就用a
np.where(pd.isnull(a),b,a)


# In[30]:


#2
b[:-2].combine_first(a[2:])


# In[31]:


#3
df1 = DataFrame({'a': [1., np.nan, 5., np.nan],
                 'b': [np.nan, 2., np.nan, 6.],
                 'c': range(2, 18, 4)})
df2 = DataFrame({'a': [5., 4., np.nan, 3., 7.],
                 'b': [np.nan, 3., 4., 6., 8.]})
df1.combine_first(df2)


# In[32]:


###重塑层次化索引
#1
data = DataFrame(np.arange(6).reshape((2, 3)),
                 index=pd.Index(['Ohio', 'Colorado'], name='state'),
                 columns=pd.Index(['one', 'two', 'three'], name='number'))
data


# In[33]:


result = data.stack()
result


# In[34]:


result.unstack()


# In[38]:


result.unstack(0)


# In[36]:


result.unstack('state')


# In[39]:


#2
s1 = Series([0, 1, 2, 3], index=['a', 'b', 'c', 'd'])
s2 = Series([4, 5, 6], index=['c', 'd', 'e'])
data2 = pd.concat([s1, s2], keys=['one', 'two'])
data2.unstack()


# In[40]:


data2.unstack().stack()


# In[41]:


data2.unstack().stack(dropna=False)


# In[42]:


#3
df = DataFrame({'left': result, 'right': result + 5},
               columns=pd.Index(['left', 'right'], name='side'))
df


# In[43]:


df.unstack('state')


# In[44]:


df.unstack('state').stack('side')


# In[46]:


###长宽格式的转换
#1
data = pd.read_csv('data/macrodata.csv')
periods = pd.PeriodIndex(year=data.year, quarter=data.quarter, name='date')
data = DataFrame(data.to_records(),
                 columns=pd.Index(['realgdp', 'infl', 'unemp'], name='item'),
                 index=periods.to_timestamp('D', 'end'))

ldata = data.stack().reset_index().rename(columns={0: 'value'})
wdata = ldata.pivot('date', 'item', 'value')


# In[47]:


#2
ldata[:10]


# In[48]:


pivoted = ldata.pivot('date', 'item', 'value')
pivoted.head()


# In[49]:


ldata['value2'] = np.random.randn(len(ldata))
ldata[:10]


# In[50]:


pivoted = ldata.pivot('date', 'item')
pivoted[:5]


# In[53]:


pivoted['value'][:5]


# In[54]:


unstacked = ldata.set_index(['date', 'item']).unstack('item')
unstacked[:7]


# In[55]:


###移除重复数据
data = DataFrame({'k1': ['one'] * 3 + ['two'] * 4,
                  'k2': [1, 1, 2, 3, 3, 4, 4]})
data


# In[56]:


data.duplicated()


# In[57]:


data.drop_duplicates()


# In[58]:


data['v1'] = range(7)#增加一列


# In[59]:


data


# In[60]:


data.drop_duplicates(["k1"])


# In[67]:


data.drop_duplicates(['k1', 'k2'])


# In[102]: