查看DataFrame有多少行、列：df.shape
仅查看DF有多少行：df.index.size
解决写入中文乱码问题：df.to_csv(file, encoding='utf_8_sig')
画箱型图
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
ipl_data = {'Team': ['Riders', 'Riders', 'Devils', 'Devils', 'Kings',
'kings', 'Kings', 'Kings', 'Riders', 'Royals', 'Royals', 'Riders'],
'Rank': [1, 2, 2, 3, 3,4 ,1 ,1,2 , 4,1,2],
'Year': [2014,2015,2014,2015,2014,2015,2016,2017,2016,2014,2015,2017],
'Points':[876,789,863,673,741,812,756,788,694,701,804,690]}
df = pd.DataFrame(ipl_data)
df.boxplot() # 对df的所有数值(非字符数据)列画箱型图
plt.show()
df.boxplot(column= ['Points']) # 指定用某一列画箱型图
plt.show()
df['Points'].plot.box() # 这里的方法属于matplotlib，而前面的方法属于pandas
plt.show()分组取平均
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
ipl_data = {'Team': ['Riders', 'Riders', 'Devils', 'Devils', 'Kings',
'kings', 'Kings', 'Kings', 'Riders', 'Royals', 'Royals', 'Riders'],
'Rank': [1, 2, 2, 3, 3,4 ,1 ,1,2 , 4,1,2],
'Year': [2014,2015,2014,2015,2014,2015,2016,2017,2016,2014,2015,2017],
'Points':[876,789,863,673,741,812,756,788,694,701,804,690]}
df = pd.DataFrame(ipl_data)
grouped = df.groupby('Team') # 按队('Team')分组
avg = grouped['Points', 'Year'].agg(np.mean) # 计算各组分数、年份的平均值
print (avg)
print (type(avg)) # 结果是 <class 'pandas.core.frame.DataFrame'>
for group in avg: # 结果是Points Year
print( group)
avg2 = grouped['Points'].agg(np.mean) # 计算各组分数、年份的平均值
print (type(avg2)) # 结果是 <class 'pandas.core.series.Series'>
for group in avg2: # 结果是一列浮点数（各组均值）
print( group)
'''
上面说明Pandas的聚合函数(agg)处理时，需要结果是DataFrame则DataFrame
若结果只一列则Series之，DataFrame、Series都有索引列
'''分组，依据相应的列的值生成新的列，或者改变已有列的值：
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
ipl_data = {'Team': ['Riders', 'Riders', 'Devils', 'Devils', 'Kings',
'kings', 'Kings', 'Kings', 'Riders', 'Royals', 'Royals', 'Riders'],
'Rank': [1, 2, 2, 3, 3,4 ,1 ,1,2 , 4,1,2],
'Year': [2014,2015,2014,2015,2014,2015,2016,2017,2016,2014,2015,2017],
'Points':[876,789,863,673,741,812,756,788,694,701,804,690]}
df = pd.DataFrame(ipl_data)
grouped = df.groupby('Team') # 按队('Team')分组
print(df)
# 分组后得到的对象是一个tuple——数据结构类似字典——key：组名，value：分组结果
# 因为分组后，我们没有改变索引，所以组内每一行的索引值都来自原数据，我们可依此定为并改变原数据的值
for name, group in grouped:
for elem in group.index: # 获取组中索引
if df.loc[elem, 'Year'] >2016:
df.loc[elem, 'C']=0 # 这里利用索引和列定位，并改变原数据的值，列名可以是新列或已有列
else:
df.loc[elem, 'C']=1
print(df)上面虽然能够帮我生成新的列，或者改变已有列的值，但在实践中我发现这样效率很低，数据量大的时候会花费颇多时间。因为定位实际上就是在查找和对比嘛，数据量越大，查找时比较的次数就越多。
下面我介绍的这种方法经我实践后发现可以加快效率，因为不再定位了，直接对每次遍历得到的group对象进行处理（增加或修改列值），然后将每次遍历得到的group与前面的group连接起来最后也能达到目的。
另外我发现如果能把数据给分成多个小文件处理，也会加快效率，可能后面数据量大了之后会拖累速度吧。
apply,applymap和map的应用 总结:apply用在dataframe上，用于对row或者column进行计算；applymap用于dataframe上，是元素级别的操作，即对单个的元素操作，比apply更细；map（其实是python自带的）用于series上，是元素级别的操作。
关于高阶函数map()可参考map
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
ipl_data = {'Team': ['Riders', 'Riders', 'Devils', 'Devils', 'Kings',
'kings', 'Kings', 'Kings', 'Riders', 'Royals', 'Royals', 'Riders'],
'Gender': ['male', 'female', 'female', 'female', 'male', 'female',
'male', 'female','male', 'male', 'male', 'female',],
'Rank': [1, 2, 2, 3, 3,4 ,1 ,1,2 , 4,1,2],
'Year': [2014,2015,2014,2015,2014,2015,2016,2017,2016,2014,2015,2017],
'Points':[876,789,863,673,741,812,756,788,694,701,804,690]}
df = pd.DataFrame(ipl_data)
grouped = df.groupby('Team') # 按队('Team')分组
print(df)
# 性别男则'帅'之，性别女则'美'之，未知性别则'未知性别'之
def f(gender):
if gender == 'female':
return 'beautiful' # 美
elif gender == 'male':
return 'handsome' # 帅
else:
return 'unknown gender' # 未知性别
new_df = 0
cnt = 1
# 我们不在原始数据上进行更改，而是对每个分组的具体情况进行修改，并新增相应地列，最后将这些分组合并在new_df上，得到新的完整数据
# 下面这个循环在这种情况下有点多此一举，但实际上在有些情况下还是很有用的——比如我们有一组数据，需要根据小时分组，
# 再每小时的值算出一个小时全局(比如小时内的风向均值)的值，来处理数据
for name, group in grouped:
#group['describe'] = list(map(f, group['Gender'])) #根据性别这一列的值，新增一列，来对其进行描述，下面一行是另一种实现方法
group['describe'] = group['Gender'].apply(f) # 这种新增列的操作会有警告出现，可忽略之，因为结果是对的
if cnt == 1:
new_df = group #
cnt = 0
else:
new_df = new_df.append(group)
print(group, '\n')
print(new_df)
pandas获取groupby分组里最大值所在的行:
http://www.cnblogs.com/dylan9/p/9226957.html
pandas删除一行或一列：
life is short, you need python!
创建DataFrame、计算相关系数
import math
import pandas as pd
# 计算均方根误差
def rmse_f(lst1, lst2):
err_lst = []
for elem1, elem2 in zip(lst1, lst2):
err_lst.append((elem2 - elem1)**2)
err_sum = sum(err_lst)
rmse = math.sqrt(err_sum/len(err_lst))
return rmse
if __name__ == '__main__':
# 未分层GF
unstrtf_lst = [[2.1363578579344553, 1.778159659230212, 1.7461109776520496, 2.5650176964100058, 1.8729746077047558, 2.4125212473758144, 1.8131378210077445, 1.7203791476410082, 1.932176169703592, 1.9874966416464654, 2.035476472069343, 2.1781720205641535, 2.0496316896837037, 2.015503076760883, 1.6445972422920334, 1.7555358479027758, 1.8860403926573988, 2.106449216415071, 2.1377906629835963, 1.9141163955526903, 1.9835279705796847, 1.9055804443583697, 1.87077070319819, 1.9390509289545736, 1.810095752719198, 1.9304942682030173, 1.8979078915059369, 1.8015313995700164, 2.0083572196295227, 1.7239937688183304, 1.8228648048180662, 1.6359701708848993, 1.7735626967597569, 2.0545260692580856, 1.9337350425476532, 1.6293154939534678, 2.5188480005914657, 2.0933138240531814, 2.0039132581755728, 1.7927243831635933, 1.5636118528163134, 1.961936771908866, 2.175862474358458, 1.84555420903568, 1.8157478273063956, 2.018132722206774, 1.7082027177199874, 2.465498188552706, 1.899241304911715, 1.523358162773028, 1.4097992169699092, 2.102237127978442, 2.065209158150456, 2.40153945887694, 2.0909915155623016, 1.8670260337542834, 1.769808760804003, 1.4656342105489872, 2.02876880332766, 1.6593144568711944, 1.6259734718666612, 1.976719588334435, 1.836769571197318, 2.130456782328913, 2.2406696990643686, 2.184332199816752, 2.344971813941753, 1.8327028095055091, 2.112783901499557, 1.76393363916495, 1.8591739526387743, 1.8678292971398633, 1.8347280399622345, 1.9055594171490333, 2.236563822752378, 1.8461443260993826, 1.8710247338225503, 1.769483042695348, 1.9280994083081056, 1.8305827044735423, 1.87501371941072, 2.0394676216112755, 2.2397468910459577, 1.8350481624243522, 1.850988173648409]
,[2.1705108509302296, 1.831349725830108, 1.7144354961723647, 2.5070119877990993, 1.7932149216788924, 2.5256108032056024, 1.8287156965627762, 1.7053073887640058, 1.9536164306821924, 2.0165733314095484, 2.0219532407272367, 2.15962408389747, 2.0591612501692533, 1.9663915329316037, 1.6605235412016797, 1.752070773449648, 1.8835505261691532, 2.20298084330088, 2.1816417248308912, 1.9697915510943689, 2.0027300493080635, 1.8753583244593715, 1.8515835391325501, 1.8844389452321684, 1.7739928638003744, 1.9158563751480149, 1.935882528674244, 1.808727767581996, 1.9261869435891357, 1.7168695849567477, 1.8406103433305763, 1.5901906138296908, 1.781209430463494, 2.0155474259248365, 1.897624720096371, 1.6572617492955173, 2.457603934271022, 2.1340834716198884, 2.032257820069647, 1.7848302835018754, 1.5749260991093992, 1.9594582149312278, 2.110070680121267, 1.8536274936727972, 1.8255496939025995, 1.9915475470891886, 1.7057170663908845, 2.41881624836319, 1.853988086419106, 1.5137201690837159, 1.3699066656182317, 2.0838775300742207, 2.0241094432259845, 2.3975607898159215, 1.954998320761276, 1.858850222876411, 1.758513584665346, 1.440576776793846, 2.0594670410365756, 1.652794581031125, 1.583183905771709, 1.9870492550090213, 1.839520744037398, 2.106043455083318, 2.261842743902759, 2.130334304091718, 2.371488547564126, 1.77621537824661, 2.1166072536485903, 1.7334043813895863, 1.8144915554048153, 1.8392673643670565, 1.8221073814375517, 1.882852864820385, 2.229513787171235, 1.8033542431800091, 1.8939166624091932, 1.7827963004335363, 1.9105410645385281, 1.8133481834516405, 1.849921371931809, 2.0042578515681173, 2.190705418004792, 1.8226117001058084, 1.809095171276665]
,[2.1572879151479003, 1.8732456165854612, 1.8021950195988314, 2.761396667303968, 1.7329910942117082, 2.50581030614301, 1.8421096808065078, 1.7650227215947034, 1.938483822143869, 2.0578323954052005, 1.931773557791433, 2.1955561312824994, 2.0038751572456794, 2.125604883507349, 1.664452399612406, 1.6981417766566513, 1.8989112768204548, 2.072501705907433, 2.1165817006598804, 2.083222704453948, 1.9718149342934317, 1.9690049995349723, 1.8648757876503859, 1.9369795903109537, 1.751516384479702, 1.9386115963915693, 1.926581991479464, 1.8043800075543694, 2.0702974778427268, 1.7254418958733437, 1.846008942263762, 1.500309119735106, 1.8043451922011515, 2.099752339587066, 1.9322863346314587, 1.7730958826481285, 2.4307315077025016, 2.0879597521760767, 2.0796598831566544, 1.8123784146112378, 1.5922748859137077, 1.9529811213293085, 2.044460443054776, 2.018932478066154, 1.8463628442698425, 2.061164904792595, 1.7714139713080599, 2.254364488252947, 1.8910648819975235, 1.5362007373888638, 1.3556999686411453, 1.9518926729355925, 2.222028472870814, 2.4271605745669063, 2.0154610576288046, 1.8726635786617294, 1.7899451750430633, 1.3844805992385176, 1.981236136025878, 1.6650448525276538, 1.81468791161442, 2.0057538089332207, 1.8689871109475358, 2.1017620938981745, 2.2486886446720398, 2.2701437119850403, 2.2964873844393434, 1.8144217949929822, 2.099006816811984, 1.7024279618371951, 1.6879977674716204, 1.8898124441096433, 1.8198271951536638, 1.9265635760185849, 2.1618226263648372, 1.8251940859581977, 1.997543429302503, 1.8107412281289608, 1.999929048528067, 1.8417695924175534, 1.7932634397869371, 2.1153714379421045, 2.301483201844252, 1.7891439341814215, 1.8263241651607782]
,[2.127465604815821, 1.7435981329981285, 1.7469925728187536, 2.548492404889417, 1.9394516313113324, 2.2963834108754537, 1.8075434420398742, 1.7095633809547433, 1.9011864061179378, 1.905645849579405, 2.0736817612179297, 2.16671678481412, 2.1127875667675204, 2.0438490539887053, 1.6318899302090566, 1.8209363731343235, 1.9396815952611905, 2.075548926116798, 2.1138411930924343, 1.836810355960725, 1.9775143575128646, 1.9042372600054018, 1.871749050992368, 1.9803766684660382, 1.8858206872464, 1.9226023878964362, 1.875303554237344, 1.7989243976760343, 1.9922957627857394, 1.7043915815219441, 1.8122279568988722, 1.7154211182773855, 1.7561964212026768, 2.0605773918750407, 1.9403360340368525, 1.5535472603591154, 2.5916331782183315, 2.065236202893155, 1.9828771746857574, 1.8023360161240602, 1.570438242411317, 1.9552131453436665, 2.214970521267441, 1.7647276262259532, 1.7961026573169705, 2.005608569200872, 1.662339602826159, 2.5733661206488314, 1.9152348830728956, 1.5432754448406762, 1.4389508077937503, 2.1595951346743645, 2.011665174943963, 2.4196512325487505, 2.2680643087606396, 1.8864643334260363, 1.766903164578892, 1.526725478045081, 2.073016313835559, 1.650352023433284, 1.5666242163104553, 2.015856416997733, 1.819420208388161, 2.152735278331005, 2.225459658895069, 2.2374333678738383, 2.327354050767716, 1.8769174709173695, 2.115095548209552, 1.8042807872341624, 1.9385772601512996, 1.8667525044441757, 1.8398968892930259, 1.9052233013176596, 2.3021441958120366, 1.8826397181400107, 1.7982343952469901, 1.7245406293425234, 1.893488504381276, 1.8455838450609297, 1.9160701102011286, 2.025458770576844, 2.267509591004735, 1.866525170029646, 1.8773635842125924]
,[2.0893312084816147, 1.6642699472556945, 1.7202783586720822, 2.4413516244056934, 2.030595403006813, 2.3206350319671456, 1.7733334610825282, 1.7023498282309177, 1.9354299849846581, 1.9684150157305658, 2.118766838446224, 2.190854077869349, 2.022973770919618, 1.924964759375417, 1.6209758325222385, 1.7503680465366664, 1.8219279865803366, 2.0739204792604102, 2.138792011988631, 1.763914943796737, 1.9819437819388206, 1.8731604534060744, 1.8952660370676917, 1.9552479226584358, 1.829118459522991, 1.9451489443809487, 1.8527845612250686, 1.7939610191815445, 2.045554476589397, 1.7496890230278257, 1.7925069592381986, 1.7406631877422731, 1.7519551513735, 2.0419456678317602, 1.9653913341354163, 1.5316022390884816, 2.5982417563178135, 2.085504606726917, 1.9226384886507224, 1.7713041939769747, 1.5168737598226512, 1.9803453574167327, 2.3377792144390988, 1.7429308508176682, 1.7944818668054086, 2.014020786766918, 1.6925703326109975, 2.6182933510425754, 1.9376002712707523, 1.5001608550211696, 1.4759383208905805, 2.2162500600865176, 2.0026050643658397, 2.361327832962134, 2.129767086599434, 1.8500381639561239, 1.7638223152628805, 1.5125464151009647, 2.001480595006969, 1.6694246950308196, 1.5379206704040849, 1.897317810046932, 1.8187468085928538, 2.1625676164174674, 2.2261681199304824, 2.098808003173525, 2.385761766718095, 1.8645099170513009, 2.1206188276307105, 1.8183630561766648, 1.9996520742700048, 1.8758229069660182, 1.85760699837217, 1.9076828855591454, 2.2539408129138554, 1.8744569228028842, 1.7905429728367388, 1.758874038649419, 1.907694776752748, 1.8215000281849234, 1.9438074466253656, 2.0120801264831925, 2.201224178338021, 1.862650418406529, 1.8919626527513929]
]
# 此行将作为DataFrame的col name
column_lst = ['whole_year', 'spring', 'summer', 'autumn', 'winter']
# 创建数据字典，为生成Dataframe做准备
data_dict = {}
for col, gf_lst in zip(column_lst, unstrtf_lst):
data_dict[col] = gf_lst
# pandas计算各列数据两两间的相关系数矩阵
unstrtf_pd = pd.DataFrame(data_dict)
cor1 = unstrtf_pd.corr()
print(cor1)
print(unstrtf_pd.columns.tolist())