python爬虫
最近在看数据分析的书籍，想着自己分析一点东西来，本人比较喜欢NBA，自然就先拿NBA作为分析的对象了，首先要获得最全的NBA数据，根据搜索的结果对比发现，stat-nba.com该网站的数据非常全面详细，真是业界良心。
数据源找到后第一件事情自然就是获取数据，这里用python的原生代码直接爬取的，个人觉的，beautifulSoup还是比较好的，只是一开始没有用，也就后来没用了，废话少叙，直接上代码。
#coding=utf-8
import urllib
import re
import csv
import sys
#计数，初始化
count = 0
#以下定义的与之对应的是球员姓名、赛季、胜负、比赛、首发、时间、投篮命中率、投篮命中数、投篮出手数、三分命中率、三分命中数、三分出手数、罚球命中率、罚球命中数、罚球次数、总篮板数、前场篮板数、后场篮板数、助攻数、抢断数、盖帽数、失误数、犯规数、得分
list0 = []
list1 = []
list2 = []
list3 = []
list4 = []
list5 = []
list6 = []
list7 = []
list8 = []
list9 = []
list10 = []
list11 = []
list12 = []
list13 = []
list14 = []
list15 = []
list16 = []
list17 = []
list18 = []
list19 = []
list20 = []
list21 = []
list22 = []
list23 = []
list24 = []
list25 = []
list26 = []
#定义获取页面函数
def getHtml(url):
page = urllib.urlopen(url)
html = page.read()
return html
#获取数据并存入数据库中
for k in range(0,51):
#获取当前页面，该页面只有LBJ的职业生涯常规赛的数据，截止到2016.12.26
html = getHtml(
"http://www.stat-nba.com/query.php?QueryType=game&GameType=season&Player_id=1862&crtcol=season&order=1&page=" + str(
k))
# 获取球员姓名、赛季、胜负、比赛、首发、时间、投篮命中率、投篮命中数、投篮出手数、三分命中率、三分命中数、三分出手数、罚球命中率、罚球命中数、罚球次数、总篮板数、前场篮板数、后场篮板数、助攻数、抢断数、盖帽数、失误数、犯规数、得分
#正则得到相对应的数值
playerdata = re.findall(r'<td class ="normal player_name_out change_color col1 row.+"><a.*>(.*)</a></td>'
r'\s*<td class ="current season change_color col2 row.+"><a.*>(.*)</a></td>'
r'\s*<td class ="normal wl change_color col3 row.+">(.*)</td>'
r'\s*<td class ="normal result_out change_color col4 row.+"><a.*>(\D*|76人)(\d+)-(\d+)(\D*)</a></td>'
r'\s*<td class ="normal gs change_color col5 row.+">(.*)</td>'
r'\s*<td class ="normal mp change_color col6 row.+">(.*)</td>'
r'\s*<td class ="normal fgper change_color col7 row.+">(.*%|\s*)</td>'
r'\s*<td class ="normal fg change_color col8 row.+">(.*)</td>'
r'\s*<td class ="normal fga change_color col9 row.+">(.*)</td>'
r'\s*<td class ="normal threepper change_color col10 row.+">(.*%|\s*)</td>'
r'\s*<td class ="normal threep change_color col11 row.+">(.*)</td>'
r'\s*<td class ="normal threepa change_color col12 row.+">(.*)</td>'
r'\s*<td class ="normal ftper change_color col13 row.+">(.*%|\s*)</td>'
r'\s*<td class ="normal ft change_color col14 row.+">(.*)</td>'
r'\s*<td class ="normal fta change_color col15 row.+">(.*)</td>'
r'\s*<td class ="normal trb change_color col16 row.+">(.*)</td>'
r'\s*<td class ="normal orb change_color col17 row.+">(.*)</td>'
r'\s*<td class ="normal drb change_color col18 row.+">(.*)</td>'
r'\s*<td class ="normal ast change_color col19 row.+">(.*)</td>'
r'\s*<td class ="normal stl change_color col20 row.+">(.*)</td>'
r'\s*<td class ="normal blk change_color col21 row.+">(.*)</td>'
r'\s*<td class ="normal tov change_color col22 row.+">(.*)</td>'
r'\s*<td class ="normal pf change_color col23 row.+">(.*)</td>'
r'\s*<td class ="normal pts change_color col24 row.+">(.*)</td>', html)
#获取每条数据，
for data in playerdata:
#将元组数据复制给列表，进行修改，数据中有空值，和含有%号的值，进行处理，得到数值
data1 = [data[0], data[1], data[2], data[3], int(data[4]), data[5], data[6], data[7], data[8], data[9],
data[10], data[11], data[12], data[13], data[14], data[15], data[16], data[17], data[18], data[19],
data[20], data[21], data[22], data[23], data[24], data[25], data[26]]
#将百分号去掉，只保留数值部分
if (data1[15] == ' '):
data1[15] = 0
else:
data1[15] = float("".join(re.findall(r'(.*)%', data1[15])))
if (data1[9] == ' '):
data1[9] = 0
else:
data1[9] = float("".join(re.findall(r'(.*)%', data1[9])))
if (data1[12] == ' '):
data1[12] = 0
else:
data1[12] = float("".join(re.findall(r'(.*)%', data1[12])))
list0.append(data1[0])
list1.append(data1[1])
list2.append(data1[2])
list3.append(data1[3])
list4.append(data1[4])
list5.append(data1[5])
list6.append(data1[6])
list7.append(data1[7])
list8.append(data1[8])
list9.append(data1[9])
list10.append(data1[10])
list11.append(data1[11])
list12.append(data1[12])
list13.append(data1[13])
list14.append(data1[14])
list15.append(data1[15])
list16.append(data1[16])
list17.append(data1[17])
list18.append(data1[18])
list19.append(data1[19])
list20.append(data1[20])
list21.append(data1[21])
list22.append(data1[22])
list23.append(data1[23])
list24.append(data1[24])
list25.append(data1[25])
list26.append(data1[26])
# 记录数据数量
count += 1
#建立csv存储文件，wb写 a+追加模式
csvfile = file('nbadata.csv', 'ab+')
writer = csv.writer(csvfile)
#将提取的数据合并
data2 = []
for i in range(0,count):
data2.append((list0[i],list1[i],list2[i],list3[i],list4[i],list5[i],list6[i],list7[i],list8[i]
,list9[i],list10[i],list11[i],list12[i],list13[i],list14[i],list15[i],list16[i]
,list17[i],list18[i],list19[i],list20[i],list21[i],list22[i],list23[i],list24[i]
, list25[i],list26[i]))
#将合并的数据存入csv
writer.writerows(data2)
csvfile.close()经过爬取数据后得到了nbadata.csv文件，数据到手，下面就是分析了。
数据分析及可视化
这里只是简单的对LBJ职业生涯常规赛数据进行了分析，时间有限，就做了两个分析，一个是对常规赛的得分相同的次数进行统计。得出每个的分段的得分总次数；另一个是对过去13个赛季的五项能力（包括得分、篮板、助攻、盖帽、抢断）进行分析得出13个能力值图。
可视化部分应用的是flask框架搭建的web网站，前端用百度echart.js进行图像的搭建（强烈推荐echart，作图太666）
直接上代码：
mydata.py
#coding:utf-8
import sys
reload(sys)
sys.setdefaultencoding("utf-8")
from flask import Flask,render_template
from flask_bootstrap import Bootstrap
from pandas import DataFrame,Series
import pandas as pd
import numpy as np
import csv
#文件路径
srcFilePath = "c:/myflask/nbadata.csv"
#读取cvs格式的数据文件
reader = csv.reader(file(srcFilePath,'rb'))
#csv中各列属性代表的含义（1）代表第一列
# 球员姓名(1)、赛季(2)、胜负(3)、对手球队名称(4)、对手球队总得分（5）、己方球队总得分（6）
# 、己方球队名称（7）、首发（8）【1为首发，0为替补】、上场时间（9）、投篮命中率（10）、投篮命中数（11）
# 、投篮出手数（12）、三分命中率（13）、三分命中数（14）、三分出手数（15）、罚球命中率（16）
# 、罚球命中数（17）、罚球次数（18）、总篮板数（19）、前场篮板数（20）、后场篮板数（21）、助攻数（22）
# 、抢断数（23）、盖帽数（24）、失误数（25）、犯规数（26）、得分（27）
records = [line for line in reader]
frame = DataFrame(records)
#获取得分数对应的场次数目
pts_count = frame[26].value_counts()
a = []
b = []
for i in pts_count.keys():
a.append(i)
for i in pts_count:
b.append(i)
c = {}
for i in range(0,len(a)):
c[int(a[i])] = int(b[i])
d = sorted(c.items(), key=lambda c:c[0])
#存储得分分数
e = []
#存储相应分数的次数
f = []
for i in d:
e.append(i[0])
f.append(i[1])
#15-16赛季球员得分助攻篮板抢断盖帽平均值
records_p1 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '03-04']
records_p2 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '04-05']
records_p3 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '05-06']
records_p4 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '06-07']
records_p5 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '07-08']
records_p6 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '08-09']
records_p7 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '09-10']
records_p8 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '10-11']
records_p9 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '11-12']
records_p10 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '12-13']
records_p11 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '13-14']
records_p12 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '14-15']
records_p13 = [(int(line[26]),int(line[21]),int(line[18]),int(line[22]),int(line[23])) for line in records if line[1] == '15-16']
g1 = [float('%0.1f' % i) for i in DataFrame(records_p1).mean()]
g2 = [float('%0.1f' % i) for i in DataFrame(records_p2).mean()]
g3 = [float('%0.1f' % i) for i in DataFrame(records_p3).mean()]
g4 = [float('%0.1f' % i) for i in DataFrame(records_p4).mean()]
g5 = [float('%0.1f' % i) for i in DataFrame(records_p5).mean()]
g6 = [float('%0.1f' % i) for i in DataFrame(records_p6).mean()]
g7 = [float('%0.1f' % i) for i in DataFrame(records_p7).mean()]
g8 = [float('%0.1f' % i) for i in DataFrame(records_p8).mean()]
g9 = [float('%0.1f' % i) for i in DataFrame(records_p9).mean()]
g10 = [float('%0.1f' % i) for i in DataFrame(records_p10).mean()]
g11 = [float('%0.1f' % i) for i in DataFrame(records_p11).mean()]
g12 = [float('%0.1f' % i) for i in DataFrame(records_p12).mean()]
g13 = [float('%0.1f' % i) for i in DataFrame(records_p13).mean()]
app = Flask(__name__)
#引入bootstrap前端框架
bootstrap = Bootstrap(app)
@app.route('/')
def hello_world():
return render_template('index.html', a=e, b=f, c1=g1,c2=g2,c3=g3,c4=g4,c5=g5,c6=g6,c7=g7,c8=g8,c9=g9,c10=g10,c11=g11,c12=g12,c13=g13)
if __name__ == '__main__':
app.run(debug=True)前端：
index.html
{% extends "base.html" %}
{% block title %}Flasky{% endblock %}
{% block page_content %}
<div class="page-header">
<h1>数据分析</h1>
</div>
<!-- 为ECharts准备一个具备大小（宽高）的Dom -->
<div id="main" style="height:400px; width: auto"></div>
<div id="main2" style="height:600px; width: auto; background-color: #333">
<div id="s1" style="height:600px; width: auto">
</div>
</div>
<!-- ECharts单文件引入 -->
<script src="../static/echarts.js"></script>
<!-- 主题文件引入 -->
<script src="../static/dark.js"></script>
<script type="text/javascript">
// 基于准备好的dom，初始化echarts图表
var myChart = echarts.init(document.getElementById('main'));
var option = {
title: {
text: '得分次数图',
subtext: '数据来源：www.stat-nba.com'
},
tooltip: {
trigger: 'axis'
},
legend: {
data: ['次数']
},
calculable: true,
xAxis: [
{
type: 'category',
boundaryGap: false,
axisLabel: {
formatter: '{value}分',
rotate: 45,
},
data: {{ a }}
}
],
yAxis: [
{
type: 'value',
axisLabel: {
formatter: '{value} 次数'
}
}
],
series: [
{
name: '次数',
type: 'bar',
data:{{ b }},
markPoint: {
data: [
{type: 'max', name: '最大次数'},
{type: 'min', name: '最小次数'}
]
}
},
]
}
// 为echarts对象加载数据
myChart.setOption(option);
</script>
<script type="text/javascript">
// 基于准备好的dom，初始化echarts图表
var myChart1 = echarts.init(document.getElementById('s1'),'dark');
option = {
legend: {
data: ['03-04赛季', '04-05赛季', '05-06赛季', '06-07赛季', '07-08赛季'
, '08-09赛季', '09-10赛季', '10-11赛季', '11-12赛季', '12-13赛季', '13-14赛季'
, '14-15赛季', '15-16赛季'],
textStyle:{
fontSize:8,
}
},
radar: [
//03-04赛季
{
indicator: [
{name: '得分', max: 28.0},
{name: '助攻', max: 9.2},
{name: '篮板', max: 13.9},
{name: '抢断', max: 2.4},
{name: '盖帽', max: 3.6}
],
center: ['10%', '25%'],
radius: 80,
name:{
textStyle: {
color:'#67d15d',
fontSize: 6
}
}
},
//04-05赛季
{
indicator: [
{name: '得分', max: 30.7},
{name: '助攻', max: 11.5},
{name: '篮板', max: 13.5},
{name: '抢断', max: 2.9},
{name: '盖帽', max: 3.0}
],
center: ['30%', '25%'],
radius: 80,
name:{
textStyle: {
color:'#d1c373',
fontSize: 6
}
}
},
//05-06赛季
{
indicator: [
{name: '得分', max: 35.4},
{name: '助攻', max: 10.5},
{name: '篮板', max: 12.7},
{name: '抢断', max: 2.3},
{name: '盖帽', max: 3.2}
],
center: ['50%', '25%'],
radius: 80,
name:{
textStyle: {
color:'#d16a62',
fontSize: 6
}
}
},
//06-07赛季
{
indicator: [
{name: '得分', max: 31.6},
{name: '助攻', max: 11.6},
{name: '篮板', max: 12.8},
{name: '抢断', max: 2.1},
{name: '盖帽', max: 3.3}
],
center: ['70%', '25%'],
radius: 80,
name:{
textStyle: {
color:'#d170b6',
fontSize: 6
}
}
},
//07-08赛季
{
indicator: [
{name: '得分', max: 30.0},
{name: '助攻', max: 11.6},
{name: '篮板', max: 14.2},
{name: '抢断', max: 2.7},
{name: '盖帽', max: 3.6}
],
center: ['90%', '25%'],
radius: 80,
name:{
textStyle: {
color:'#8f45d1',
fontSize: 6
}
}
},
//08-09赛季
{
indicator: [
{name: '得分', max: 30.2},
{name: '助攻', max: 11.0},
{name: '篮板', max: 13.8},
{name: '抢断', max: 2.8},
{name: '盖帽', max: 2.9}
],
center: ['10%', '55%'],
radius: 80,
name:{
textStyle: {
color:'#4048d1',
fontSize: 6
}
}
},
//09-10赛季
{
indicator: [
{name: '得分', max: 30.1},
{name: '助攻', max: 11.0},
{name: '篮板', max: 13.2},
{name: '抢断', max: 2.3},
{name: '盖帽', max: 2.8}
],
center: ['30%', '55%'],
radius: 80,
name:{
textStyle: {
color:'#d11872',
fontSize: 6
}
}
},
//10-11赛季
{
indicator: [
{name: '得分', max: 27.7},
{name: '助攻', max: 11.4},
{name: '篮板', max: 15.2},
{name: '抢断', max: 2.4},
{name: '盖帽', max: 2.6}
],
center: ['50%', '55%'],
radius: 80,
name:{
textStyle: {
color:'#d1c80e',
fontSize: 6
}
}
},
//11-12赛季
{
indicator: [
{name: '得分', max: 28.0},
{name: '助攻', max: 11.7},
{name: '篮板', max: 14.5},
{name: '抢断', max: 2.5},
{name: '盖帽', max: 3.7}
],
center: ['70%', '55%'],
radius: 80,
name:{
textStyle: {
color:'#09e8ac',
fontSize: 6
}
}
},
//12-13赛季
{
indicator: [
{name: '得分', max: 28.7},
{name: '助攻', max: 9.7},
{name: '篮板', max: 12.4},
{name: '抢断', max: 2.4},
{name: '盖帽', max: 3.0}
],
center: ['90%', '55%'],
radius: 80,
name:{
textStyle: {
color:'#9c8eca',
fontSize: 6
}
}
},
//13-14赛季
{
indicator: [
{name: '得分', max: 32.0},
{name: '助攻', max: 10.7},
{name: '篮板', max: 13.7},
{name: '抢断', max: 2.5},
{name: '盖帽', max: 2.8}
],
center: ['10%', '85%'],
radius: 80,
name:{
textStyle: {
color:'#a6fdaa',
fontSize: 6
}
}
},
//14-15赛季
{
indicator: [
{name: '得分', max: 28.1},
{name: '助攻', max: 10.2},
{name: '篮板', max: 15.0},
{name: '抢断', max: 2.3},
{name: '盖帽', max: 2.9}
],
center: ['30%', '85%'],
radius: 80,
name:{
textStyle: {
color:'#faa60d',
fontSize: 6
}
}
},
//15-16赛季
{
indicator: [
{name: '得分', max: 30.1},
{name: '助攻', max: 11.7},
{name: '篮板', max: 14.8},
{name: '抢断', max: 2.1},
{name: '盖帽', max: 3.7}
],
center: ['50%', '85%'],
radius: 80,
name:{
textStyle: {
color:'#72ACD1',
fontSize: 6
}
}
}
],
series: [
//03-04赛季
{
name: '03-04赛季',
type: 'radar',
radarIndex: 0,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c1 }},
name : '03-04赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//04-05
{
name: '04-05赛季',
type: 'radar',
radarIndex: 1,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c2 }},
name : '04-05赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//05-06
{
name: '05-06赛季',
type: 'radar',
radarIndex: 2,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c3 }},
name : '05-06赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//06-07
{
name: '06-07赛季',
type: 'radar',
radarIndex: 3,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c4 }},
name : '06-07赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//07-08
{
name: '07-08赛季',
type: 'radar',
radarIndex: 4,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c5 }},
name : '07-08赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//08-09
{
name: '08-09赛季',
type: 'radar',
radarIndex: 5,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c6 }},
name : '08-09赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//09-10
{
name: '09-10赛季',
type: 'radar',
radarIndex: 6,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c7 }},
name : '09-10赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//10-11
{
name: '10-11赛季',
type: 'radar',
radarIndex: 7,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c8 }},
name : '10-11赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//11-12
{
name: '11-12赛季',
type: 'radar',
radarIndex: 8,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c9 }},
name : '11-12赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//12-13
{
name: '12-13赛季',
type: 'radar',
radarIndex: 9,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c10 }},
name : '12-13赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//13-14
{
name: '13-14赛季',
type: 'radar',
radarIndex: 10,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c11 }},
name : '13-14赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//14-15
{
name: '14-15赛季',
type: 'radar',
radarIndex: 11,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c12 }},
name : '14-15赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
//15-16
{
name: '15-16赛季',
type: 'radar',
radarIndex: 12,
textStyle:{
color:'#fff'
},
data : [
{
value : {{ c13 }},
name : '15-16赛季',
label: {
normal: {
show: true,
textStyle:{
color:"#fff",
fontSize:8
}
}
},
areaStyle: {
normal: {
color: 'rgba(100, 100, 255, 0.5)',
}
},
}
]
},
]
};
// 为echarts对象加载数据
myChart1.setOption(option);
</script>
{% endblock %}显示效果：
马上下班，有空再写。