本次我们将爬取Ajax动态加载数据并进行简单数据分析，其主要方式是找到数据的json包，将其保存到本地目录，进行数据分析
一、业务分析
目标网站：NBA中国官方网站https://china.nba.com/statistics/
爬取字段：
使用到的库：requests， json，csv，pandas numpy ，matplotlib
其中requests，json进行数据抓取
cxv保存到本地
pandas，numpy进行数据分析
matplotlib可视化
二、步骤
1.找到数据
首先，我们要找到储存数据的json包，F12进入开发者模式
点击网络，选择XHR，进行刷新，就可以看到json包了
这里我们可以得到请求头信息以及json包
这就是一会儿要抓取的数据
2.抓取数据
这里我选择了抓取本赛季前50球员的数据，在json包中寻找，可以看到
这里存放的是本赛季的数据
导入库
import requests
import json
import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
先定义抓取json包方法
def getJson(url):
headers={
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.53'
}
response = requests.get(url,headers=headers)
json_data = json.loads(response.text)
return json_data
定义抓取数据的方法
def getData(json_data):
playerList=[]
for item in json_data['payload']['players']:
player_dataDict={
}
#球员名字
name=item['playerProfile']['code']
#出场次数
games=item['statAverage']['games']
#先发
gamesStarted=item['statAverage']['gamesStarted']
#分钟
mins=item['statAverage']['minsPg']
#三分命中
tpm=item['statAverage']['tppct']
#罚球命中
ftm=item['statAverage']['ftpct']
#进攻
offRebs=item['statAverage']['offRebsPg']
#防守
defRebs=item['statAverage']['defRebsPg']
#篮板
rebs=item['statAverage']['rebsPg']
#助攻
assists=item['statAverage']['assistsPg']
#抢断
steals=item['statAverage']['stealsPg']
#盖帽
blocks=item['statAverage']['blocksPg']
#失误
turnovers=item['statAverage']['turnoversPg']
#犯规
fouls=item['statAverage']['foulsPg']
#得分
points=item['statAverage']['pointsPg']
player_dataDict['球员']=name
player_dataDict['场次']=games
player_dataDict['先发']=gamesStarted
player_dataDict['出场时间']=mins
player_dataDict['三分命中率']=tpm
player_dataDict['罚球命中率']=ftm
player_dataDict['进攻效率']=offRebs
player_dataDict['防守效率']=defRebs
player_dataDict['篮板']=rebs
player_dataDict['助攻']= assists
player_dataDict['抢断']=steals
player_dataDict['盖帽']=blocks
player_dataDict['失误']=turnovers
player_dataDict['犯规']=fouls
player_dataDict['得分']=points
print(player_dataDict)
playerList.append(player_dataDict)
return playerList
接下来进行存储
def writeData(playerList):
#写入数据
with open('player_data.csv','w',encoding='utf-8',newline='')as f:
write=csv.DictWriter(f, fieldnames=['球员','场次','先发','出场时间','三分命中率','罚球命中率',
'进攻效率','防守效率','篮板','助攻',
'抢断','盖帽','失误','犯规','得分'])
write.writeheader()
for each in playerList:
write.writerow(each)
完整代码
import requests
import json
import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
url='https://china.nba.com/static/data/league/playerstats_All_All_All_0_All_false_2021_2_All_Team_points_All_perGame.json'
def getJson(url):
headers={
'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36 Edg/95.0.1020.53'
}
response = requests.get(url,headers=headers)
json_data = json.loads(response.text)
return json_data
def getData(json_data):
playerList=[]
for item in json_data['payload']['players']:
player_dataDict={
}
#球员名字
name=item['playerProfile']['code']
#出场次数
games=item['statAverage']['games']
#先发
gamesStarted=item['statAverage']['gamesStarted']
#分钟
mins=item['statAverage']['minsPg']
#三分命中
tpm=item['statAverage']['tppct']
#罚球命中
ftm=item['statAverage']['ftpct']
#进攻
offRebs=item['statAverage']['offRebsPg']
#防守
defRebs=item['statAverage']['defRebsPg']
#篮板
rebs=item['statAverage']['rebsPg']
#助攻
assists=item['statAverage']['assistsPg']
#抢断
steals=item['statAverage']['stealsPg']
#盖帽
blocks=item['statAverage']['blocksPg']
#失误
turnovers=item['statAverage']['turnoversPg']
#犯规
fouls=item['statAverage']['foulsPg']
#得分
points=item['statAverage']['pointsPg']
player_dataDict['球员']=name
player_dataDict['场次']=games
player_dataDict['先发']=gamesStarted
player_dataDict['出场时间']=mins
player_dataDict['三分命中率']=tpm
player_dataDict['罚球命中率']=ftm
player_dataDict['进攻效率']=offRebs
player_dataDict['防守效率']=defRebs
player_dataDict['篮板']=rebs
player_dataDict['助攻']= assists
player_dataDict['抢断']=steals
player_dataDict['盖帽']=blocks
player_dataDict['失误']=turnovers
player_dataDict['犯规']=fouls
player_dataDict['得分']=points
print(player_dataDict)
playerList.append(player_dataDict)
return playerList
def writeData(playerList):
#写入数据
with open('player_data.csv','w',encoding='utf-8',newline='')as f:
write=csv.DictWriter(f, fieldnames=['球员','场次','先发','出场时间','三分命中率','罚球命中率',
'进攻效率','防守效率','篮板','助攻',
'抢断','盖帽','失误','犯规','得分'])
write.writeheader()
for each in playerList:
write.writerow(each)
if __name__ == "__main__":
json_data = getJson(url)
playerList=[]
playerList += getData(json_data)
writeData(playerList)
3.分析数据
数据都存放到本地了，我们当然可以为所欲为
这里我们选取了几个字段，生成了每个球员的雷达图，方便进行比较
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
df=pd.read_csv('player_data.csv')
for i in range(50):
x=df.loc[i]
name=x.loc[['球员']]
y=x.loc[['犯规','篮板','助攻','抢断','盖帽','失误']]
labels=np.array(['犯规','篮板','助攻','抢断','盖帽','失误'])
data=np.array(y)
plt.rcParams['font.sans-serif']=['SimHei']
angles=np.linspace(0, 2*np.pi,len(labels),endpoint=False)
labels=np.concatenate((labels,[labels[0]]))
data=np.concatenate((data,[data[0]]))
angles=np.concatenate((angles,[angles[0]]))
plt.polar(angles, data,'bo-',linewidth=1)
plt.thetagrids(angles*180/np.pi,labels)
plt.fill(angles, data,facecolor='b',alpha=0.25)
plt.title(str(name))
plt.show()
就不一个个上图了
总结
Ajax动态数据还是非常容易爬取的，同时pandas和numpy库也非常值得学习
最后一句
“湖人总冠军”