1.主题式网络爬虫名称：爱奇艺电视剧热播数据分析
2.主题式网络爬虫爬取的内容：爱奇艺电视剧热播
3设计方案概述：
实现思路：爬取网站网页源代码，得到想要的数据位置，提取数据，之后数据可视化等操作
主题页面的结构特征分析
1 主题页面的结构与特征分析
打开网页 点击鼠标右键 点击检查 得到想要的数据位置
可以得到蓝框里面的就是我们所需要的数据位置
网络爬虫程序设计
1.数据爬取与采集
import requests
def get_url(url):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'}
try:
f = requests.get(url,headers=headers)
return f.text
except:
print('产生异常')
def main():
url = 'https://www.iqiyi.com/dianshiju/index.html'
some = get_url(url)
with open('try.txt','w',encoding='utf-8') as f:
f.write(some)
main()
对网页进行解析:
import bs4
def bs(text):
soup = bs4.BeautifulSoup(text,'html.parser')
return soup
def main():
url='https://list.iqiyi.com/www/2/-------------4-1-1-iqiyi--.html'#电视剧热门链接
some = get_url(url)
soup = bs(some)
with open('test.txt','w',encoding='utf-8')as file:
file .write(soup.text)
main()
解析内容
挑取几个电视剧做例子
柱状图
""" ==================== Horizontal bar chart ==================== This example showcases a simple horizontal bar chart. """ import matplotlib.pyplot as plt plt.rcdefaults() import numpy as np import matplotlib.pyplot as plt plt.rcdefaults() fig, ax = plt.subplots() # Example data #我是余欢水 猫冬 教场 侦探K9 猎狐 people = ('woshiyuhuangshui', 'maodong', 'jiaochang', 'zhentanK9', 'liehu') y_pos = np.arange(len(people)) performance = 3 + 10 * np.random.rand(len(people)) error = np.random.rand(len(people)) ax.barh(y_pos, performance, xerr=error, align='center', color='green', ecolor='black') ax.set_yticks(y_pos) ax.set_yticklabels(people) ax.invert_yaxis() # labels read top-to-bottom ax.set_xlabel('Performance') ax.set_title('iqiyi')#爱奇艺 plt.show()
点状分布图
""" =========================== Rotating custom tick labels =========================== Demo of custom tick-labels with user-defined rotation. """ import matplotlib.pyplot as plt x = [1, 2, 3, 4] y = [1, 4, 9, 6] labels = ['liehu', 'maodong', 'jiaochang', 'zhentanL9']#同上用英文代替 plt.plot(x, y, 'ro') # You can specify a rotation for the tick labels in degrees or with keywords. plt.xticks(x, labels, rotation='vertical') # Pad margins so that markers don't get clipped by the axes plt.margins(0.2) # Tweak spacing to prevent clipping of tick-labels plt.subplots_adjust(bottom=0.15) plt.show()
折线图
import numpy as np from numpy import ma import matplotlib.pyplot as plt x = np.arange(1, 7, 0.4) y0 = np.sin(x) y = y0.copy() + 2.5 plt.step(x, y, label='Y')#Y=我是余欢水 y -= 0.5 plt.step(x, y, where='mid', label='L')#L=猎狐 y -= 0.5 plt.step(x, y, where='post', label='J')#J=教场 y = ma.masked_where((y0 > -0.15) & (y0 < 0.15), y - 0.5) plt.step(x, y, label='M')#M=猫冬 plt.legend() plt.xlim(0, 7) plt.ylim(-0.5, 4) plt.show()
将上述代码全部合并
import requests def get_url(url): headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'} try: f = requests.get(url,headers=headers) return f.text except: print('产生异常') def main(): url = 'https://list.iqiyi.com/www/2/-------------4-1-1-iqiyi--.html' some = get_url(url) with open('try.txt','w',encoding='utf-8') as f: f.write(some) main() import bs4 def bs(text): soup = bs4.BeautifulSoup(text,'html.parser') return soup def main(): url='https://list.iqiyi.com/www/2/-------------4-1-1-iqiyi--.html'#电视剧热门链接 some = get_url(url) soup = bs(some) with open('test.txt','w',encoding='utf-8')as file: file .write(soup.text) main() #柱状图 #随便举例几个 中文图做不出来 用英文代替 """ ==================== Horizontal bar chart ==================== This example showcases a simple horizontal bar chart. """ import matplotlib.pyplot as plt plt.rcdefaults() import numpy as np import matplotlib.pyplot as plt plt.rcdefaults() fig, ax = plt.subplots() # Example data #我是余欢水 猫冬 教场 侦探K9 猎狐 people = ('woshiyuhuangshui', 'maodong', 'jiaochang', 'zhentanK9', 'liehu') y_pos = np.arange(len(people)) performance = 3 + 10 * np.random.rand(len(people)) error = np.random.rand(len(people)) ax.barh(y_pos, performance, xerr=error, align='center', color='green', ecolor='black') ax.set_yticks(y_pos) ax.set_yticklabels(people) ax.invert_yaxis() # labels read top-to-bottom ax.set_xlabel('Performance') ax.set_title('iqiyi')#爱奇艺 plt.show() """ =========================== Rotating custom tick labels =========================== Demo of custom tick-labels with user-defined rotation. """ import matplotlib.pyplot as plt x = [1, 2, 3, 4] y = [1, 4, 9, 6] labels = ['liehu', 'maodong', 'jiaochang', 'zhentanL9']#同上用英文代替 plt.plot(x, y, 'ro') # You can specify a rotation for the tick labels in degrees or with keywords. plt.xticks(x, labels, rotation='vertical') # Pad margins so that markers don't get clipped by the axes plt.margins(0.2) # Tweak spacing to prevent clipping of tick-labels plt.subplots_adjust(bottom=0.15) plt.show() import numpy as np from numpy import ma import matplotlib.pyplot as plt x = np.arange(1, 7, 0.4) y0 = np.sin(x) y = y0.copy() + 2.5 plt.step(x, y, label='Y')#Y=我是余欢水 y -= 0.5 plt.step(x, y, where='mid', label='L')#L=猎狐 y -= 0.5 plt.step(x, y, where='post', label='J')#J=教场 y = ma.masked_where((y0 > -0.15) & (y0 < 0.15), y - 0.5) plt.step(x, y, label='M')#M=猫冬 plt.legend() plt.xlim(0, 7) plt.ylim(-0.5, 4) plt.show()
结论：
数据爬取要注意细节 比如div 标签等等 编写代码时要注意大小写 是否英 注意缩进 让我对bs4库 seaborn库 和gallery库更加了解 是我对python的热爱更加深了 唯一遗憾的就是学的太慢 代码经常打错 很多地方其实不是太懂 所以需要勤加练习阿