抓包时发现子菜单请求数据时一般需要rid,但的确存在一些如游戏->游戏赛事不使用rid,对于这种未进行处理,此外rid一般在主菜单的响应中,但有的如番剧这种,rid在子菜单的url中,此外返回的data中含有页数相关信息,可以据此定义爬取的页面数量
1 # -*- coding: utf-8 -*-
2 # @author: Tele
3 # @Time : 2019/04/08 下午 1:01
4 import requests
5 import json
6 import os
7 import re
8 import shutil
9 from lxml import etree
10
11
12 # 爬取每个菜单的前5页内容
13 class BiliSplider:
14 def __init__(self, save_dir, menu_list):
15 self.target = menu_list
16 self.url_temp = "https://www.bilibili.com/"
17 self.headers = {
18 "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
19 "Cookie": "LIVE_BUVID=AUTO6715546997211617; buvid3=07192BD6-2288-4BA5-9259-8E0BF6381C9347193infoc; stardustvideo=1; CURRENT_FNVAL=16; sid=l0fnfa5e; rpdid=bfAHHkDF:cq6flbmZ:Ohzhw:1Hdog8",
20 }
21 self.proxies = {
22 "http": "http://61.190.102.50:15845"
23 }
24 self.father_dir = save_dir
25
26 def get_menu_list(self):
27 regex = re.compile("//")
28 response = requests.get(self.url_temp, headers=self.headers)
29 html_element = etree.HTML(response.content)
30 nav_menu_list = html_element.xpath("//div[@id='primary_menu']/ul[@class='nav-menu']/li/a")
31
32 menu_list = list()
33 for item in nav_menu_list:
34 menu = dict()
35 title = item.xpath("./*/text()")
36 menu["title"] = title[0] if len(title) > 0 else None
37 href = item.xpath("./@href")
38 menu["href"] = "https://" + regex.sub("", href[0]) if len(href) > 0 else None
39
40 # 子菜单
41 submenu_list = list()
42 sub_nav_list = item.xpath("./../ul[@class='sub-nav']/li")
43 if len(sub_nav_list) > 0:
44 for sub in sub_nav_list:
45 submenu = dict()
46 sub_title = sub.xpath("./a/span/text()")
47 submenu["title"] = sub_title[0] if len(sub_title) > 0 else None
48 sub_href = sub.xpath("./a/@href")
49 submenu["href"] = "https://" + regex.sub("", sub_href[0]) if len(sub_href) > 0 else None
50 submenu_list.append(submenu)
51 menu["submenu_list"] = submenu_list if len(submenu_list) > 0 else None
52 menu_list.append(menu)
53 return menu_list
54
55 # rid=tid
56 def parse_index_url(self, url):
57 result_list = list()
58 # 正则匹配
59 regex = re.compile("<script>window.__INITIAL_STATE__=(.*)</script>")
60 response = requests.get(url, headers=self.headers)
61 result = regex.findall(response.content.decode())
62 temp = re.compile("(.*);\(function").findall(result[0]) if len(result) > 0 else None
63 sub_list = json.loads(temp[0])["config"]["sub"] if temp else list()
64 if len(sub_list) > 0:
65 for sub in sub_list:
66 # 一些子菜单没有rid,需要请求不同的url,暂不处理
67 if "tid" in sub:
68 if sub["tid"]:
69 sub_menu = dict()
70 sub_menu["rid"] = sub["tid"] if sub["tid"] else None
71 sub_menu["title"] = sub["name"] if sub["name"] else None
72 result_list.append(sub_menu)
73 else:
74 pass
75
76 return result_list
77
78 # 最新动态 region?callback
79 # 数据 newlist?callback
80 def parse_sub_url(self, item):
81 self.headers["Referer"] = item["referer"]
82 url_pattern = "https://api.bilibili.com/x/web-interface/newlist?rid={}&type=0&pn={}&ps=20"
83
84 # 每个菜单爬取前5页
85 for i in range(1, 6):
86 data = dict()
87 url = url_pattern.format(item["rid"], i)
88 print(url)
89 try:
90 response = requests.get(url, headers=self.headers, proxies=self.proxies, timeout=10)
91 except:
92 return
93 if response.status_code == 200:
94 data["content"] = json.loads(response.content.decode())["data"]
95 data["title"] = item["title"]
96 data["index"] = i
97 data["menu"] = item["menu"]
98 # 保存数据
99 self.save_data(data)
100 else:
101 print("请求超时") # 一般是403,被封IP了
102
103 def save_data(self, data):
104 if len(data["content"]) == 0:
105 return
106 parent_path = self.father_dir + "/" + data["menu"] + "/" + data["title"]
107 if not os.path.exists(parent_path):
108 os.makedirs(parent_path)
109 file_dir = parent_path + "/" + "第" + str(data["index"]) + "页.txt"
110
111 # 保存
112 with open(file_dir, "w", encoding="utf-8") as file:
113 file.write(json.dumps(data["content"], ensure_ascii=False, indent=2))
114
115 def run(self):
116 # 清除之前保存的数据
117 if os.path.exists(self.father_dir):
118 shutil.rmtree(self.father_dir)
119
120 menu_list = self.get_menu_list()
121 menu_info = list()
122 # 获得目标菜单信息
123 # 特殊列表,一些菜单的rid必须从子菜单的url中获得
124 special_list = list()
125 for menu in menu_list:
126 for t in self.target:
127 if menu["title"] == t:
128 if menu["title"] == "番剧" or menu["title"] == "国创" or menu["title"] == "影视":
129 special_list.append(menu)
130 menu_info.append(menu)
131 break
132
133 # 目标菜单的主页
134 if len(menu_info) > 0:
135 for info in menu_info:
136 menu_index_url = info["href"]
137 # 处理特殊列表
138 if info in special_list:
139 menu_index_url = info["submenu_list"][0]["href"]
140 # 获得rid
141 result_list = self.parse_index_url(menu_index_url)
142 print(result_list)
143 if len(result_list) > 0:
144 for item in result_list:
145 # 大菜单
146 item["menu"] = info["title"]
147 item["referer"] = menu_index_url
148 # 爬取子菜单
149 self.parse_sub_url(item)
150
151
152 def main():
153 target = ["动画", "番剧", "国创", "音乐", "舞蹈", "游戏", "科技", "数码", "生活", "鬼畜", "时尚", "广告", "娱乐", "影视"]
154 splider = BiliSplider("f:/bili_splider", target)
155 splider.run()
156
157
158 if __name__ == '__main__':
159 main()
可以看到番剧少了新番时间表与番剧索引,因为这两个请求不遵循https://api.bilibili.com/x/web-interface/newlist?rid={}&type=0&pn={}&ps=20的格式,类似的不再赘述
