1 import time
2 from selenium import webdriver
3
4
5 class DouyuSpider(object):
6 def __init__(self):
7 self.start_rul = 'https://www.douyu.com/directory/all'
8 self.driver = webdriver.Chrome()
9
10 def get_content_list(self):
11 time.sleep(10) # 强制等待10秒,否则可能报错
12 li_list = self.driver.find_elements_by_xpath('//ul[@class="layout-Cover-list"]/li')
13 content_list = []
14 for li in li_list:
15 item = {}
16 item['room_img'] = li.find_element_by_xpath('.//img[@class="DyImg-content is-normal "]').get_attribute('src')
17 item['room_title'] = li.find_element_by_xpath('.//h3[@class="DyListCover-intro"]').text
18 item['root_category'] = li.find_element_by_xpath('.//span[@class="DyListCover-zone"]').text
19 item['author_name'] = li.find_element_by_class_name('DyListCover-user').text
20 item['watch_num'] = li.find_element_by_class_name('DyListCover-hot').text
21 content_list.append(item)
22 print(item) # 打印每次获取到的直播房间的信息
23 # 获取下一页的元素,为了防止没有报错,这里用elements,翻到最后一页一定就没有了,返回一个列表
24 next_url = self.driver.find_elements_by_xpath('//li[@class=" dy-Pagination-next"]')
25 next_url = next_url[0] if len(next_url) > 0 else None
26 return content_list, next_url
27
28 def save_content_list(self, content_list):
29 pass # 保存数据这里就不再做演示
30
31 def run(self): # 实现主要逻辑
32 # 1.start_url
33 # 2.发送请求,获取响应
34 self.driver.maximize_window()
35 self.driver.get(self.start_rul)
36 # 3.提取数据,提取下一页的元素
37 content_list, next_url = self.get_content_list()
38 # 4.保存数据
39 self.save_content_list(content_list)
40 # 4.点击下一页元素,循环
41 while next_url is not None:
42 next_url.click()
43 content_list, next_url = self.get_content_list()
44 self.save_content_list(content_list)
45
46
47
48 if __name__ == '__main__':
49 douban = DouyuSpider()
50 douban.run()