1.简介
selenium可以认为是反反爬虫的最佳利器，它基本可以等同于真实的浏览器访问，用它可以加载到动态数据，也省去了cookie的操作，但是用这个有一个重大的效率问题。所以selenium可以用来爬取一些对爬虫限制很大的网站。
2.基本使用
# -*-coding:utf8 -*- #selenium+chromedriver获取动态数据 #selenium相当于是一个机器人。可以模拟人类在浏览器上的一些行为，比如点击，填充数据，删除cookie等。 #chromedirver是一个驱动chrome浏览器的驱动程序，使用他才可以驱动浏览器。当然针对不同的浏览器有不同的driver #1.Chrome：https://sites.google.com/a/chromium.org/chromedriver/downloads #2.Firefox:https://github.com/mozilla/geckodriver/releases #3.Edge:https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver #4.Safari:https://webkit.org/blog/6900/webdriver-support-in-safari-10/ #安装selenium #pip3 install selenium #安装chromedriver：下载完成后，放到不需要权限的纯英文目录下即可 from selenium import webdriver driver_path=r'D:\chromedriver\chromedriver.exe' driver=webdriver.Chrome(executable_path=driver_path) driver.get('https://www.baidu.com') # print(driver.page_source) import time time.sleep(5) #关闭页面 # driver.close() #关闭页面 # driver.quit() #退出整个浏览器
3.查找元素
# -*-coding:utf8 -*- from selenium import webdriver driver_path=r'D:\chromedriver\chromedriver.exe' driver=webdriver.Chrome(executable_path=driver_path) driver.get('https://www.baidu.com') from selenium.webdriver.common.by import By #定位元素 ''' 1.find_element_by_id:根据id来查找某个元素 inputTag=driver.find_element_by_id('kw') inputTag=driver.find_element(By.ID,'kw') 2.find_element_by_class_name:根据类名查找元素 submitTag=driver.find_element_by_class_name('su') submitTag=driver.find_element(By.CLASS_NAME,'su') 3.find_element_by_name:根据name属性的值来查找元素 submitTag=driver.find_element(By.NAME,'su') submitTag=driver.find_element_by_name('su') 4.find_element_by_tag_name:根据标签名来查找元素 submitTag=driver.find_element_by_tag_name('div') submitTag=driver.find_element(By.TAG_NAME,'div') 5.find_element_by_xpath:根据xpath语法来获取元素 submitTag=driver.find_element_by_xpath('//div') submitTag=driver.find_element(By.XPATH,'//div') 6.find_element_by_css_selector:根据css选择器选择元素 submitTag=driver.find_element(By.CSS_SELECTOR,'//div') submitTag=driver.find_element_by_css_selector('//div') 每个方法都有find_element_by和find_elements_by：查找一个和查找多个的区别 ''' # inputTag=driver.find_element_by_id('kw') # inputTag=driver.find_element_by_name('wd') # inputTag=driver.find_element_by_class_name('s_ipt') # inputTag=driver.find_element_by_xpath('//input[@id="kw"]') # inputTag=driver.find_element_by_css_selector('.quickdelete-wrap > input') inputTag=driver.find_elements_by_css_selector('.quickdelete-wrap > input')[0] inputTag.send_keys('python') #1.如果只是想要解析网页中的数据，那么推荐将网页源代码扔给lxml来解析。因为lxml底层 #使用的是C语言，所以解析效率会更高 #2.如果是想要对元素进行一些操作，比如给一个文本输入值，或者是点击某个按钮，那么就必须使用selenuim给 #我们提供的查找元素的方法 from selenium import webdriver from lxml import etree from selenium.webdriver.common.by import By #使用By查找元素 driver_path='D:\chromedriver\chromedriver' driver=webdriver.Chrome(executable_path=driver_path) driver.get('https://www.baidu.com') inputTag=driver.find_element(By.ID,'kw') inputTag.send_keys('python') # get_attribute('innerHTML')的方式可以获取某个元素的html # get_attribute('outerHTML')
4.操作表单元素
# -*-coding:utf8 -*- # 常见的表单元素 # button # checkbox # select，下拉列表 # input from selenium import webdriver from selenium.webdriver.common.by import By import time driver_path = r'D:\chromedriver\chromedriver' driver = webdriver.Chrome(executable_path=driver_path) # driver.get('https://www.baidu.com') # # inputTag = driver.find_element(By.ID, 'kw') # #输入内容python # inputTag.send_keys('python') # time.sleep(5) # #清除输入的内容 # inputTag.clear() #checkbox 选中 # driver.get('https://www.douban.com') # rememberBth=driver.find_element_by_name('remember') # rememberBth.click() #选择select # driver.get('http://www.dobai.cn/') from selenium.webdriver.support.ui import Select #使用选择，要用Select先进行实例化 # selectBtn=Select(driver.find_element_by_name('jumpMenu')) #通过索引进行选择 # selectBtn.select_by_index(1) #通过value进行选择 # selectBtn.select_by_value('http://m.95xiu.com/') #通过可见文本选择 # selectBtn.select_by_visible_text('95秀客户端') #取消所有选中 # selectBtn.deselect_all() #按钮的点击事件 driver.get('https://www.baidu.com') inputTag=driver.find_element_by_id('kw') inputTag.send_keys('python') submitTag=driver.find_element_by_id('su') time.sleep(5) submitTag.click()
5.行为链
# -*-coding:utf8 -*- #行为链 #有时候在页面中的操作可能要有很多步，那么这时候可以使用鼠标行为链类ActionChains来完成。 #行为链在爬虫中使用不是很多 from selenium import webdriver from selenium.webdriver.common.action_chains import ActionChains import time driver_path = r'D:\chromedriver\chromedriver' driver = webdriver.Chrome(executable_path=driver_path) driver.get('https://www.baidu.com') inputTag=driver.find_element_by_id('kw') submitBtn=driver.find_element_by_id('su') actions=ActionChains(driver) actions.move_to_element(inputTag) actions.send_keys_to_element(inputTag,'python') actions.move_to_element(submitBtn) actions.click() actions.perform() #还有更多的鼠标相关的操作。 #click_and_hold(element):点击但不松开鼠标 #context_click(element):右键点击 #double_click(element):双击
6.操作cookie
# -*-coding:utf8 -*- #cookie操作 #1.获取所有的cookie # driver.get_cookies() #2.根据cookie的key获取value # value=driver.get_cookies(key) #3.删除所有的cookie # driver.delete_all_cookies() #4.删除某个cookie # driver.delete_cookie(key) import time from selenium import webdriver driver_path=r'D:\chromedriver\chromedriver' driver=webdriver.Chrome(executable_path=driver_path) #只能获得当前网页https://www.baidu.com的所有cookie信息，并不能获取其他页面的cookie信息 driver.get('https://www.baidu.com') # for cookie in driver.get_cookies(): # print(cookie) print(driver.get_cookie('PSTM')) # driver.delete_cookie('PSTM') # print(driver.get_cookie('PSTM')) #删除所有cookie # driver.delete_all_cookies()
7.隐式等待和显式等待
# -*-coding:utf8 -*- from selenium import webdriver from selenium.webdriver.common.action_chains import ActionChains import time driver_path = r'D:\chromedriver\chromedriver' driver = webdriver.Chrome(executable_path=driver_path) # 页面等待： # 现在的网页越来越多采用了Ajax技术，这样程序便不能确定何时某个元素完全加载出来了。 # 如果实际页面等待时间过长导致某个dom元素还没出来，但是代码直接使用了这个webElement， # 那么就会抛出NullPointer的异常。为了解决这个问题，所以Selenium提供了 # 两种等待方式：一种是隐式等待、一种是显式等待 # 1.隐式等待：调用driver.implicitly_wait。那么在获取不可用的元素之前，会先等待10秒 driver.get('https://www.douban.com/') #不设置等待时间，马上报错 # driver.find_element_by_id('shdiasjdsdas') #设置等待时间 # driver.implicitly_wait(20) # driver.find_element_by_id('shdiasjdsdas')#等待20s才报错 # 2.显式等待：显式等待是表名某个条件成立后才执行获取元素的操作。也可以在等待的时候指定一个 #显示等待更加智能一点 # 最大的时间，如果超过这个时间那么就抛出一个异常 from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.common.by import By #后面可以接一个条件 # WebDriverWait(driver,10).until( # #某个元素加载完成,只能传一个参数.所以以一个元组传进去 # EC.presence_of_element_located((By.ID,'asdasdasdasda')) # ) #如果能获取的话，则不会等待10s element=WebDriverWait(driver,10).until( EC.presence_of_element_located((By.ID,'anony-book')) ) print(element)
8.打开多窗口和切换窗口
# -*-coding:utf8 -*- #切换页面： #有时候窗口中有很多子tab页面，这时候肯定是需要进行切换的。selenium提供了一个叫做switch_to_window #来进行切换，具体切换到哪个页面，可以从driver.window_handles中找到 from selenium import webdriver import time driver_path = r'D:\chromedriver\chromedriver' driver = webdriver.Chrome(executable_path=driver_path) driver.get('https://www.baidu.com') #又打开一个豆瓣页面 driver.execute_script("window.open('https://www.douban.com')") #但是打印当前所在网页的话，显示还是百度，要去豆瓣页面的话，需要进行切换 print(driver.current_url) #switch_to_window来进行切换页面 #要切换页面首先要拿到窗口句柄，driver每打开一个页面，会有一个句柄 #放在wind_handles里 # print(driver.window_handles) driver.switch_to.window(driver.window_handles[1]) print(driver.current_url)
9.使用代理
# -*-coding:utf8 -*- #设置代理ip： #有时候频繁爬取一些网页，服务器发现你是爬虫后会封掉你的ip地址。这时我们可以更换代理ip。 from selenium import webdriver options=webdriver.ChromeOptions() options.add_argument('--proxy-server=http://113.124.87.163:9999') driver_path = r'D:\chromedriver\chromedriver' driver=webdriver.Chrome(executable_path=driver_path,chrome_options=options) driver.get('http://httpbin.org/ip')
10.补充
# -*-coding:utf8 -*- from selenium import webdriver from selenium.webdriver.remote.webdriver import WebElement driver_path = r'D:\chromedriver\chromedriver' driver=webdriver.Chrome(executable_path=driver_path) driver.get('https://www.baidu.com') submitBtn=driver.find_element_by_id('su') # print(type(submitBtn)) print(submitBtn.get_attribute('value')) driver.save_screenshot('baidu.png')#保存屏幕截图
11.实战：爬取拉勾网
# -*-coding:utf8 -*- import re import requests import time from lxml import etree headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36', 'Referer': 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=', 'Cookie':'JSESSIONID=ABAAABAAAGFABEFFA5F21EB50BF5A6DCE619C8EEA6CB14A; SEARCH_ID=1146364cc73d498abea7c5b4dde4c1e3; user_trace_token=20190417144437-71ba273c-c709-43be-ae40-d1c531c2a4d7; X_HTTP_TOKEN=42daf4b72327b2817743845551bf5e71415983ed09' } #拉勾设置了反爬机制，每次cookie都会变，该怎么做？后面再学 def request_list_page(): url = 'https://www.lagou.com/jobs/positionAjax.json?needAddtionalResult=false' data = { 'first': False, 'pn': 1, 'kd': 'python' } for x in range(1, 14): data['pn'] = x response = requests.post(url, headers=headers, data=data) result = response.json() positions = result['content']['positionResult']['result'] for position in positions: positionId = position['positionId'] position_url = 'http://www.lagou.com/jobs/%s.html' % positionId parse_position_detail(position_url) break break # print(response.json()) # json方法，如果返回来的是json数据，这个方法会自动load成字典 def parse_position_detail(url): response = requests.get(url, headers=headers) text = response.text html = etree.HTML(text) position_name = html.xpath('//span[@class="name"]/text()')[0] job_request_spans = html.xpath('//dd[@class="job_request"]//span') salary_span = job_request_spans[0] salary = salary_span.xpath('.//text()')[0].strip() city = job_request_spans[1].xpath('.//text()')[0].strip() city = re.sub(r'[\s/]', '', city, re.S) work_years = job_request_spans[2].xpath('.//text()')[0].strip() work_years = re.sub(r'[\s/]', '', work_years, re.S) education = job_request_spans[3].xpath('.//text()')[0].strip() education = re.sub(r'[\s/]', '', education, re.S) desc = "".join(html.xpath("//dd[@class='job_bt']//text()")).strip() position={ 'position_name':position_name, 'salary':salary, 'city':city, 'work_years':work_years, 'education':education, 'desc':desc, } print(position) def main(): request_list_page() if __name__ == '__main__': main()
12.实战：爬取拉勾网
# -*-coding:utf8 -*- from selenium import webdriver from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as EC from lxml import etree from selenium.webdriver.common.by import By import re import time class LagouSpider(object): driver_path = r'D:\chromedriver\chromedriver' def __init__(self): self.driver = webdriver.Chrome(executable_path=LagouSpider.driver_path) self.url = 'https://www.lagou.com/jobs/list_python?labelWords=&fromSearch=true&suginput=' self.positions = [] def run(self): self.driver.get(self.url) while True: # self.driver.page_source拿到的是整个网页源代码信息，ajax请求的数据也能看到 source = self.driver.page_source WebDriverWait(driver=self.driver, timeout=10).until( #这个等待加载的XPATH不要加text(),否则会异常 EC.presence_of_element_located((By.XPATH, '//div[@class="pager_container"]/span[last()]')) ) self.parse_list_page(source) try: # 一页爬完后，点击下一页,继续爬取 next_btn = self.driver.find_element_by_xpath('//div[@class="pager_container"]/span[last()]') # 判断一下，下一页是否可以点击，因为最后一页的时候，下一页是无法点击的 if "pager_next_disabled" in next_btn.get_attribute('class'): pass else: next_btn.click() time.sleep(7) except Exception: print(source) def parse_list_page(self, source): html = etree.HTML(source) links = html.xpath('//a[@class="position_link"]/@href') for link in links: # 当前的页面一个一个的请求 # 设置请求频率 self.request_detail_page(link) time.sleep(10) def request_detail_page(self, url): # 切换窗口，打开新的页面 self.driver.execute_script("window.open('%s')" % url) # 切换到新的窗口上去 self.driver.switch_to.window(self.driver.window_handles[1]) source = self.driver.page_source WebDriverWait(self.driver, timeout=10).until( EC.presence_of_element_located((By.XPATH, '//span[@class="name"]')) ) self.parse_detail_page(source) # close()是关闭当前页面 self.driver.close() # 切换回职位列表页 self.driver.switch_to.window(self.driver.window_handles[0]) def parse_detail_page(self, source): html = etree.HTML(source) position_name = html.xpath('//span[@class="name"]/text()')[0] job_request_spans = html.xpath('//dd[@class="job_request"]//span') salary_span = job_request_spans[0] salary = salary_span.xpath('.//text()')[0].strip() city = job_request_spans[1].xpath('.//text()')[0].strip() city = re.sub(r'[\s/]', '', city, re.S) work_years = job_request_spans[2].xpath('.//text()')[0].strip() work_years = re.sub(r'[\s/]', '', work_years, re.S) education = job_request_spans[3].xpath('.//text()')[0].strip() education = re.sub(r'[\s/]', '', education, re.S) desc = "".join(html.xpath("//dd[@class='job_bt']//text()")).strip() company_name=html.xpath("//h2[@class='fl']/text()")[0].strip() position = { 'name': position_name, 'company_name':company_name, 'salary': salary, 'city': city, 'work_years': work_years, 'education': education, 'desc': desc } self.positions.append(position) print(position) print('=' * 40) if __name__ == '__main__': spider = LagouSpider() spider.run() print(spider.positions)