python爬虫—selenium模块、12306模拟登录
回顾：单线程+多任务，速度会很快。
server.py
import time
from flask import Flask, render_template
app = Flask(__name__)
@app.route('/index')
def index():
time.sleep(2)
return render_template('test.html')
@app.route('/index1')
def index1():
time.sleep(2)
return render_template('test.html')
@app.route('/index2')
def index2():
time.sleep(2)
return render_template('test.html')
if __name__ == '__main__':
app.run()
jiexi.py
import aiohttp
import asyncio
import time
from lxml import etree
start = time.time()
urls = [
'http://127.0.0.1:5000/index',
'http://127.0.0.1:5000/index1',
'http://127.0.0.1:5000/index2',
]
# 特殊的函数:请求发送和响应数据的捕获
# 细节1：在每一个with前加上async
# 细节2：在每一个阻塞操作的前边加上await
async def get_request(url):
async with aiohttp.ClientSession() as s:
# with s.get(url, headers,proxy="http://ip:port")
async with await s.get(url) as response:
page_text = await response.text() # read()返回的时byte
return page_text
# 回调函数
def parse(task):
page_text = task.result()
tree = etree.HTML(page_text)
parse_data = tree.xpath('//li/text()')
print(parse_data)
tasks = []
for url in urls:
c = get_request(url)
task = asyncio.ensure_future(c)
task.add_done_callback(parse)
tasks.append(task)
loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))
print(time.time() - start)
"""
['i am hero', 'i am superman', 'i am spider']
['i am hero', 'i am superman', 'i am spider']
['i am hero', 'i am superman', 'i am spider']
2.0393407344818115
"""
selenium模块在爬虫中的使用
- 概念：是一个基于浏览器自动化的模块。
- 爬虫之间的关联：
- 便捷的捕获到动态加载的数据。（可见即可得）
- 实现模拟登录
- 环境安装：pip install selenium
- 基本使用：
- 准备号某一款浏览器的驱动程序（谷歌）：https://chromedriver.storage.googleapis.com/index.html 谷歌驱动版本要对应
import time
from selenium import webdriver
bro = webdriver.Chrome(executable_path='chromedriver.exe')
bro.get('https://www.jd.com')
time.sleep(1)
# 进行标签定位
search_input = bro.find_element_by_id('key')
search_input.send_keys('mac pro')
btn = bro.find_element_by_xpath('//*[@id="search"]/div/div[2]/button')
btn.click()
time.sleep(2)
# 滚轮转动
# window.scrollTo(0,document.body.scrollHeight)
# 执行js
bro.execute_script('window.scrollTo(0,document.body.scrollHeight)')
time.sleep(2)
# 获取页面源码数据
page_text = bro.page_source
print(page_text)
time.sleep(2)
bro.quit()
动态获取数据
"""
爬取药监总局相关企业的详细信息：
http://scxk.nmpa.gov.cn:81/xk/
"""
from time import sleep
from selenium import webdriver
from lxml import etree
bro = webdriver.Chrome(executable_path='chromedriver.exe')
bro.get('http://scxk.nmpa.gov.cn:81/xk/')
sleep(1)
page_text = bro.page_source
page_text_list = [page_text]
for i in range(3):
bro.find_element_by_id('pageIto_next').click() # 点击下一页
sleep(1)
page_text_list.append(bro.page_source)
for page_text in page_text_list:
tree = etree.HTML(page_text)
li_list = tree.xpath('//*[@id="gzlist"]/li')
for li in li_list:
title = li.xpath('./dl/@title')[0]
num = li.xpath('./ol/@title')[0]
print(title+':'+num)
sleep(2)
bro.quit()
模拟登录12306
-
动作量
-
一系列连续的动作
-
from selenium.webdriver import ActionChains
-
实现标签定位时，如果发现定位的标签是存在于iframe标签之中的，则在定位时必须执行一个固定的操作：bro.switch_to.fram(‘iframe_id’)
-
拖到=点击+滑动
-
action =ActionChains(bro) action.click_and_hold(div_tag) for i in range(5): # perform 让动作链立即执行 action.move_by_offset(17, 5).perform() action.release()
-
模拟登录
"""
https://www.cnblogs.com/bobo-zhang/articles/10815914.html
模拟登录12306：
https://kyfw.12306.cn/otn/login/init
"""
from time import sleep
from selenium import webdriver
from chaojiying_Python.chaojiying import Chaojiying_Client
from selenium.webdriver import ActionChains
from PIL import Image
from lxml import etree
def get_text(imgPath, imgType):
chaojiying = Chaojiying_Client('用户名', '密码', '911057') # 用户中心>>软件ID 生成一个替换 96001
im = open('imgPath', 'rb').read() # 本地图片文件路径 来替换 a.jpg 有时WIN系统须要//
return chaojiying.PostPic(im, imgType)['pic_str']
bro = webdriver.Chrome(executable_path='chromedriver.exe')
bro.get('https://kyfw.12306.cn/otn/login/init')
sleep(1)
# 截图
bro.save_screenshot('main.png')
code_img_tag = bro.find_element_by_xpath('//*[@id="loginForm"]/div/ul[2]/li[4]/div/div/div[3]/img')
location = code_img_tag.location # 左下角的坐标
size = code_img_tag.size # 长和宽
print(location, size)
# 裁剪的区域范围
rangle = (int(location['x']), int(location['y']), int(location['x']+size['width']), int(location['y']+size['height']))
i = Image.open('main.png')
code_img_name = 'code.png'
frame = i.crop(rangle)
frame.save(code_img_name)
result = get_text('code.png', 9004)
# print(img_location) # '55,70|267,133' == [[55,70], [267,133]]
all_list = []
if '|' in result:
list_1 = result.split('|')
count_1 = len(list_1)
for i in range(count_1):
xy_list = []
x = int(list_1[i].split(',')[0])
y = int(list_1[i].split(',')[1])
xy_list.append(x)
xy_list.append(y)
all_list.append(xy_list)
else:
x = int(result.split(',')[0])
y = int(result.split(',')[1])
xy_list = []
xy_list.append(x)
xy_list.append(y)
all_list.append(xy_list)
print(all_list)
for a in all_list:
x = a[0]
y = a[1]
action = ActionChains(bro)
# 先把坐标原点偏移到验证码的原点
action.move_to_element_with_offset(code_img_tag, x, y).click().perform()
sleep(1)
bro.find_element_by_id('username').send_keys('[email protected]')
bro.find_element_by_id('password').send_keys('123456')
bro.find_element_by_id('loginSub').click()
sleep(3)
# bro.quit()