代理
爬虫中为什么需要使用代理
一些网站会有相应的反爬虫措施，例如很多网站会检测某一段时间某个IP的访问次数，如果访问频率太快以至于看起来不像正常访客，它可能就会会禁止这个IP的访问。
所以我们需要设置一些代理IP，每隔一段时间换一个代理IP，就算IP被禁止，依然可以换个IP继续爬取。
代理的分类
正向代理：代理客户端获取数据。正向代理是为了保护客户端防止被追究责任。
反向代理：代理服务器提供数据。反向代理是为了保护服务器或负责负载均衡。
免费代理ip提供网站
http://www.goubanjia.com/ 西祠代理 快代理
简单运用示例
import requests # ip 代理网站 http://www.goubanjia.com/ headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' } url = 'https://www.baidu.com/s?wd=ip' page_text = requests.get(url=url,headers=headers,proxies={'https':'212.119.229.18:33852'}).text # print(page_text) with open('./ip.html','w',encoding = 'utf-8') as f: f.write(page_text) # proxy = [{},{},{}] 代理池的使用
乱码
import requests from urllib import request from lxml import etree headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36', 'Connection':'close' } url = "http://pic.netbian.com/4kmeinv/" response = requests.get(url,headers) response.encoding = 'gbk' # print(response.encoding) page_text = response.text # print(page_text) tree = etree.HTML(page_text) li_list = tree.xpath('//div[@class="slist"]/ul/li') for li in li_list: img_src = 'http://pic.netbian.com' + li.xpath('./a/img/@src')[0] img_name = li.xpath('./a/img/@alt')[0] # img_name = img_name.encode('gbk').decode('utf-8') # img_name = img_name.encode('ISO-8859-1').decode('gbk') # ISO-8859-1 范围 大于 gbk 大于 utf-8 print(img_src,img_name)
验证码
1. 云打码平台
需要云打码平台在线解析验证码
1.官方下载的 不用动 直接加载就可以
import http.client, mimetypes, urllib, json, time, requests ###################################################################### class YDMHttp: apiurl = 'http://api.yundama.com/api.php' username = '' password = '' appid = '' appkey = '' def __init__(self, username, password, appid, appkey): self.username = username self.password = password self.appid = str(appid) self.appkey = appkey def request(self, fields, files=[]): response = self.post_url(self.apiurl, fields, files) response = json.loads(response) return response def balance(self): data = {'method': 'balance', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey} response = self.request(data) if (response): if (response['ret'] and response['ret'] < 0): return response['ret'] else: return response['balance'] else: return -9001 def login(self): data = {'method': 'login', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey} response = self.request(data) if (response): if (response['ret'] and response['ret'] < 0): return response['ret'] else: return response['uid'] else: return -9001 def upload(self, filename, codetype, timeout): data = {'method': 'upload', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'codetype': str(codetype), 'timeout': str(timeout)} file = {'file': filename} response = self.request(data, file) if (response): if (response['ret'] and response['ret'] < 0): return response['ret'] else: return response['cid'] else: return -9001 def result(self, cid): data = {'method': 'result', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid)} response = self.request(data) return response and response['text'] or '' def decode(self, filename, codetype, timeout): cid = self.upload(filename, codetype, timeout) if (cid > 0): for i in range(0, timeout): result = self.result(cid) if (result != ''): return cid, result else: time.sleep(1) return -3003, '' else: return cid, '' def report(self, cid): data = {'method': 'report', 'username': self.username, 'password': self.password, 'appid': self.appid, 'appkey': self.appkey, 'cid': str(cid), 'flag': '0'} response = self.request(data) if (response): return response['ret'] else: return -9001 def post_url(self, url, fields, files=[]): for key in files: files[key] = open(files[key], 'rb'); res = requests.post(url, files=files, data=fields) return res.text
2.普通用户的调用
def get_code_text(code_type,img_path): # 用户名 username = '普通用户账号 # 密码 password = 普通用户密码 # 软件ＩＤ，开发者分成必要参数。登录开发者后台【我的软件】获得！ appid = 6578 # 软件密钥，开发者分成必要参数。登录开发者后台【我的软件】获得！ appkey = '2b3ef98633145e0b478800905af4e10b' # 图片文件 filename = img_path # 验证码类型，# 例：1004表示4位字母数字，不同类型收费不同。请准确填写，否则影响识别率。在此查询所有类型 http://www.yundama.com/price.html codetype = code_type # 1004 # 超时时间，秒 timeout = 25 # 检查 if (username == 'username'): print('请设置好相关参数再测试') else: # 初始化 yundama = YDMHttp(username, password, appid, appkey) # 登陆云打码 uid = yundama.login(); print('uid: %s' % uid) # 查询余额 balance = yundama.balance(); print('balance: %s' % balance) # 开始识别，图片路径，验证码类型ID，超时时间（秒），识别结果 cid, result = yundama.decode(filename, codetype, timeout); print('cid: %s, result: %s' % (cid, result)) return result
模拟登陆后抓取个人信息数据
import requests from lxml import etree from urllib import request # 获取一个session对象 session = requests.Session() #session对象和requests作用几乎一样,都可以进行请求的发送,并且请求发送的方式也是一致的, #session进行请求的发送,如果会产生cookie的话,则cookie会自动被存储到session对象中 #1 获取验证码图片 url = 'http://www.renren.com/' headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' } page_text = session.get(url=url,headers=headers).text tree = etree.HTML(page_text) code_img_src = tree.xpath('//*[@id="verifyPic_login"]/@src')[0] request.urlretrieve(url=code_img_src,filename='./code.jpg') # 保存到本地 code = get_code_text('2004','./code.jpg') # 获取验证码文本信息 # 模拟登陆 login_url = 'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2019031945879' # data信息需要抓包工具 获取
data = { "email":人人账号, "icode":code if code else '', "origURL":"http://www.renren.com/home", "domain":"renren.com", "key_id":"1", "captcha_type":"web_login", "password":通过抓包工具获取对应账号的加密密码, "rkey":"d4287c72b3f7ddf41b62170adec10265", "f":"http%3A%2F%2Fwww.renren.com%2F969397225", } print(session) #进行登录,当登录成功之后,可以获取cookie response = session.post(url=login_url,headers=headers,data=data) # #对登录成功后对应的当前用户的个人详情页进行请求发送 detail_url = "http://www.renren.com/969397225/profile" # 个人信息网址 page_text = session.get(url=detail_url,headers=headers).text with open('./renren.html','w',encoding='utf-8') as fp: fp.write(page_text) print('over')
古诗网
第一步 和人人一样调用云打码
主代码
import requests from lxml import etree from urllib import request # 获取一个session对象 session = requests.Session() #session对象和requests作用几乎一样,都可以进行请求的发送,并且请求发送的方式也是一致的, #session进行请求的发送,如果会产生cookie的话,则cookie会自动被存储到session对象中 headers = { 'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36' } url = 'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx' page_text = requests.get(url=url,headers=headers).text # print(page_text) tree = etree.HTML(page_text) code_img_src = 'https://so.gushiwen.org' + tree.xpath('//*[@id="imgCode"]/@src')[0] # 获取验证码生成地址 img_data = session.get(url=code_img_src,headers=headers).content # print(img_data) with open('./gushi.jpg',"wb") as fp: fp.write(img_data) code_text = get_code_text('1004','./gushi.jpg') print(code_text) # 每次访问都会修改 每次访问都获取依稀 __VIEWSTATE = tree.xpath('//*[@id="__VIEWSTATE"]/@value')[0] __VIEWSTATEGENERATOR = tree.xpath('//*[@id="__VIEWSTATEGENERATOR"]/@value')[0] login_url = "https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx" data = { "__VIEWSTATE":__VIEWSTATE, # 动态获取 "__VIEWSTATEGENERATOR":__VIEWSTATEGENERATOR, # 动态获取 "from":"http://so.gushiwen.org/user/collect.aspx", "email":"古诗文网站好", "pwd":"古诗文网的密码.", "code":code_text, "denglu":"登录", } page_text = session.post(url=login_url,headers=headers,data=data).text with open('./gushiwen.html','w',encoding='utf-8') as fp: fp.write(page_text) print('ok')