python利用代理爬取数据、处理cookie、处理验证码登入

一.利用代理

http连接池耗尽：

headers中加入Conection:"close"

import requests
headers = {
    
    
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
}
url = 'https://www.baidu.com/s?wd=ip'

#使用代理
page_text = requests.get(url,headers=headers,proxies={
    
    'https':'121.226.188.232:9999'}).text

with open('./ip.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

import requests
import random
headers = {
    
    
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36'
}
url = 'https://www.baidu.com/s?wd=ip'

proxy_list = [
    {
    
    'https':'121.226.188.232:9999'},
    {
    
    'https':'111.226.188.232:9999'},
    {
    
    'https':'133.226.188.232:9999'}
]

#从列表里随机取ip
page_text = requests.get(url,headers=headers,proxies=random.choice(proxy_list)).text

with open('./ip.html','w',encoding='utf-8') as fp:
    fp.write(page_text)

返回响应状态码：

response.status_code == 200

二.处理cookie
手动处理cookie:

import requests
url = 'https://xueqiu.com/statuses/hot/listV2.json?since_id=-1&max_id=183926&size=15'
headers = {
    
    
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36',
    #headers里面添加cooKie
    'Cookie':'xq_a_token=a4b3e3e158cfe9745b677915691ecd794b4bf2f9; xqat=a4b3e3e158cfe9745b677915691ecd794b4bf2f9; xq_r_token=b80d3232bf315f8710d36ad2370bc777b24d5001; xq_id_token=eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJ1aWQiOi0xLCJpc3MiOiJ1YyIsImV4cCI6MTYxNzc2MzQxOCwiY3RtIjoxNjE2Mzc3NTkwNzQ2LCJjaWQiOiJkOWQwbjRBWnVwIn0.Y_1uREgZCeK_mxRxDyfB18jcmswaxABJeky8fXbM06gd3s-1MGjYyCYhhmLp2FsX4XGjXmNpSwaIG0H2bqFOTYrkhuA4PeRrRb_bes68qplrF-1NSt37BkiHXsEtVk8-qlheFokeVp1bda4n_K4Etw2q47h2ZVPHnv_YAaBNV6wuIe7STC-sVnllpmiciXXHniRc1O5GhE8_BEWki0O6MK3CdRO2kfoZv6tMKwLlXkkklMMPmA6ZgzTJ-pBhp6Jm9RFBrgZwu1hF0vsvqCA40vt6qqqQuK1F_EBxdq2_WyfYhPSFC68Lxzba1EbV1ZUqOFlaCLj0DPPWpSwlFVyuhw; u=611616377636831; device_id=24700f9f1986800ab4fcc880530dd0ed; Hm_lvt_1db88642e346389874251b5a1eded6e3=1616377638,1616377715; acw_tc=2760820016163797460667217e7e83aaa58d9dadbf315742715c423b896ecd; Hm_lpvt_1db88642e346389874251b5a1eded6e3=1616380377'
}
page_text = requests.get(url,headers=headers).json()
page_text

自动处理：

import requests
#创建一个session对象
s = requests.Session()
#获取cookie
s.get('https://xueqiu.com/',headers=headers)

url = 'https://xueqiu.com/statuses/hot/listV2.json?since_id=-1&max_id=183926&size=15'
headers = {
    
    
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36',
}
#携带cookie访问
page_text = s.get(url,headers=headers).json()
page_text

三.验证码处理
使用相关第三方平台来识别验证码，比如超级鹰（http://www.chaojiying.com/）

import requests
from lxml import etree
s = requests.Session()
url = 'https://so.gushiwen.cn/user/login.aspx?from=http://so.gushiwen.cn/user/collect.aspx'
headers = {
    
    
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.90 Safari/537.36',
}
page_text = s.get(url,headers=headers).text 
tree = etree.HTML(page_text)
img_src = 'https://so.gushiwen.cn' + tree.xpath('//*[@id="imgCode"]/@src')[0]

#动态变化的请求参数
__VIEWSTATE = tree.xpath('//*[@id="__VIEWSTATE"]/@value')[0]
__VIEWSTATEGENERATOR = tree.xpath('//*[@id="__VIEWSTATEGENERATOR"]/@value')

img_data = s.get(img_src,headers=headers).content
with open('./a.jpg','wb') as fp:
    fp.write(img_data)

#超级鹰代码，获取图片之后，开始用超级鹰来获取验证码
#!/usr/bin/env python
# coding:utf-8

import requests
from hashlib import md5

class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password =  password.encode('utf8')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
    
    
            'user': self.username,
            'pass2': self.password,
            'softid': self.soft_id,
        }
        self.headers = {
    
    
            'Connection': 'Keep-Alive',
            'User-Agent': 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
    
    
            'codetype': codetype,
        }
        params.update(self.base_params)
        files = {
    
    'userfile': ('ccc.jpg', im)}
        r = requests.post('http://upload.chaojiying.net/Upload/Processing.php', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
    
    
            'id': im_id,
        }
        params.update(self.base_params)
        r = requests.post('http://upload.chaojiying.net/Upload/ReportError.php', data=params, headers=self.headers)
        return r.json()
  
def tran(type):
    chaojiying = Chaojiying_Client('jiangyouren', 'qwertyuiop', '914312')
    im = open('./a.jpg', 'rb').read()
    return chaojiying.PostPic(im, type)['pic_str']

#获取验证码
code = tran(1004)
print(code)

#实现模拟登入
login_url = 'https://so.gushiwen.cn/user/login.aspx?from=http%3a%2f%2fso.gushiwen.cn%2fuser%2fcollect.aspx'
data = {
    
    
    '__VIEWSTATE':__VIEWSTATE,
    '__VIEWSTATEGENERATOR': __VIEWSTATEGENERATOR,
    'from': 'http://so.gushiwen.cn/user/collect.aspx',
    'email': '13077903313',
    'pwd': 'qwertyuiop',
    'code':code,
    'denglu': '登录'
}
page_text = s.post(login_url,headers=headers,data=data).text
with open('login.html','w',encoding='utf-8') as fp:
    fp.write(page_text)