8.14爬虫笔记2

一、封装代码
from urllib import request,parse
from urllib.error import HTTPError,URLError
import json
from http import cookiejar

class session():
def init(self):
cookie_object = cookiejar.CookieJar()
handler = request.HTTPCookieProcessor(cookie_object)
self.opener = request.build_opener(handler)
def get(self,url,headers=None,):
return get(url,headers,opener=self.opener)

def post(self,url,form=None,headers=None,):
    return post(url, headers, opener=self.opener)

def get(url,headers=None,opener=None):
”’
:param url: get 请求的路径
:param headers: 请求头
:return: 二进制源代码
”’
html_byte = urlrequests(url,headers=headers,opener=opener)
return html_byte

def post(url,form=None,headers=None,opener=None):
”’
:param url: post 请求的路径
:param form: 请求的参数
:param headers: post 请求头
:return: json 列表
”’
html_byte = urlrequests(url,form=form,headers=headers,opener=opener)

return html_byte

def urlrequests(url,form=None,headers=None,opener=None):
if headers == None:
user_Agent = ‘User-Agent: Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36’
headers = {
‘User_Agent’: user_Agent
}
html_byte = b”

try:
    if form == None:
        req = request.Request(url, headers=headers)
    else:
        form_dict = form
        form_byte = parse.urlencode(form_dict)
        req = request.Request(url,data=form_byte.encode('utf-8'),headers=headers)
    if opener:
        response = opener.open(req)
    else:
        response = request.urlopen(req)
    html_byte = response.read()
except HTTPError as e:
    print(e)
except URLError as e:
    print(e)
return html_byte

if name == ‘main‘:
url = ‘http://www.baidu.com’
res = get(url)
print(res.decode())

# url = 'http://fanyi.baidu.com/sug'
# res = post(url,form={'kw':'漂亮'})
# print(json.loads(res.decode('utf-8')))

二、有道翻译并引入封装的代码模块一
import time
import random
import json
from daima_fengzhuang import post

def md5_my(need_str):
import hashlib

# 创建md5对象
md5_o = hashlib.md5()
# 需要有bytes, 作为参数
# 由str, 转换成 bytes encode-------str.encode('utf-8')
# 由bytes转换成 str, decode---------bytes.decode('utf-8')
sign_bytes = need_str.encode('utf-8')
print(type(sign_bytes))
# 更新md5 object的值
md5_o.update(sign_bytes)
sign_str = md5_o.hexdigest()
return sign_str

＃url

def translate(kw):
url = ‘http://fanyi.youdao.com/translate_o?smartresult=dict&smartresult=rule’

headers = {
    'Accept': 'application/json, text/javascript, */*; q=0.01',
    #'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Connection': 'keep-alive',
    #'Content-Length': '223',
    'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
    'Cookie': '[email protected]; OUTFOX_SEARCH_USER_ID_NCOO=38624120.26076847; SESSION_FROM_COOKIE=unknown; JSESSIONID=aaabYcV4ZOU-JbQUha2uw; ___rl__test__cookies=1534210912076',
    'Host': 'fanyi.youdao.com',
    'Origin': 'http://fanyi.youdao.com',
    'Referer': 'http://fanyi.youdao.com/',
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest',
}


# form 的生成1. i 需要确定, 2, salt, 3, sign
key= kw

# salt : ((new Date).getTime() + parseInt(10 * Math.random(), 10))
salt = int(time.time()*1000 + random.randint(0,10))
print(salt)
salt_str = str(salt)

# sign : o = u.md5(S + n + r + D);
# S = "fanyideskweb"
# D = "ebSeFb%=XZ%T[KZ)c(sy!"
# n = key
# r = salt_str
S = "fanyideskweb"
D = "ebSeFb%=XZ%T[KZ)c(sy!"
sign_str = S + key + salt_str + D
# md5 加密的方法
sign_md5_str = md5_my(sign_str)

form = {
    'i': key,
    'from': 'AUTO',
    'to': 'AUTO',
    'smartresult': 'dict',
    'client': 'fanyideskweb',
    'salt': salt_str,
    'sign': sign_md5_str,
    'doctype': 'json',
    'version': '2.1',
    'keyfrom': 'fanyi.web',
    'action': 'FY_BY_REALTIME',
    'typoResult': 'false',
}

html_bytes = post(url, form, headers=headers)

# 将 json 类型的 str, 转化成, 字典
res_dict = json.loads(html_bytes.decode('utf-8'))
#print(html_bytes.decode('utf-8'))

translate_res = res_dict['translateResult'][0][0]['tgt']

return translate_res

if name == ‘main‘:
ret = translate(‘青青河边草’)

print('青青河边草的翻译是:' + ret)

三、人人网爬取登陆页面
from daima_fengzhuang import get

＃0.url
＃1.构造headers
＃2.调用get函数
＃3.保存页面

url = ‘http://www.renren.com/967453648’
headers = {
＃ Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,/;q=0.8
＃ Accept-Encoding: gzip, deflate
＃ Accept-Language: zh-CN,zh;q=0.9
＃ Cache-Control: max-age=0
＃ Connection: keep-alive
‘Cookie’: ‘anonymid=jkahzzcq-z7ueyh; r01=1; depovince=GW; JSESSIONID=abcrTQncYla9welS2K2uw; ick_login=8c84eb0f-b12f-4bd8-9fb3-216cf65fcebe; ick=db03341e-0fdd-4171-9ea1-e8c1b2f93e5a; jebe_key=c11e281d-08b8-40c1-bda9-2e48a5a22c5b%7C28fbe2bcce822b7cb115297c595cdcde%7C1534232609195%7C1%7C1534232616415; first_login_flag=1; ln_uact=13132198676; ln_hurl=http://head.xiaonei.com/photos/0/0/men_main.gif; wp_fold=0; _ga=GA1.2.184601674.1534251988; _gid=GA1.2.1138286915.1534251988; jebecookies=459b076e-44c9-444d-a750-61d4d7472f67|||||; _de=14150AE6CF2592C32702E4F15A940482; p=77bf6a2ce99e7e32f44878e63a08a3958; t=17c56604318619657413dfe323a252778; societyguester=17c56604318619657413dfe323a252778; id=967453648; xnsid=c771f5d7; ver=7.0; loginfrom=null’,
＃Host: www.renren.com
＃Referer: http://www.renren.com/SysHome.do
＃Upgrade-Insecure-Requests: 1
＃User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36
}

html = get(url,headers=headers)
with open(‘renren.html’,’wb’) as f:

四、人人网 cookie 爬取
from daima_fengzhuang import post,get
import json
from urllib import request,parse

＃保护cookie
from http import cookiejar
＃通过对象保存cookie
cookie_object = cookiejar.CookieJar()
＃handler 对应一个操作
handler = request.HTTPCookieProcessor(cookie_object)
＃opener遇到有cookie的response的时候，
＃调用handler内部的一个函数，存储到cookie object
opener = request.build_opener(handler)

＃cookiejar
＃ def store(cookie):
＃ cookie_object.append(cookie)
#
＃ def check_response(response):
＃ if ‘cookie’ in response:
＃ handler.store(response[‘cookie’])

url = ‘http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=201872213137’

＃form
form = {
‘email’: ‘13132198676’,
‘icode’: ”,
‘origURL’: ‘http://www.renren.com/home‘,
‘domain’: ‘renren.com’,
‘key_id’: ‘1’,
‘captcha_type’: ‘web_login’,
‘password’: ‘3d0a93ad51e5b2cd109e3b6a795c9cd89a97f105b47348338ea199a9eb78bf6f’,
‘rkey’: ‘f7f882bf398b60106ad169618066b5c7’,
‘f’: ‘http%3A%2F%2Fwww.renren.com%2F967453648’,
}

＃post
form_bytes = parse.urlencode(form).encode(‘utf-8’)

＃ response = request.urlopen(url, form_bytes)
＃ opener = request.build_opener()
＃ opener.open()
response = opener.open(url,form_bytes)

html_bytes = response.read()
＃html_bytes = post(url, form=form)
＃打印结果
＃print(html_bytes)
＃通过json获取一个字典类型
res_dict = json.loads(html_bytes.decode(‘utf-8’))

home_url = res_dict[‘homeUrl’]

＃访问页面
response = opener.open(home_url)
html_bytes = response.read()
print(html_bytes.decode(‘utf-8’))

五、人人网 session 爬取
from daima_fengzhuang import session
import json
＃ url
url = ‘http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=2018721441132’
＃ form
form = {
‘email’: ‘13132198676’,
‘icode’: ”,
‘origURL’: ‘http://www.renren.com/home‘,
‘domain’: ‘renren.com’,
‘key_id’: ‘1’,
‘captcha_type’: ‘web_login’,
‘password’: ‘3d0a93ad51e5b2cd109e3b6a795c9cd89a97f105b47348338ea199a9eb78bf6f’,
‘rkey’: ‘f7f882bf398b60106ad169618066b5c7’,
‘f’: ‘http%3A%2F%2Fwww.renren.com%2F967453648’,
}
＃调用class
s = session()
＃封装的post
html_bytes = s.post(url, form)

＃html_bytes = post(url, form=form)
＃打印结果
＃print(html_bytes)
＃通过json获取一个字典类型
print(html_bytes)
res_dict = json.loads(html_bytes.decode(‘utf-8’))

home_url = res_dict[‘homeUrl’]

＃访问页面
html_bytes = s.get(home_url)
print(html_bytes.decode(‘utf-8’))

猜你喜欢