21 days to build a distributed crawler -urllib library (a)

1.1.urlopen usage function

#encoding:utf-8

from urllib import request

res = request.urlopen("https://www.cnblogs.com/")

print(res.readlines())


#urlopen的参数
#def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
#            *, cafile=None, capath=None, cadefault=False, context=None):

1.2.urlretrieve function

Save the file on a web page to your local

#coding:utf-8

from urllib import request

res = request.urlretrieve("https://www.cnblogs.com/",'cnblog.html')


#urlretrieve参数
#def urlretrieve(url, filename=None, reporthook=None, data=None):

1.3 parameter encoding and decoding functions

urlencode function for encoding Chinese characters and special

# Urlencode function 

# simple use 
# from the urllib Import the parse 
# Data = { 'name': 'Drake', 'Age': 100} 
# QS = parse.urlencode (Data) 
# Print (QS) # name = E5% % E7 B7 the BE%%%% 91 is 9E% 8B E5% 85% Age = 100 & 

# actual use cases 
from the urllib Import Request, the parse 
URL = " http://www.baidu.com/s " 
the params = { " WD " : " blog Park " } 
QS = parse.urlencode (params) 
url = url + " ? " + qs
res = request.urlopen(url)
print(res.read())

parse_qs function for decoding coded parameters after the url.

from urllib import parse

qs = "name=%E5%BE%B7%E7%91%9E%E5%85%8B&age=100"
print(parse.parse_qs(qs))   #{'name': ['德瑞克'], 'age': ['100']}

1.4.urlparse and urlsplit function usage

urlparse和urlsplit都是用来对url的各个组成部分进行分割的，唯一不同的是urlsplit没有"params"这个属性.

from urllib import request,parse

url = "https://www.baidu.com/s?wd=cnblog#2"
result = parse.urlparse(url)
print(result)
#ParseResult(scheme='https', netloc='www.baidu.com', path='/s', params='', query='wd=cnblog', fragment='2')

print('scheme:',result.scheme)   #协议
print('netloc:',result.netloc)   #域名
print('path:',result.path)       Path#
Print ( ' Query: ' , result.query)      # query parameter 

# Results 
# scheme: HTTPS 
# netloc: www.baidu.com 
# path: / S 
# Query: WD = cnblog

1.5.Request pull hook climbing net jobs

Request class parameters

class Request:

    def __init__(self, url, data=None, headers={},
                 origin_req_host=None, unverifiable=False,
                 method=None):

Retractor climbing net jobs

Jobs pull hook inside the network is in Ajax.json

Code:

# Use the Request class climbing pull hook net jobs 

from urllib Import Request, the parse 

url = " https://www.lagou.com/jobs/positionAjax.json?city=%E5%8C%97%E4%BA%AC&needAddtionalResult to false = " 

# request header 
headers = {
     " the User-- Agent " : " the Mozilla / 5.0 (the Windows NT 6.1; Win64; x64-) AppleWebKit / 537.36 (KHTML, like the Gecko) the Chrome / 64.0.3282.140 Safari / 537.36 " ,
     " the Referer " : " https://www.lagou.com/jobs/list_python?city=%E5%8C%97%E4%BA%AC&cl=false&fromSearch=true&labelWords=&suginput= "
}
#post request to be submitted data 
Data = {
     ' First ' : ' to true ' ,
     ' PN ' :. 1 ,
     ' KD ' : ' Python ' 
} 
# Data post request data byte type must be encoded 
req = request.Request (URL, headers = headers, parse.urlencode = Data (Data) .encode ( ' UTF-. 8 ' ), Method = ' the POST ' )    # establish a request object 
RES = request.urlopen (REQ)
 # acquired information word section type, need to be decoded 
Print (res.read (). decode ( 'utf-8'))

1.6.ProxyHandler agent

Acting principle: before requesting purpose of the site, the first request to the proxy server, and then let the proxy server to request that the target site, access to data, and then return to us.

# Use of the agent 
from the urllib Import Request 

URL = " https://www.baidu.com/s?wd=cnblog " 

# 1. Construction of a handler used ProxyHandler incoming proxy 
# handler request.ProxyHandler = ({ 'HTTP': '115.210.31.236.55: 9000'}) 
handler request.ProxyHandler = ({ ' HTTP ' : ' 115.210.31.236.55: 9000 ' })
 # 2. handler created using a construct opener 
opener = request.build_opener (handler )
 # 3 to send a request using the opener 
RES = opener.open (URL)
 Print (res.read ())