Basic usage of Python Urllib library

import urllib.request #request module
import urllib.parse #url parsing module
import urllib.error #Exception handling module
import socket #To the necessary functional modules for network applications

response = urllib.request.urlopen('http://www.baidu.com')
print(response.read().decode('utf-8')) #response.read() can get the content of the web page


data = bytes(urllib.parse.urlencode({'word': 'hello'}), encoding='utf8')#urllib.parse, the post data can be converted and put through bytes(urllib.parse.urlencode()) into the data parameter of urllib.request.urlopen. This completes a post request.
print(data)
response = urllib.request.urlopen('http://httpbin.org/post', data=data) #If we add the data parameter, it is a post request, and if there is no data parameter, it is a get request
print(response.read())

try:
   response = urllib.request.urlopen('http://httpbin.org/get', timeout=0.1)# timeout timeout time setting
except urllib.error.URLError as e:
   if isinstance(e.reason, socket.timeout):
       print('TIME OUT')

response = urllib.request.urlopen('https://www.python.org')
print(type(response))
# print(response.status) #Get the status code [syntax error encountered]
print (response.getheaders())#Get response header information
print (response.getheader("server"))#Find header information

request = urllib.request.Request('https://python.org') #request() wrapping the request
response = urllib.request.urlopen(request)
print(response.read().decode('utf-8'))


from urllib import request, parse

url = 'http://httpbin.org/post'
#The following sets the header information
headers = {
    'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)',
    'Host': 'httpbin.org'
}
dict = {
    'name': 'alex'
}
data = bytes(parse.urlencode(dict), encoding='utf8')
req = request.Request(url=url, data=data, headers=headers, method='POST')
response = request.urlopen(req)
print(response.read().decode('utf-8'))



#Get cookies and store cookies
import http.cookiejar, urllib.request
cookie = http.cookiejar.CookieJar()#declare a CookieJar object instance to save cookies
handler = urllib.request.HTTPCookieProcessor(cookie)#Use the HTTPCookieProcessor object of the urllib library to create a cookie processor
opener = urllib.request.build_opener(handler)#通过handler构建opener
response = opener.open('http://www.baidu.com')
for item in cookie:
    print(item.name+"="+item.value)

#Get the cookie and save it to a file
import http.cookiejar, urllib.request
filename = "cookie.txt" #Save the cookie file
cookie = http.cookiejar.MozillaCookieJar(filename)#Declare a MozillaCookieJar object instance (cookie) to save the cookie and write to the file later
handler = urllib.request.HTTPCookieProcessor(cookie)#Or create a processor
opener = urllib.request.build_opener(handler) #Create an opener object that supports processing HTTP requests
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)#Save cookie to file
#ignore_discard means that the cookie will be saved even if it will be discarded, ignore_expires means that if the cookie already exists in the file, the original file will be overwritten and written

#Get the cookie from the file and access it
import http.cookiejar, urllib.request
cookie = http.cookiejar.LWPCookieJar()#declare a CookieJar object instance to save cookies
cookie.load('cookie.txt', ignore_discard=True, ignore_expires=True)#Read content from file into cookie variable
handler = urllib.request.HTTPCookieProcessor(cookie)#processor
opener = urllib.request.build_opener(handler) #Create an opener object that supports processing HTTP requests
response = opener.open('http://www.baidu.com')
print(response.read().decode('utf-8'))