A crawler demo, requests, beatuifulsoup use

Crawler demo, requests, beatuifulsoup

import os,re
import pickle
import requests
import chardet

import random
import time
from bs4 import BeautifulSoup
from multiprocessing import Pool

user_agent_list = [
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
            "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
            "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
            "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
            "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
            "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
            "The Mozilla / 5.0 (the Windows NT 6.2; the WOW64) AppleWebKit / 535.24 (KHTML, like the Gecko) the Chrome / 19.0.1055.1 Safari / 535.24" 
        ] 
the UA = The random.choice (user_agent_list) ## a random string withdrawn from the self.user_agent_list 
headers = { 'User-Agent' : UA} ## is configured into a complete User-Agent (UA behalf random string is taken out of the above-oh) 


URL0 = 'HTTP: //www.tjxqda.com/column/ xxdt ' 
with requests.get (URL0, headers = headers, timeout =. 5) AS Response: 
    # Print (type (response.text), response.text) 
    # Print (type (response.content), response.content) 
    # Print (of the type (response.status_code), response.status_code) 
    # Print (of the type (Response.Headers), Response.Headers) 
    # Print (of the type (Response.Cookies), Response.Cookies) 
    # Print (of the type (response.url), response.url)
    # print(type(response.history), response.history)

    # response.encoding = 'gb18030'
    soup = BeautifulSoup(response.text, 'lxml')

    print(soup.title.text)  #打印title

    li_list=soup.find('ul', class_='list-main-style').find_all('li')
    # print(li_list)
    for li_quick in li_list:
        print('最灵活的查找方法:', li_quick.get_text())
        print(li_quick.a.get('href'))



    # print(soup.prettify())
    # print(soup.select('ul[class="list_16 mt10"]'))[0].text
    # for x in soup.select('li a'):
    #     print(x.text)