Python爬取豆瓣影评

import urllib.request
import requests
from lxml import etree
from pyquery import PyQuery as pq
import json

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36'}
for i in (0,20,40,60,80):
    # 注意要把整数i强转为字符串
    url = "https://movie.douban.com/review/best/?start="+str(i)
    html = urllib.request.urlopen(url).read().decode('utf-8')
    # print(html)

    aaa = []
    s = etree.HTML(html)

    for i in range(1,11):
        # 要使用i作为id传入XPath语句，注意字符串的书写
        id = s.xpath('//*[@id="content"]/div/div[1]/div[1]/div['+str(i)+']/@data-cid')[0]
        name = s.xpath("//*[@id="+id+"]/header/a[2]//text()")[0]
        date = s.xpath("//*[@id="+id+"]/header/span[2]/text()")[0]
        movie_name = s.xpath("//*[@id="+id+"]/div[1]/h2/a/text()")[0]
        reivew_url = "https://movie.douban.com/j/review/"+id+"/full"
        response = requests.get(reivew_url,headers=headers)
        if response.status_code == 200:
            re_json = response.json()
            if re_json:
                html = re_json.get('html')
                print("id:%s,name:%s,movie_name:%s"%(id,name,movie_name))
                reivew = pq(html).text()

        info = {}
        info['id'] = id
        info['name'] = name
        info['date'] = date
        info['review'] = reivew
        aaa.append(info)

# 保存为json格式，注意encoding="utf-8",否则编码会出现错误
with open('F:/Python/douban_review.json','w',encoding="utf-8") as f:
    json.dump(aaa, f, ensure_ascii=False)