Python crawling Douban movie review

import urllib.request
import requests
from lxml import etree
from pyquery import PyQuery as pq
import json

headers = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36' }
 for i in ( 0 , 20 , 40 , 60 , 80 ):
     #Note that the integer i must be converted to a string
     url = "https://movie.douban.com/review/best/?start=" + str (i)
    html = urllib.request.urlopen(url).read().decode('utf-8')
    # print(html)
aaa = []
    
    s = etree.HTML(html)

    for i in range ( 1 , 11 ):
         #To use i as the id to pass in the XPath statement, pay attention to the writing of the string
         id = s.xpath( '//*[@id="content"]/div/div[1 ]/div[1]/div[' + str (i)+ ']/@data-cid' )[ 0 ]
        name = s.xpath("//*[@id="+id+"]/header/a[2]//text()")[0]
        date = s.xpath("//*[@id="+id+"]/header/span[2]/text()")[0]
        movie_name = s.xpath("//*[@id="+id+"]/div[1]/h2/a/text()")[0]
        reivew_url = "https://movie.douban.com/j/review/"+id+"/full"
        response = requests.get(reivew_url,headers=headers)
        if response.status_code == 200:
            re_json = response.json()
            if re_json:
                html = re_json.get('html')
                print("id:%s,name:%s,movie_name:%s"%(id,name,movie_name))
                reivew = pq(html).text()

        info = {}
        info['id'] = id
        info['name'] = name
        info['date'] = date
        info['review'] = reivew
        aaa.append(info)

#Save in json format, pay attention to encoding="utf-8", otherwise there will be an error in encoding
 with open ( 'F:/Python/douban_review.json' , 'w' , encoding = "utf-8" ) as f:
    json.dump(aaa, f, ensure_ascii=False)


Guess you like

Origin http://43.154.161.224:23101/article/api/json?id=324524679&siteId=291194637