import urllib.request import requests from lxml import etree from pyquery import PyQuery as pq import json headers = { 'User-Agent' : 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36' } for i in ( 0 , 20 , 40 , 60 , 80 ): #Note that the integer i must be converted to a string url = "https://movie.douban.com/review/best/?start=" + str (i) html = urllib.request.urlopen(url).read().decode('utf-8') # print(html) aaa = [] s = etree.HTML(html) for i in range ( 1 , 11 ): #To use i as the id to pass in the XPath statement, pay attention to the writing of the string id = s.xpath( '//*[@id="content"]/div/div[1 ]/div[1]/div[' + str (i)+ ']/@data-cid' )[ 0 ] name = s.xpath("//*[@id="+id+"]/header/a[2]//text()")[0] date = s.xpath("//*[@id="+id+"]/header/span[2]/text()")[0] movie_name = s.xpath("//*[@id="+id+"]/div[1]/h2/a/text()")[0] reivew_url = "https://movie.douban.com/j/review/"+id+"/full" response = requests.get(reivew_url,headers=headers) if response.status_code == 200: re_json = response.json() if re_json: html = re_json.get('html') print("id:%s,name:%s,movie_name:%s"%(id,name,movie_name)) reivew = pq(html).text() info = {} info['id'] = id info['name'] = name info['date'] = date info['review'] = reivew aaa.append(info) #Save in json format, pay attention to encoding="utf-8", otherwise there will be an error in encoding with open ( 'F:/Python/douban_review.json' , 'w' , encoding = "utf-8" ) as f: json.dump(aaa, f, ensure_ascii=False)