1 # -*- coding: utf-8 -*-
2 # @author: Tele
3 # @Time : 2019/04/14 下午 3:48
4 # 多线程版
5 import time
6 import requests
7 import os
8 import json
9 from fake_useragent import UserAgent
10 from lxml import etree
11 import threading
12 from concurrent.futures import ThreadPoolExecutor, wait, as_completed
13
14
15 class JDSplier:
16 executor = ThreadPoolExecutor(max_workers=6)
17 mutex = threading.Lock()
18 flag = True
19
20 @staticmethod
21 def get_proxy():
22 return requests.get("http://127.0.0.1:5010/get/").content.decode()
23
24 @staticmethod
25 def get_ua():
26 ua = UserAgent()
27 return ua.random
28
29 def __init__(self, kw_list):
30 self.kw_list = kw_list
31 # 评论url
32 self.url_temp = "https://sclub.jd.com/comment/productPageComments.action?&productId={}&score=0&sortType=5&page={}&pageSize=10&isShadowSku=0&rid=0&fold=1"
33 self.headers = {
34 "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36",
35
36 }
37 self.proxies = {
38 "http": None
39 }
40 self.parent_dir = None
41 self.file_dir = None
42
43 # ua,proxy
44 def check(self):
45 self.headers["User-Agent"] = JDSplier.get_ua()
46 proxy = "http://" + JDSplier.get_proxy()
47 self.proxies["http"] = proxy
48 print("ua:", self.headers["User-Agent"])
49 print("proxy:", self.proxies["http"])
50
51 # 评论
52 def parse_url(self, product_id, page):
53 url = self.url_temp.format(product_id, page)
54 response = requests.get(url, headers=self.headers, proxies=self.proxies, verify=False)
55 if response.status_code == 200:
56 print(url)
57 data = None
58 if len(response.content) < 0:
59 return
60 # 很奇葩
61 try:
62 data = json.loads(response.content.decode("gbk"))
63 except:
64 data = json.loads(response.content.decode())
65 finally:
66 # 评论
67 if not data:
68 return
69 comment_list = data["comments"]
70 if len(comment_list) > 0:
71 item_list = list()
72 for comment in comment_list:
73 item = dict()
74 # 商品名
75 item["referenceName"] = comment["referenceName"]
76 # 评论时间
77 item["creationTime"] = comment["creationTime"]
78 # 内容
79 item["content"] = comment["content"]
80 item_list.append(item)
81
82 # 保存
83 with open(self.file_dir, "a", encoding="utf-8") as file:
84 file.write(json.dumps(item_list, ensure_ascii=False, indent=2))
85 file.write("\n")
86 time.sleep(5)
87 else:
88 JDSplier.flag = False
89 else:
90 print("请求失败!")
91
92 # 提取id
93 def get_product_info(self):
94 url_temp = "https://search.jd.com/Search?keyword={}&enc=utf-8"
95 result_list = list()
96 for kw in self.kw_list:
97 url = url_temp.format(kw)
98 response = requests.get(url, headers=self.headers, proxies=self.proxies, verify=False)
99 if response.status_code == 200:
100 item_dict = dict()
101 id_list = list()
102 html_element = etree.HTML(response.content)
103 # 获得该关键词下第一页的商品id,前10个
104 id_list = html_element.xpath("//div[@id='J_goodsList']/ul/li[position()<11]/@data-sku")
105 item_dict["title"] = kw
106 item_dict["id_list"] = id_list
107 result_list.append(item_dict)
108 else:
109 pass
110 return result_list
111
112 def get_comment(self, item_list):
113 if len(item_list) > 0:
114 for item in item_list:
115 id_list = item["id_list"]
116 item_title = item["title"]
117 if len(id_list) > 0:
118 # 检查目录
119 self.parent_dir = "f:/jd_comment/" + item_title + time.strftime("-%Y-%m-%d-%H-%M-%S",
120 time.localtime(time.time()))
121 if not os.path.exists(self.parent_dir):
122 os.makedirs(self.parent_dir)
123 task_list = list()
124 # 每个商品开启一个线程爬取评论
125 for product_id in id_list:
126 t = JDSplier.executor.submit(self.job, product_id)
127 time.sleep(10)
128 task_list.append(t)
129 for re in as_completed(task_list):
130 re.result(timeout=500)
131 # wait(task_list, timeout=500)
132 else:
133 print("---error,empty id list---")
134 else:
135 print("---error,empty item list---")
136
137 def job(self, product_id):
138 self.check()
139 JDSplier.mutex.acquire()
140 page = 0
141 self.file_dir = self.parent_dir + "/" + str(product_id) + "_ratecontent.txt"
142 # 爬取评论
143 while JDSplier.flag:
144 self.parse_url(product_id, page)
145 page += 1
146 JDSplier.flag = True
147 JDSplier.mutex.release()
148
149 def run(self):
150 # self.check()
151 item_list = self.get_product_info()
152 print(item_list)
153 self.get_comment(item_list)
154 JDSplier.executor.shutdown()
155
156
157 def main():
158 # "华为p30pro", "华为mate20pro",
159 # "vivoz3""oppok1","荣耀8x", "小米9", "小米mix3", "三星s9", "iphonexr", "iphonexs"
160 kw_list = ["vivoz3"]
161 splider = JDSplier(kw_list)
162 splider.run()
163
164
165 if __name__ == '__main__':
166 main()
ps:能多睡就多睡会,虽然很慢,但不会触发jd的安全系统,爬这种电商平台还是他们的活动日时比较好爬,那个时候为了应对超高的访问量,一般会暂时关闭反爬机制