爬虫_百度图片下载

帮别的院的同学批量下载点图片，并进行简单筛选

 1 import requests
 2 import re
 3 import os
 4 from lxml import etree
 5 import json
 6 
 7 
 8 
 9 def get_html(url, param):
10     headers = {
11         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36'
12     }
13     response = requests.get(url, params=param, headers=headers)
14     response.encoding = response.apparent_encoding
15     # 返回json数据,str格式
16     json_str = response.text
17     response.encoding = 'utf-8'
18     return json_str
19 
20 
21 def parse_page(html):
22     data = json.loads(html, strict=False)
23     objs = data['data'][:-1]
24     urls = []
25     for obj in objs:
26         url = obj['middleURL']
27         key = obj['fromPageTitleEnc']
28         # if '纹枯病' in key:
29         if '全蚀病' in key:
30             urls.append(url)
31         # if '叶锈病' in key:
32         #     urls.append(url)
33         # elif '条锈病' in key:
34         #     urls.append(url)
35         else:
36             print('该标题被筛选掉: '+key)
37     print(len(urls))
38     a = len(urls)
39     return urls, a
40     # return url_list
41 
42 
43 def run(keyword, path):
44     url = "https://image.baidu.com/search/acjson"
45     # https://image.baidu.com/search/acjson?ipn=rj&tn=resultjson_com&word=小麦纹枯病矢量图大图&pn=30
46     i = 0
47     sum_pic = 0
48     for j in range(30, 1800, 30):
49         params = {
50                     "ipn": "rj", 
51                     "tn": "resultjson_com", 
52                     "word": keyword, 
53                     "pn": str(j)
54                 }
55         html = get_html(url, params)
56         lists, num_pic = parse_page(html)
57         sum_pic += num_pic
58   
59         for item in lists:
60             try:
61                 img_data = requests.get(item, timeout=10).content
62                 with open(path + "/" + str(i) + ".jpg", "wb") as f:
63                     f.write(img_data)
64                     f.close()
65                 i = i+1
66             except requests.exceptions.ConnectionError:
67                 print('can not download')
68                 continue
69 
70 def make_dir(keyword):
71     path = "images/"
72     path = path+keyword
73     is_exists = os.path.exists(path)
74     if not is_exists:
75         os.makedirs(path)
76         return path
77     else:
78         print(path + '目录已存在')
79         return path
80 
81 
82 def main():
83     # keyword = '小麦纹枯病矢量图大图'
84     keyword = '小麦全蚀病'
85     path = make_dir(keyword)
86     run(keyword, path)
87 
88 
89 
90 if __name__ == '__main__':
91     main()