小项目 爬虫爬取图片 --站长素材网

要点： xpath
懒加载问题

t urllib.request
import urllib.parse
import time
from lxml import etree
import os

def handle_request(url,page):
# 第一页页码url单独判断
if page==1:
url = url.format(’’)
else:
url = url.format(’_’+str(page))
# print(url)
headers = {
‘User-Agent’ : ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.3.4000’
}
request = urllib.request.Request(url=url,headers=headers)
return request
def download_img(image_src):
dirpath = ‘xinggan’
if not os.path.exists(dirpath):
os.mkdir(dirpath)
# 文件名
filename = os.path.basename(image_src)
# 搞图片路径
filepath = os.path.join(dirpath,filename)
# 发送请求保存图片
headers = {
‘User-Agent’ : ‘Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.79 Safari/537.36 Maxthon/5.2.3.4000’
}
request = urllib.request.Request(url=image_src,headers=headers)
response = urllib.request.urlopen(request)
with open(filepath,‘wb’) as fp:
fp.write(response.read())

def parse_content(content):
# 解析内容 获取图片
tree = etree.HTML(content)
image_list = tree.xpath(’//div[@id=“container”]/div/div/a/img/@src2’)
# print(image_list)
# print(len(image_list))
for image_src in image_list:
download_img(image_src)
def main():
url = ‘http://sc.chinaz.com/tupian/xingganmeinvtupian{}.html’
# url = http://sc.chinaz.com/tupian/xingganmeinvtupian_3.html
start_page = int(input(‘请输入起始页码：’))
end_page = int(input(‘请输入结束页码：’))
for page in range(start_page,end_page+1):
request = handle_request(url,page)
content = urllib.request.urlopen(request).read().decode()
parse_content(content)
time.sleep(2)

if name == ‘main’:
main()