# -*- coding: utf-8 -*-
__author__ = '木之易'
__date__ = '2018/8/7 20:17'
import os
import re
from urllib import request
class WeddingImageSpider(object):
def __init__(self, t_id):
# 拼接完整地址
self.url = 'http://www.ivsky.com'+t_id
self.title = 'images'
self.html = ''
self.headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:61.0) Gecko/20100101 Firefox/61.0'}
self.create_directry()
self.count = 0
def create_directry(self):
"""
创建存放图片的文件夹
:return:
"""
# 获取源代码
self.get_html(self.url)
# 2.获取帖子标题
pattern = re.compile(r'<title>(.*?)</title>', re.S)
# print(pattern)
rs = re.search(pattern, self.html)
# 3.提取帖子标题,并赋值给self.title
if rs:
self.title = rs.group(1)
# 4.判断文件/目录是否存在
if not os.path.exists(self.title):
# 没有self.title这个文件夹,创建这个文件夹
# mkdir() 创建文件夹
os.mkdir(self.title)
def get_total(self):
"""获取下一页"""
self.get_html(self.url)
pattern = re.compile(r'<div class="pagelist">(.*?)</div>', re.S)
res = re.search(pattern, self.html)
# 若找到
if res:
ul_html = res.group()
# 找到含图片网址和关键字的标签
# links = re.findall(re.compile(r"<a class='page-next' href='(.*?)</a>", re.S), ul_html)
links = re.findall(re.compile(r"<a.*?href='(.*?)'", re.S), ul_html)
print(links)
print(111111111111111)
if 'index' in links[-1]:
# 拼接下一页完整地址
# self.url = 'http://www.ivsky.com' + links[0][0:len(links)-6]
self.url = 'http://www.ivsky.com' + links[-1]
return True
else:
return False
else:
return False
def get_html(self, url):
# 构建请求对象
req = request.Request(url=url, headers=self.headers)
# 发送请求
response = request.urlopen(req)
# 读取相应数据
self.html = response.read().decode('utf-8', 'ignore')
def parser_html(self):
# 下载地址 href = "http://img.ivsky.com/img/tupian/pic/201802/20/shatan_hunli_zhuangshi.jpg?download"
# 图片地址 src = "http://img.ivsky.com/img/tupian/t/201802/20/shatan_hunli_zhuangshi.jpg"
# 1.准备正则
pattern = re.compile(r'<div class="il_img.*?<img src="(.*?)"', re.S)
# 2.从源代码中提取数据
result = re.findall(pattern, self.html)
# 3.遍历下载每一张图片
for link in result:
self.count += 1
print('正在下载第%s张图片' % self.count)
# 4.拼接图片完整的存放路径
# 自然风光图片 - 自然风景图片 (天堂图片网)/meilidehaitan-004.jpg
path = self.title + '/' + '%s.jpg' % self.count
# 5.下载图片
request.urlretrieve(link, path)
def run(self):
count = 0
while True:
self.get_html(self.url)
count += 1
print('.........正在爬取第%s页数据,请稍后........' % count)
self.parser_html()
if not self.get_total():
print('图片爬取完毕')
break
if __name__ == '__main__':
wedding = WeddingImageSpider('/tupian/meinv_t50/')
wedding.run()
正则爬取天堂图片网并存储在自动生成文件夹中
猜你喜欢
转载自blog.csdn.net/A_fool_Program_ape/article/details/81569684
今日推荐
周排行