爬虫--lxml爬取妹子图

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/MR_HJY/article/details/81878849

import requests
from lxml import etree
import os

# 下载图片
def  download_img(img_url,referer):
    # print(img_url)
    headers = {
        'referer': referer,
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',
    }
    # print(headers)
    if os.path.exists('download'):
        pass
    else:
        os.mkdir('download')
    filename = 'download/'+img_url.split('/')[-1]
    response = requests.get(img_url,headers=headers)
    with open(filename,'wb') as f:
        f.write(response.content)

# 图片的获取
def pares_detailed_page(url_href):
    response = requests.get(url_href)
    html_element = etree.HTML(response.text)
    max_page = html_element.xpath('//div[@class="pagenavi"]/a/span/text()')[-2]
    # print(max_page)
    for i in range(1,int(max_page)+1):
        page_url = url_href + '/' + str(i)
        response = requests.get(page_url)
        html_element = etree.HTML(response.text)
        img_url = html_element.xpath('//div[@class="main-image"]/p/a/img/@src')[0]
        download_img(img_url,url_href)


url = 'http://www.mzitu.com/'
response = requests.get(url)
# with open('meizi.html','wb') as f:
#     f.write(response.content)

html_element = etree.HTML(response.text)
href_list = html_element.xpath('//ul[@id="pins"]/li/a/@href')
for href in href_list:
    # print(href)
    pares_detailed_page(href)