python3 爬虫学习日志之爬取今日头条街拍图片

python3今日头条读取美女图片

以下是主程序的代码，gallery匹配的这个地方需要根据实际情况更改

import requests
from urllib.parse import urlencode
from requests.exceptions import RequestException
import json
import hashlib
import os
from bs4 import BeautifulSoup
import re
from config import *
import pymongo
from _md5 import md5
from multiprocessing import Pool
client=pymongo.MongoClient(MONGO_URL)
db=client[MONGO_DB]


headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'}
def get_page_index(offset,keyword):
    data={'offset':offset,
'format': 'json',
'keyword': keyword,
'autoload': 'true',
'count': '20',
'cur_tab': 3}
    url='https://www.toutiao.com/search_content/?'+urlencode(data)
    response=requests.get(url)
    try:
       if response.status_code==200:
           return response.text
       return None
    except RequestException:
        print('请求索引页出错')
        return None
def parse_page_index(html):
    data=json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            yield item.get('article_url')
def get_oage_detail(url):
    try:
       response=requests.get(url,headers=headers)
       if response.status_code==200:
           return response.text
       return None
    except RequestException:
        print('请求详情页出错',url)
        return None
def parse_page_detail(html,url):
    soup=BeautifulSoup(html,'lxml')
    title=soup.select('title')[0].get_text()
    print(title)
    images_pattern=re.compile('.*?gallery: JSON.parse(.*?)siblingList',re.S)
    result=re.search(images_pattern,html)
    if result:
        result_url=str(result.group(0))
        images = ''
        images = re.findall(r'url\\":\\"(.*?)\\"', result_url, re.S)
        images_url = [item for item in images]
        for image in images:download_image(image)
        return {
                'title': title,
                'url': url,
                'images': images_url
            }
    return None

def save_to_mongo(result):
    if db[MONGO_TABLE].insert(result):
        print('存储到mongodb成功',result)
        return True
    return False
def download_image(url):
    # pattern=re.compile("'(.*?)'",re.S)
    # url=re.findall(pattern,url)
    url = re.sub('\\\\', '', url)
    print('正在下载',url)
    try:
       response=requests.get(url,headers=headers)
       if response.status_code==200:
           save_image(response.content)
       return None
    except RequestException:
        print('请求图片出错',url)
        return None
def save_image(content):
    path= 'G://untitled4'
    file_path='{0}/{1}.{2}'.format(os.getcwd(),md5(content).hexdigest(),'jpg')
    if not os.path.exists(file_path):
        with open(file_path,'wb') as f:
            f.write(content)
            f.close()

def main(offset):
    html=get_page_index(offset,'街拍')
    for url in parse_page_index(html):
        html=get_oage_detail(url)
        if html:
            result=parse_page_detail(html,url)
            if result:save_to_mongo(result)

if __name__=='__main__':
    groups=[x*20 for x in range (GROUP_START,GROUP_END)]
    pool=Pool()
    pool.map(main,groups)

配置程序如下 config.py

MONGO_URL='localhost'
MONGO_DB='toutiao'
MONGO_TABLE='toutiao'
GROUP_START=1
GROUP_END=20