微信sougo中 风景关键字搜索 文章爬取

####config 文件的参数

#mongodb的参数
MONGO_URL = 'localhost'
MONGO_DB = 'weixin'
MONGO_TABLE = 'article_data'

POOL_PROXY_URL = 'http://127.0.0.1:5000/get'   # 初始化代理
KEYWORDS= '风景'  # 搜索的内容
base_url = 'https://weixin.sogou.com/weixin?'

MAX_COUNT = 5

from pyquery import PyQuery as pq
import requests
from urllib.parse import urlencode
import pymongo
from config import *

# 设置cookies 保证自己处于登陆状态
headers = {
    'Cookie': 'SMYUV=1543916558046770; IPLOC=CN3210; SUID=824A933D2E18960A000000005C064C19; usid=xOxPgW3N3IoHtGbK; SUV=00EC55C13D934A825C064C1975830150; ld=ZZllllllll2tx6ZPlllllVZpFj9lllll5B23kkllll9lllllxllll5@@@@@@@@@@; LSTMV=247%2C70; LCLKINT=3969; ABTEST=0|1543916582|v1; weixinIndexVisited=1; SNUID=C008D27C41443EE5E68EBAC74227C258; sct=3; JSESSIONID=aaapI9Xh-AseD_U-Nw_Cw; ppinf=5|1544421555|1545631155|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0OlRpbmF8Y3J0OjEwOjE1NDQ0MjE1NTV8cmVmbmljazo0OlRpbmF8dXNlcmlkOjQ0Om85dDJsdUJYcHpaSHVCXzRoMG5rQ2tQdS00bGdAd2VpeGluLnNvaHUuY29tfA; pprdig=NsDEAzGvJ1re-8nnesvi4j3YCyLTG05BToQzQjD1RX5uGMNh3s53s1UDapzy3GV_GmE6xp51ZvYjLa1QquTvsy8WOu3WeqMxzq6cHiQy3YsFnn31696RjyfBmDK_uyA1tbKZu61HU3HCR19g1Og3YUXuisHQqCXg0m2HXy6Iv_4; sgid=11-38266993-AVwOALPb4EmMTeL4sY1JzFg; ppmdig=1544421555000000a5d278737afc6497b87750cd0af731fa',
    'Host': 'weixin.sogou.com',
    'Upgrade-Insecure-Requests': '1',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
}
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
proxy = None


def get_url(KEYWORDS, page):   # 1、构造url，进行微信关键词搜索
    base_url = 'https://weixin.sogou.com/weixin?'
    data = {
        'query': KEYWORDS,
        'type': 2,
        'page': page
    }
    url = base_url + urlencode(data)
    return url


def get_index_html(url, count=1):   # 2、请求url，得到索引页html
    print('正在爬取', url)
    print('Trying Count', count)
    global proxy  # 使用global申明全局变量，声明后可在函数内改变proxy的值
    if count >= MAX_COUNT:
        print('Tried many times')
        return None
    try:
        if proxy:
            print('正在使用代理',proxy)
            proxies = {
                'http':'http://' + proxy
            }
            # allow_redirects=False 关闭重定向， 默认为True
            response = requests.get(url, headers=headers, allow_redirects=False, proxies=proxies)
        else:
            response = requests.get(url, headers=headers, allow_redirects=False)
        if response.status_code == 200:
            return response.text
        if response.status_code == 302:
            # Need proxy
            print('302')  # 302 状态表示请求网页临时移动到新的位置
            proxy = get_proxy()  # 添加代理
            if proxy:
                return get_index_html(url)
            else:
                print('请求代理失败')
                return None
    except ConnectionError as e:
        print('Error Occurred', e.args)
        proxy = get_proxy()
        count += 1
        return get_index_html(url)


def get_proxy():   # 3、请求url过程中，可能会遇到反爬虫措施，这时候需要开启代理
    print('正在请求代理')
    try:
        response = requests.get(POOL_PROXY_URL)
        if response.status_code == 200:
            return response.text
        else:
            print('请求代理失败')
            return None
    except ConnectionError:
        return None


# def delete_proxy(proxy):  # 删除频繁出错的代理
#     requests.get('http://127.0.0.1:5000/delete/?proxy={}'.format(proxy))


def get_article_url(html):    # 4、分析索引页html代码，返回微信详情页url
    doc = pq(html)
    url_lists = doc('.news-box .news-list li').items()
    for item in url_lists:
        # print(item.find('h3 a').attr('href'))
        yield item.find('h3 a').attr('href')


def get_article_html(article_url):   # 5、请求微信详情页url，得到详情页html
    try:
        response = requests.get(article_url)
        if response.status_code == 200:
            return response.text
        return None
    except ConnectionError:
        return None


def parse_article_html(article_html):   # 6、分析详情页html代码，得到微信标题、公众号、发布日期，文章内容等信息
    doc = pq(article_html)
    title = doc('#img-content .rich_media_title').text()
    content = doc('.rich_media_area_primary_inner').text()
    date = doc('.rich_media_meta.rich_media_meta_text').text()
    nikname = doc('.rich_media_meta_list .rich_media_meta_nickname').text()
    artile_data = {
        'title': title,
        'content': content,
        'date': date,
        'nikname': nikname
    }
    return artile_data


def save_to_mongo(article_data):    # 保存到数据库MongoDB
    # MongoDB中update()使用'$set'指定一个键的值，如果不存在该值就创建（去重）
    # multi：默认是false，只更新找到的第一条记录。如果为true，把按条件查询出来的记录全部更新。
    if db[MONGO_TABLE].update({'title':article_data['title']},{'$set': article_data}, True):
        print('Saved to MongoDB', article_data['title'])
    else:
        print('Saved to Mongo Failed', article_data['title'])
    # try:
    #     if db[MONGO_TABLE].insert(article_data):
    #         print('保存到MONGODB成功',article_data)
    # except Exception:
    #     print('保存到MONGODB失败！',article_data)


def main():    # 调试模块
    for page_num in range(1, 4):  # 由于我维护的代理池质量不好，所以这里只取前3页的文章内容
        index_url = get_url(KEYWORDS, page_num)  # 获取微信搜索页面的url
        # print(index_url)
        index_html = get_index_html(index_url)  # 获取搜索页面url的text
        if index_html:
            article_urls = get_article_url(index_html)  # 获取上述html中包含的文章的URL，返回的是一个迭代器
            for article_url in article_urls:  # 遍历上述的url
                # print(article_url)
                article_html = get_article_html(article_url)  # 得到url的text
                if article_html:
                    article_data = parse_article_html(article_html)   # 解析url的text并获取我们想要的内容(content, title, nikname, date...)
                    print(article_data)  # 把上述的data打印出来
                    save_to_mongo(article_data) # 把上述的data存储到mongodb 数据库中


if __name__ == '__main__':
    main()