python用requests爬取新浪财经首页要闻

目的：爬取新浪财经首页要闻模块下的全部新闻标题及内容

工具：python， 第三方库requests模块， lxml模块

requests，lxml需要安装（pip安装即可，或者去官网下载压缩包）

代码：

import requests
from lxml import etree


def search_article(url):
    """
    请求所传的url
    args：
        url: 所要请求的url
    return:
        类lxml.etree._Element的元素, 可以直接用xpath解析
    """

    header = {
        'Accept': '*/*',
        'User - Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                        'AppleWebKit/537.36 (KHTML, like Gecko) '
                        'Chrome/66.0.3359.181 Safari/537.36'
    }
    resp = requests.get(url, headers=header)
    html_str = str(resp.content, 'utf-8')
    selector = etree.HTML(html_str)
    return selector


def parse_html(selector):
    """
    解析提取的lxml.etree._Element的元素
    return:
        type:dict
            key:news title
            value: contents of news
    """
    try:
        title_text = selector.xpath('//h1/text()')[0]
        # 获取class=article的div下面的p标签的所有text()
        article_text = selector.xpath('//div[@class="article"]/p//text()')

        return {title_text: article_text}
    except Exception as e:
        return {'解析错误': [e]}


def write_article(article):
    """
    将所传的新闻字典写入文件news.txt中
    args：
        article：dict {news_title:[contents,]}
    No return
    """
    file_name = 'news.txt'
    f = open(file_name, 'a', encoding='utf-8')
    title = list(article.keys())[0]
    f.write("题目："+title + '\n')
    for content in article[title]:
        f.write(content+"\n")
    f.write("\n\n")
    f.close()


def extract_url(url):
    href_lists = search_article(url).xpath("//div[@id='fin_tabs0_c0']//a//@href")
    return href_lists


if __name__ == '__main__':
    url = "http://finance.sina.com.cn/"
    href_list = extract_url(url=url)
    for href in href_list:
        # 排除非新浪连接
        if href.startswith("http://finance.sina.com.cn/"):
            try:
                html = search_article(href)
                article = parse_html(html)
                write_article(article)
            except Exception as e:
                print(e)