#!/usr/bin/env python
# -*- coding=utf-8 -*-
# coding: utf-8
from pyquery import PyQuery
import requests, json, csv
import urllib3 # 为了忽略InsecureRequestWarning
urllib3.disable_warnings()
url = 'https://www.zhihu.com/explore'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/55.0.2883.87 UBrowser/6.2.4098.3 '
'Safari/537.36'
}
html = requests.get(url, headers=headers, verify=False).text
doc = PyQuery(html)
items = doc('.explore-feed').items() # 返回generator object PyQuery.items
def wr_txt(data_txt):
with open('explore.txt', 'a', encoding='utf-8') as fp:
fp.write('\n'.join(data_txt))
def wr_json(data_json):
with open('explore.json', 'a', encoding='utf-8') as fp:
json.dump(obj=data_json, fp=fp)
def wr_csv(data_csv):
with open('explore.csv', 'a', encoding='utf-8') as fp:
writer = csv.writer(fp)
writer.writerow(data_csv)
for item in items:
question = item.find('h2').text()
author = item.find('.author-link').text()
answer = PyQuery(item.find('.content').html()).text()
data = [question, author, answer]
# 写入到TXT文件
wr_txt(data_txt=data)
# 写入到JSON文件
wr_json(data_json=data)
# 写入到csv文件
wr_csv(data_csv=data)
Python3:小爬虫成长记(三)---爬取数据并写入到文件
猜你喜欢
转载自blog.csdn.net/yijinaqingan/article/details/90647710
今日推荐
周排行