# -*- coding: utf-8 -*-
import json
import scrapy
from scrapy import Request
from testzh.items import TestzhItem
class ZhihuSpider(scrapy.Spider):
name = 'zhihu'
allowed_domains = ['zhihu.com']
start_urls = [
'https://www.zhihu.com/api/v4/members/excited-vczh/followers?include=data[*].answer_count,articles_count,gender,follower_count,is_followed,is_following,badge[?(type=best_answerer)].topics&offset=40&limit=20']
user_url = 'https://www.zhihu.com/api/v4/members/{user}?include=allow_message,is_followed,is_following,is_org,is_blocking,employments,answer_count,follower_count,articles_count,gender,badge[?(type=best_answerer)].topics'
def parse(self, response):
results = json.loads(response.text)
if 'data' in results.keys():
for i in results.get('data'):
yield Request(self.user_url.format(user=i.get('url_token')), self.parse_follow)
if 'paging' in results.keys() and results.get('paging').get("is_end") == False:
next_page = results.get('paging').get("next")
yield Request(next_page, self.parse)
def parse_follow(self, response):
item = TestzhItem()
results = json.loads(response.text)
for i in item.fields:
if i in results.keys():
item[i] = results.get(i)
yield item
import json
import scrapy
from scrapy import Request
from testzh.items import TestzhItem
class ZhihuSpider(scrapy.Spider):
name = 'zhihu'
allowed_domains = ['zhihu.com']
start_urls = [
'https://www.zhihu.com/api/v4/members/excited-vczh/followers?include=data[*].answer_count,articles_count,gender,follower_count,is_followed,is_following,badge[?(type=best_answerer)].topics&offset=40&limit=20']
user_url = 'https://www.zhihu.com/api/v4/members/{user}?include=allow_message,is_followed,is_following,is_org,is_blocking,employments,answer_count,follower_count,articles_count,gender,badge[?(type=best_answerer)].topics'
def parse(self, response):
results = json.loads(response.text)
if 'data' in results.keys():
for i in results.get('data'):
yield Request(self.user_url.format(user=i.get('url_token')), self.parse_follow)
if 'paging' in results.keys() and results.get('paging').get("is_end") == False:
next_page = results.get('paging').get("next")
yield Request(next_page, self.parse)
def parse_follow(self, response):
item = TestzhItem()
results = json.loads(response.text)
for i in item.fields:
if i in results.keys():
item[i] = results.get(i)
yield item