app之---豆果美食

1.抓包

　　

2.代码

　　抓取：

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#author tom
import requests
from multiprocessing import Queue
from handle_pymongo import mongo
from concurrent.futures import ThreadPoolExecutor

class Douguo():
    def __init__(self):
        self.queue_list=Queue()
    
    #因为所有的请求的请求头都一样，所以放在这边处理，其实请求头也可以放在__init__
    def handle_request(self,url,data):
        headers={
            "client":"4",
            "version":"6920.4",
            "device":"SM-G9350",
            "sdk":"22,5.1.1",
            "imei":"861373280750547",
            "channel":"qqkp",
            # "mac":"2c:c3:82:e2:0b:03",
            "resolution":"1024*576",
            "dpi":"1.19375",
            # "android-id":"4014041355524873",
            # "pseudo-id":"28075263",
            "brand"    :"samsung",
            "scale"    :"1.19375",
            "timezone":"28800",
            "language":"zh",
            "cns":"3",
            # "imsi":"460005263415341",
            "user-agent":"Mozilla/5.0 (Linux; Android 5.1.1; SM-G9350 Build/LMY48Z) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36",
            "reach"    :"1",
            "newbie":"1",
            "Content-Type":"application/x-www-form-urlencoded; charset=utf-8",
            "Accept-Encoding":"gzip, deflate",
            # "Cookie":"duid=59758072", #能不带最好不带
            "Host":"api.douguo.net",
            # "Content":"-Length    68",
            "Connection":"keep-alive"
            }
        response = requests.post(url=url, headers=headers,data=data)
        return response

    #请求食谱首页
    def handle_index(self):
        url='http://api.douguo.net/recipe/flatcatalogs'
        data={
            "client":"4",
            # "_session":"1557318413116861373280750547",
            # "v":"1503650468",
            "_vs": "2305"
        }
        response_index=self.handle_request(url,data)
        indext_response_dict=response_index.json()
        #遍历这个三级分类
        for index_item in indext_response_dict['result']['cs']:
            for items in index_item['cs']:
                for item in items['cs']:
                    #每一个小分类
                    data2={
                        "client": "4",
                        # "_session": "1557318413116861373280750547",
                        "keyword":item['name'],
                        "order": "3",
                        "_vs": "400"
                    }
                    self.queue_list.put(data2)

    #请求具体食材的做法
    def handle_caipu_list(self,data):
        print("当前处理的食材是:",data['keyword'])
        #翻页
        for i in range(1,11):
            caipu_list_url='http://api.douguo.net/recipe/v2/search/{0}/20'.format(str(i*20))
            caipu_list_response=self.handle_request(url=caipu_list_url,data=data)
            caipu_list_dict=caipu_list_response.json()
            #判断是否有数据
            if caipu_list_dict['result']['end']==1:
                break
            for item in caipu_list_dict['result']['list']:
                caipu_info={}
                caipu_info['shicai'] = data['keyword']
                if item['type']==13:
                    caipu_info['user_name']=item['r']['an']
                    caipu_info['shicai_id']=item['r']['id']
                    caipu_info['describe']=item['r']['cookstory'].replace('/n','').replace(' ','')
                    caipu_info['caipu_name']=item['r']['n']
                    caipu_info['zuoliao_list']=item['r']['major']
                    # print(caipu_info)
                    #获取详情页的内容
                    detail_url='http://api.douguo.net/recipe/detail/'+str(caipu_info['shicai_id'])
                    detail_data={
                        "client": "4",
                        # "_session": "1557318413116861373280750547",
                        "author_id": "0",
                        "_vs": "2803",
                        "_ext":'{"query": {"kw": '+caipu_info['shicai']+', "src": "2803", "idx": "1", "type": "13", "id": '+str(caipu_info['shicai_id'])+'}}'
                    }
                    detail_response=self.handle_request(url=detail_url,data=detail_data)
                    detail_response_dict=detail_response.json()
                    caipu_info['tips']=detail_response_dict['result']['recipe']['tips']
                    caipu_info['cook_step']=detail_response_dict['result']['recipe']['cookstep']
                    print('当前入库的是：',caipu_info['caipu_name'])
                    mongo.inset_item(caipu_info)
                #g过滤掉广告
                else:
                    continue



if __name__ == '__main__':
    d=Douguo()
    d.handle_index()
    pool=ThreadPoolExecutor(max_workers=20)
    while d.queue_list.qsize()>0:
        pool.submit(d.handle_caipu_list,d.queue_list.get())

　　存储：

#!/usr/bin/env python
# -*- coding: utf-8 -*-
#author tom
import  pymongo


class Connect_Mongo(object):
    def __init__(self):
        self.client=pymongo.MongoClient(host='127.0.0.1',port=27017)
        self.db=self.client['doukou_meishi']

    def inset_item(self,item):
        self.collection=self.db['meishi']
        self.collection.insert(item)


mongo=Connect_Mongo()