python_爬取赢商网_商业地产周边配套设施数据

import os
import json
import ssl
import pandas as pd
import requests
import urllib3
from utils.read_write import writeOneJSON, readJson
os.chdir(r'D:\project\jianguiyuan\data')
ssl._create_default_https_context = ssl._create_unverified_context
header = {
    'Host': 'ytz.winshangdata.com',
    'appType': 'ytzapp',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'platform': 'android',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,en-US;q=0.9',
    'X-Requested-With': 'com.winshangdata.ytz',
    'uuid': '860000000229756,860000000229756',
    'token': '请自行注册后获取',
    'Cookie': '请自行注册后获取'
    'User-Agent': 'Mozilla/5.0 (Linux; Android 7.1.1; OPPO A73t Build/N6F26Q; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 uni-app Html5Plus/1.0 (Immersed/18.0)'
}


def getPeripheralMatchingTeTaiListCtr():
    try:
        urllib3.disable_warnings()
        response = requests.post(url1, headers=header, verify=False)
    except Exception as e:
        print('getPeripheralMatchingTeTaiListCtr')
        print(e)

def get3YeTaiCountListCtr(id, path,yeTai1,yeTai2):
    try:
        formdata = {
            "projectId": id,
            "radius": '3',
            "yeTai1": yeTai1,
            "yeTai2": yeTai2,
        }
        urllib3.disable_warnings()
        response = requests.post(url2, json=formdata, headers=header, verify=False)
        get3YeTaiCountListCtr = json.loads(response.text)
        if 'data' in get3YeTaiCountListCtr.keys():
            if get3YeTaiCountListCtr['data']:
                writeOneJSON(get3YeTaiCountListCtr, path + '\\' + str(id) + '_'+ str(yeTai1)+ '_'+ str(yeTai2) +'_radius3.json')
                print(get3YeTaiCountListCtr)
                return get3YeTaiCountListCtr
    except Exception as e:
        print('get3YeTaiCountListCtr')
        print(id)
        print(path)
        print(e)


def getZhouBianPeiTaoListCtr(id, path,yeTai1,yeTai2,yeTai3):
    try:
        formdata = {
            "projectId": id,
            "radius": '3',
            "yeTai1": yeTai1,
            "yeTai2": yeTai2,
            "yeTai3": yeTai3,
        }
        urllib3.disable_warnings()
        response = requests.post(url3, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        if 'data' in data.keys():
            if data['data']:
                writeOneJSON(data, path + '\\' + str(id) + '_'+ str(yeTai1)+ '_'+ str(yeTai2) + '_'+ str(yeTai3)+'_radius3.json')
    except Exception as e:
        print('getZhouBianPeiTaoListCtr')
        print(id)
        print(path)
        print(e)


def mall():
        project_csv = pd.read_csv(project_url + 'project_basicInfo.csv',encoding='gbk')
        for project in project_csv.itertuples():
            projectId = str(project[1])
            projectName = project[2]
            path = save_url + projectName
            path = path.strip()
            path = path.rstrip("\\")
            isExists = os.path.exists(path)
            if not isExists:
                os.makedirs(path)
            for type in businesstype:
                getPeripheralMatchingTeTaiListCtr()
                YeTaiCountListCtr = get3YeTaiCountListCtr(projectId, path,type['firstbusinesstype'], type['secondbusinesstype'])
                if YeTaiCountListCtr:
                    if YeTaiCountListCtr['data']:
                        type_id_list = YeTaiCountListCtr['data']
                        for type_id in type_id_list:
                            id = type_id['typeid']
                            getZhouBianPeiTaoListCtr(projectId, path,type['firstbusinesstype'], type['secondbusinesstype'],id)
                        print(projectId)
                        print(projectName)
                        print(type['firstbusinesstype'])
                        print(type['secondbusinesstype'])
                    else:
                        print(type['firstbusinesstype'])
                        print(type['secondbusinesstype'])
                else:
                    print(type['firstbusinesstype'])
                    print(type['secondbusinesstype'])


if __name__ == "__main__":
    url1 = 'https://ytz.winshangdata.com/mapapi/api/project/getPeripheralMatchingTeTaiListCtr'
    url2 = 'https://ytz.winshangdata.com/mapapi/api/project/get3YeTaiCountListCtr'
    url3 = 'https://ytz.winshangdata.com/mapapi/api/project/getZhouBianPeiTaoListCtr'
    project_url = 'D:\da数据\\'
    save_url = 'D:套信息\\'
    businesstypefile = 'businesstype2.json'
    businesstype = readJson(businesstypefile)
    mall()