python_爬取赢商网_项目(商业地产)数据

import os
import random

import json
import time
import ssl
import pandas as pd
import requests
import urllib3
#此文件我已上传到我的下载
from utils.read_write import writeOneJSON

ssl._create_default_https_context = ssl._create_unverified_context
header = {
    'Host': 'ytz.winshangdata.com',
    'appType': 'ytzapp',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1',
    'platform': 'android',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate',
    'Accept-Language': 'zh-CN,en-US;q=0.9',
    'X-Requested-With': 'com.winshangdata.ytz',
    'uuid': '860000000229756,860000000229756',
    'token': 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJleHBpclwiOjE1NzQzNDAxNDYxNTksXCJ1aWRcIjpcIkphY2tzb24yMjJcIixcInV1aWRcIjpcIjE1ODY5NzQyMTA4XCJ9IiwiaWF0IjoxNTczODIxNzQ2fQ.bQVKB1khKYu5LI9Do4zzkpqJd9xGDh5eNIJqcC7CORw',
    'Cookie': 'auth={%22token%22:%22eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJleHBpclwiOjE1NzM3NDg0OTE4NDEsXCJ1aWRcIjpcIkphY2tzb24yMjJcIixcInV1aWRcIjpcIjE1ODY5NzQyMTA4XCJ9IiwiaWF0IjoxNTczMjMwMDkxfQ.vtCnWjWyM5tD8lmb2lWzhLfKzg4mTw-9W8mFmzdeVq8%22%2C%22refreshToken%22:%22eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ7XCJleHBpclwiOjE1NzQxMTkxMjQ1NDUsXCJ1aWRcIjpcIkphY2tzb24yMjJcIixcInV1aWRcIjpcIjE1ODY5NzQyMTA4XCJ9IiwiaWF0IjoxNTczMjMwMDkxfQ.7RrT9bIUO06wRDVsjHG_eSLQKa2IaLvyX-qRHvAWY48%22%2C%22uid%22:%2215869742108%22%2C%22uuid%22:%22860000000229756%2C860000000229756%22}',
    'User-Agent': 'Mozilla/5.0 (Linux; Android 7.1.1; OPPO A73t Build/N6F26Q; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/62.0.3202.84 Mobile Safari/537.36 uni-app Html5Plus/1.0 (Immersed/18.0)'
}
def getProjectRadius(id, path):
    url_getBrandNumVoList = "https://ytz.winshangdata.com/mapapi/api/project/getBrandNumVoList"
    getKilometerSupportingFacilitiesCount = "https://ytz.winshangdata.com/mapapi/api/project/getKilometerSupportingFacilitiesCount"
    getProjectPeripheryXBaoGaoCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectPeripheryXBaoGaoCtr"
    getProjectZhouBianRfCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectZhouBianRfCtr"
    getProjectKmListCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectKmListCtr"
    BasicInformationCtr = "https://ytz.winshangdata.com/mapapi/api/project/getProjectBasicInformationCtr"
    try:
        formdata = {
            "projectId": id,
            "radius": '3',
            "type": '1',
        }
        urllib3.disable_warnings()
        response = requests.post(url_getBrandNumVoList, json=formdata, headers=header, verify=False)
        url_getBrandNumVoList = json.loads(response.text)
        writeOneJSON(url_getBrandNumVoList,
                     path + '\\' + str(id) + '_' + 'getBrandNumVoList_radius_3.json')
        formdata = {
            "projectId": id,
            "radius": '3',
        }
        response = requests.post(BasicInformationCtr, json=formdata, headers=header, verify=False)
        BasicInformationCtr = json.loads(response.text)
        writeOneJSON(BasicInformationCtr, path + '\\' + str(id) + '_' + 'BasicInformationCtr_radius_3.json')
        response = requests.post(getKilometerSupportingFacilitiesCount, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'getKilometerSupportingFacilitiesCount_radius_3.json')
        response = requests.post(getProjectPeripheryXBaoGaoCtr, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'getProjectPeripheryXBaoGaoCtr_radius_3.json')
        response = requests.post(getProjectZhouBianRfCtr, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'getProjectZhouBianRfCtr_radius_3.json')
        response = requests.post(getProjectKmListCtr, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'getProjectKmListCtr_radius_3.json')
        return BasicInformationCtr
    except Exception as e:
        print('getProjectRadius')
        print(id)
        print(path)
        print(e)


def getLocationValueCtr(id, path, lon, lat):
    url = "https://ytz.winshangdata.com/mapapi/api/project/getLocationValueCtr"
    try:
        formdata = {
            "latitude": lat,
            "longitude": lon,
            "radius": radius,
        }
        urllib3.disable_warnings()
        response = requests.post(url, json=formdata, headers=header, verify=False)
        getLocationValueCtr = json.loads(response.text)
        writeOneJSON(getLocationValueCtr, path + '\\' + str(id) + '_' + 'getLocationValueCtr_radius_3.json')
    except Exception as e:
        print('getLocationValueCtr')
        print(id)
        print(path)
        print( e)


def getProject(id, path):
    getYiQianYueList3_5_forYtz = "https://ytz.winshangdata.com/wsapi/project/getYiQianYueList3_5_forYtz"
    coreRadiationScopeReport = "https://ytz.winshangdata.com/mapapi/api/project/coreRadiationScopeReport"
    detailQianYueBrand3_6_forYtz = "https://ytz.winshangdata.com/wsapi/project/detailQianYueBrand3_6_forYtz"
    getRadiationRangeCtr_url = "https://ytz.winshangdata.com/mapapi/api/project/getRadiationRangeCtr"
    detailZhaoShangXuQiu = "https://ytz.winshangdata.com/wsapi/project/detailZhaoShangXuQiu"
    try:
        formdata = {
            "floor": '',
            "industryType_id": '1001',
            "projectId": id,
        }
        urllib3.disable_warnings()
        response = requests.post(getYiQianYueList3_5_forYtz, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'getYiQianYueList3_5_forYtz.json')
        formdata = {
            "projectId": id,
        }
        response = requests.post(coreRadiationScopeReport, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'coreRadiationScopeReport.json')
        response = requests.post(detailQianYueBrand3_6_forYtz, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'detailQianYueBrand3_6_forYtz.json')
        response = requests.post(getRadiationRangeCtr_url, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'getRadiationRangeCtr_url.json')
        response = requests.post(detailZhaoShangXuQiu, json=formdata, headers=header, verify=False)
        data = json.loads(response.text)
        writeOneJSON(data, path + '\\' + str(id) + '_' + 'detailZhaoShangXuQiu.json')
    except Exception as e:
        print('getProject')
        print(id)
        print(path)
        print(e)


def mall():
    project_china = pd.read_csv(project_url,encoding='gbk')
    for index,project in project_china.iterrows():
        project = project_china.loc[index]
        projectId = str(project[0])
        projectName = project[1]
        path = save_url + projectName
        path = path.strip()
        path = path.rstrip("\\")
        isExists = os.path.exists(path)
        if not isExists:
            os.makedirs(path)
        BasicInformationCtr =getProjectRadius(projectId, path)
        getProject(projectId, path)
        if BasicInformationCtr:
            if BasicInformationCtr['data']:
                lon = BasicInformationCtr['data']['longitude']
                lat = BasicInformationCtr['data']['latitude']
                getLocationValueCtr(projectId, path,lon,lat,3)
                # getLocationValueCtr(projectId, path,lon,lat,5)
        else:
            print('以下项目基本数据为空')
            print(projectId)
            print(projectName)

def project_analysis():
    for n,projectIdName in project_name_id.iterrows():
        projectIdName = project_name_id.loc[n+2419]
        projectId = int(projectIdName[0])
        projectName = projectIdName[1]
        path = save_url + projectName
        path = path.strip()
        path = path.rstrip("\\")
        isExists = os.path.exists(path)
        if not isExists:
            os.makedirs(path)
            BasicInformationCtr =getProjectRadius(projectId, path)
            getProject(projectId, path)
            if BasicInformationCtr:
                if BasicInformationCtr['data']:
                    lon = BasicInformationCtr['data']['longitude']
                    lat = BasicInformationCtr['data']['latitude']
                    getLocationValueCtr(projectId, path,lon,lat,3)
                    # getLocationValueCtr(projectId, path,lon,lat,5)
            else:
                print('以下项目基本数据为空')
                print(projectId)
                print(projectName)


if __name__ == "__main__":
    save_url = 'D:\da\\'
    project_url = 'D:\dcInfo.csv'
    project_name_id = pd.read_csv(project_url,encoding='gbk',error_bad_lines=False)
    project_analysis()
    # CommercialDistrict()