# -*- coding: utf-8 -*-
from requests_html import HTMLSession
import requests
import time
import random
import pymysql
import demjson
session = HTMLSession()
#生成所有分页链接
#rootUrl = https://sjz.58.com/shangpucz/0/pn+pageNum
def getAllPages(rootUrl,num):
urlList = []
for page in range(1,num):
urlList.append(rootUrl+str(page))
return urlList
#获取每页商铺列表链接
def getShopLinks(urls):
urlList = []
headers = {
'Host': 'sjz.58.com',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Cookie': 'f=n; commontopbar_new_city_info=241%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7Csjz; userid360_xml=BA6DEAC7B068D4CA2C0B084D64B2C988; time_create=1557562756785; f=n; commontopbar_new_city_info=241%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7Csjz; id58=c5/njVyUiEkue+APA2AWAg==; 58tj_uuid=0382a3da-caf0-41ed-9711-9a412f14cbcc; new_uv=4; als=0; xxzl_deviceid=zd9%2BGdOPhhmD68uINznQ4Kj6XA%2BaAR%2BMD9jB%2FpK0UvOLwPDosI65BYdMB8V%2BxNWL; mcity=sjz; city=sjz; 58home=sjz; wmda_uuid=ceb37f71d389b50a9f62a743a6407c4a; wmda_new_uuid=1; wmda_visited_projects=%3B2385390625025; ppStore_fingerprint=C05282C938C37C02AA98F51C8C619E78B0D04E47FDE636B0%EF%BC%BF1554971585376; f=n; commontopbar_new_city_info=241%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7Csjz; commontopbar_ipcity=sjz%7C%E7%9F%B3%E5%AE%B6%E5%BA%84%7C0; JSESSIONID=C8B96CE799B9BDAD7390E594ED5E119B; new_session=1; utm_source=; spm=; init_refer=',
'Upgrade-Insecure-Requests': '1',
'Cache-Control': 'max-age=0',
}
for i in urls:
r = session.get(i)
htmlContent = r.html
liContent = htmlContent.find('.content-side-left li')#获取页面所有商铺列表
for url in liContent:
linkStr = url.find('.pic > a')[0].attrs['href']
urlList.append(linkStr)#获取每条商铺信息的详情链接并插入列表
print(linkStr)
time.sleep(3)
return urlList
#获取商铺详情数据
def getShopDetail(shopLink):
try:
dictDetail = {}
r = session.get(shopLink)
htmlContent = r.html
#商铺标题
title = htmlContent.find('.house-title > h1')[0].text
#铺主联系电话
linkmanTel = htmlContent.find('.phone-num')[0].text
#铺主姓名
linkman = htmlContent.find('.agent-name')[0].text
#月租金
rent = htmlContent.find('.house_basic_title_money > .house_basic_title_money_num')[0].text
#配套设施
peitao = htmlContent.find('.peitao-on')
peitaoList = []
for i in peitao:
peitaoList.append(i.text)
#商铺图片
picList = htmlContent.find('.general-pic-list > li > img')
urlList = []
for i in picList:
urlList.append(i.attrs['src'])
#面积 类型 楼层 规格 状态 付款方式 经营行业 起租期 位置
basicContent1 = htmlContent.find('.house_basic_title_content_item2')
basicContent2 = htmlContent.find('.house_basic_title_content_item3')
area = basicContent1[0].text #面积
floor = basicContent1[1].text #楼层
managetype = basicContent1[2].text #经营状态
b_id = basicContent1[3].text #经营类型
shoptype = basicContent2[0].text #商铺类型
miankuan = basicContent2[1].text #面宽
paymethod = basicContent2[2].text #付款方式
startrent = basicContent2[3].text #起租期
keliu = basicContent2[4].text #付款方式
street = basicContent2[5].text #街道
dtailStreet = basicContent2[6].text #详细街道
#打印数据
dictDetail['title'] = title
dictDetail['linkman'] = linkman
dictDetail['linkmanTel'] = linkmanTel
dictDetail['rent'] = rent
dictDetail['area'] = area
dictDetail['floor'] = floor
dictDetail['managetype'] = managetype
dictDetail['b_id'] = b_id
dictDetail['shoptype'] = shoptype
dictDetail['miankuan'] = miankuan
dictDetail['paymethod'] = paymethod
dictDetail['startrent'] = startrent
dictDetail['keliu'] = keliu
dictDetail['street'] = street
dictDetail['dtailStreet'] = dtailStreet
dictDetail['peitaoList'] = demjson.encode(peitaoList)
dictDetail['urlList'] = demjson.encode(urlList)
print('商铺标题：' + title)
print('铺主姓名：' + linkman)
print('铺主联系电话：' + linkmanTel)
print('商铺租金：' + rent)
print('面积：' + area)
print('楼层：' + floor)
print('经营状态：' + managetype)
print('经营类型：' + b_id)
print('商铺类型：' + shoptype)
print('面宽：' + miankuan)
print('付款方式：' + paymethod)
print('起租期：' + startrent)
print('客流来源：' + keliu)
print('街道：' + street)
print('详细街道：' + dtailStreet)
print('配套设施:')
print(peitaoList)
print('商铺图片:')
print(urlList)
return dictDetail
except:
print('详情出错')
# 将数据生成txt文件方法 传入保存文件路径 storagePath 以及文件数据 data
def storageToLocalFiles(storagePath, pageList):
content = ''
fhandle = open(storagePath,"w")
for i in pageList:
content += i + '\n'
fhandle.write(content)
fhandle.close()
#程序执行
if __name__ == "__main__":
#获取所有列表页面
pageList = []
rootUrl = 'https://sjz.58.com/shangpucz/0/pn'
pageList = getAllPages(rootUrl,71)
#print(pageList)
# # 所有页面链接存储到txt文档
storagePath = "shopList.txt"
storageToLocalFiles(storagePath, pageList)
# #所有需要爬取的商铺详情链接
shopList = []
shopList = getShopLinks(pageList)
# #商铺详情链接储存到txt文档
storagePath = "shopDetailList.txt" #定义储存文件名
storageToLocalFiles(storagePath, shopList)
# f = open('surl.txt','r')
# string = f.read()
# shopList = string.split("\n")
# # print(shopList)
# f.close()
#商铺详情字段
for url in shopList:
try:
t = random.randint(2,4)
dictDetail = getShopDetail(url)
#插入数据库
db = pymysql.connect('127.0.0.1', port = 3306, user = 'root', passwd = 'root', db = 'wbug')
cursor = db.cursor()
sql = "insert into shop(title,linkman,linkmantel,rent,area,floor,managetype,b_id,shoptype,miankuan,paymethod,startrent,keliu,street,dtailstreet,peitaolist,urllist) values ('{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}','{}')".format(dictDetail['title'],dictDetail['linkman'],dictDetail['linkmanTel'],dictDetail['rent'],dictDetail['area'],dictDetail['floor'],dictDetail['managetype'],dictDetail['b_id'],dictDetail['shoptype'],dictDetail['miankuan'],dictDetail['paymethod'],dictDetail['startrent'],dictDetail['keliu'],dictDetail['street'],dictDetail['dtailStreet'],dictDetail['peitaoList'],dictDetail['urlList'])
re = cursor.execute(sql)
time.sleep(t)
except:
print('插入数据库出错')
db.close()
cursor.close()
欢迎交流评论建议~