本文实现爬取安居客二手房房源信息,并保存到本地csv文本中
爬取网址:https://tj.fang.anjuke.com/loupan/all/(天津新房)
#使用模块:BeautifulSoup
import requests
import math
import time
import random
from bs4 import BeautifulSoup
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
uri='https://tj.fang.anjuke.com/loupan/all/'
def get_data(url):
try:
page_headers={"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240",
"Cookie":"isp=true; als=0; sessid=EE6C479F-383E-4532-5FBE-SX0608113249; lps=http%3A%2F%2Fuser.anjuke.com%2Fajax%2FcheckMenu%2F%3Fr%3D0.9773651958602518%26callback%3DjQuery1113019512877839682363_1559964793029%26_%3D1559964793030%7Chttps%3A%2F%2Ftj.fang.anjuke.com%2Floupan%2Fall%2F; lp_lt_ut=e9d09e6a59979c288d00ea3d7f7f7573; __xsptplusUT_8=1; _ga=GA1.2.205429121.1559658612; _gid=GA1.2.1582905499.1559979223; _gat=1; isp=true; wmda_uuid=b01457fc0c7d92a795d75c630ff6262c; wmda_new_uuid=1; wmda_visited_projects=%3B8788302075828; isp=true; aQQ_ajkguid=C8BE4FEC-9EE8-C5E6-D456-C6695B138768; twe=2; wmda_session_id_8788302075828=1559979235829-aa5a0183-f840-376c; __xsptplus8=8.7.1559979226.1559979258.3%234%7C%7C%7C%7C%7C%23%23zPjxAYBc5rYjoCxhfqsuNhjRq2XqZ05X%23; 58tj_uuid=73404522-7a8e-420a-93be-a96ac163fc6d; new_session=0; init_refer=https%253A%252F%252Ftianjin.anjuke.com%252F%253Fpi%253DPZ-baidu-pc-all-biaoti; new_uv=7; Hm_lvt_c5899c8768ebee272710c9c5f365a6d8=1559658645,1559804791,1559964795; Hm_lpvt_c5899c8768ebee272710c9c5f365a6d8=1559979258; ctid=26"
}
res=requests.get(url,headers=page_headers)
status=res.status_code
data=res.text
#print(status)
soup=BeautifulSoup(data,'lxml')
#print(soup.prettify()) #输出格式化的html代码
return soup,status
except Exception as e:
print(str(e))
return 0,0
def get_hourse(url):
soup,status=get_data(url)
fang_all=soup.find_all("div",class_="item-mod ") #注,class是关键字,这里用到需要加个下划线
#print(fang_all)
with open('house_anjuke.csv','a+',encoding='utf-8') as fh:
for fang in fang_all:
try:
#url链接
fang_uri=fang.attrs.get('data-link')
#print(fang_uri)
#楼盘
fang_name=fang.h3.span.text.replace(',',':')
#位置
fang_location=fang.find('span',class_="list-map").text.replace("\xa0"," ").replace(',',":")
#标签
fang_tag=fang.find('div',class_="tag-panel").text.strip().replace('\n',":").replace(',',":")
#价格
fang_price=fang.find('a',class_="favor-pos").p.text
#户型
fang_huxing=str(fang.find('a',class_='huxing').find_all('span')).replace('<span>','').replace('</span>','').replace(',',':')
#print("楼盘:%s" % fang_name)
#print("位置:%s" % fang_location)
#print("户型:%s" % fang_huxing)
#print("价格:%s" % fang_price)
#print("标签:%s" % fang_tag)
#print("---------------------------------------------")
fh.write(fang_name+","+fang_location+","+fang_huxing+","+fang_price+","+fang_tag+","+fang_uri+"\n")
except Exception as e:
print(e)
#get_hourse(uri)
if __name__=='__main__':
url_start='https://tj.fang.anjuke.com/loupan/all/'
soup,status=get_data(url_start)
if status==200:
num=int(soup.find('span',class_='result').em.text)
page_num=math.ceil(num/60)+1
print("结果共%s页" % page_num)
with open('house_anjuke.csv','w',encoding='utf-8') as fh:
fh.write("楼盘,位置,户型,价格,标签,网页链接\n")
for i in range(1,page_num):
url=url_start+'p'+str(i)+'/'
#print(url)
print("正在爬取第 %s 页..." % i)
get_hourse(url)
#随机等待
span=round(random.random()*6,1)
time.sleep(span)
print("ok")