链家抓取

import requests
from lxml import etree
import mysql_test
# 构建代理
proxy = {
    'http' : 'http://alice:[email protected]:6666',
    'https' : 'http://alice:[email protected]:6666'
}
headers={
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.4620.400 QQBrowser/9.7.13014.400",
}




base_url = 'https://bj.lianjia.com/zufang/changping/pg%d/'
for i in range(1,4):
    url=base_url%i
    response=requests.get(url,headers=headers,proxies=proxy)
    hrml_str=response.text
    html_ele = etree.HTML(hrml_str)
    print('正在保存第一页'+str(i)+'.............')

    li_list = html_ele.xpath('//ul[@id="house-lst"]/li')
    for li_ele in li_list:
        title=li_ele.xpath('./div[2]/h2/a')[0].text
        # print(title)
        region=li_ele.xpath('./div[2]/div[1]/div[1]/a/span')[0].text.strip()

        zone = li_ele.xpath('./div[2]/div[1]/div[1]/span/span')[0].text.strip()

        meters =li_ele.xpath('./div[2]/div[1]/div[1]/span[2]')[0].text.strip()

        location =li_ele.xpath('./div[2]/div[1]/div[1]/span[3]')[0].text

        price = li_ele.xpath('./div[2]/div[2]/div[1]/span')[0].text

        lianjie= li_ele.xpath('./div[2]/h2/a/@href')[0]
        # print(lianjie)

        # 实例化存入mysql的类
        mc = mysql_test.mysql_conn()
        sql = "INSERT INTO lianjiaxinxi (title, region, zone, meters, location, price,lianjie) VALUES ('{title}','{region}','{zone}','{meters}','{location}',{price},'{lianjie}')".format(title=title,region=region,zone=zone,meters=meters,location=location,price=price,lianjie=lianjie)
        mc.execute_modify_mysql(sql)