【爬虫】链接二手房信息爬取

import requests
import parsel
import time
import csv

url = 'https://hf.lianjia.com/ershoufang/pg{}/'.format(page)
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
}


f = open('上海二手房信息1.0.csv', mode='a', encoding='utf-8-sig', newline='')
csv_writer = csv.DictWriter(f, fieldnames=['title', 'positionInfo', 'houseInfo','layout', 'area', 'direction','decoration',
                                          'floor', 'buildTime', 'totalPrice', 'unitPrice', 'specialTag'])
csv_writer.writeheader()

for page in range(1, 20):
    print('===========================正在下载第{}页数据================================'.format(page))
    time.sleep(1)
    url = 'https://sh.lianjia.com/ershoufang/pg{}/'.format(page)
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
    }

    response = requests.get(url=url, headers=headers)
    selector = parsel.Selector(response.text)
    lis = selector.css('.sellListContent li')
    dit = {}
    for li in lis:
        title = li.css('.title a::text').get()
        dit['title'] = title

        positionInfo = li.css('.positionInfo a::text').getall()
        info = ' '.join(positionInfo)
        dit['positionInfo'] = info
        
        houseInfo = li.css('.houseInfo::text').get()
        dit['houseInfo'] = houseInfo
        
        houseInfo = houseInfo.replace(' ', '')
        houseInfo = houseInfo.split('|')
        
        dit['layout'] = houseInfo[0]
        dit['area'] = houseInfo[1]
        dit['direction'] = houseInfo[2]
        dit['decoration'] = houseInfo[3]
        dit['floor'] = houseInfo[4]
        dit['buildTime'] = houseInfo[5]
    
        Price = li.css('.totalPrice span::text').get()
        dit['totalPrice'] = Price

        Price = li.css('.unitPrice span::text').get()
        unitPrice = Price.lstrip('单价')
        dit['unitPrice'] = unitPrice

        tag = li.css('.tag span::text').getall()
        tagInfo = ' '.join(tag)
        dit['specialTag'] = tagInfo

        csv_writer.writerow(dit)
        print(dit)