爬虫 Scrapy框架"链家爬取"

版权声明：Meter.Bulacn https://blog.csdn.net/qq_41500222/article/details/83998035

完整的Scarpy框架 我们爬取的是 首页里面的详情页内容

![在这里插入图片描述](https://img-blog.csdnimg.cn/20181112195618749.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzQxNTAwMjIy,size_16,color_FFFFFF,t_70

这是我们获取的内容 将它存入Mysql数据库中

下面我们开始上代码:

首先我们要配置settings.py文件:

# -*- coding: utf-8 -*-

# Scrapy settings for lianjia project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://doc.scrapy.org/en/latest/topics/settings.html
#     https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://doc.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'lianjia'

SPIDER_MODULES = ['lianjia.spiders']
NEWSPIDER_MODULE = 'lianjia.spiders'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'lianjia (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
DOWNLOAD_DELAY = 1
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  # 'Accept-Language': 'en',
    'User-Agent':'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36',
}

# Enable or disable spider middlewares
# See https://doc.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    'lianjia.middlewares.LianjiaSpiderMiddleware': 543,
#}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
   'lianjia.middlewares.LianjiaDownloaderMiddleware': 543,
}

# Enable or disable extensions
# See https://doc.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    'scrapy.extensions.telnet.TelnetConsole': None,
#}

# Configure item pipelines
# See https://doc.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
   'lianjia.pipelines.LianjiaPipeline': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
MYSQL_HOST = 'localhost'
MYSQL_PORT = 3306
MYSQL_USER = 'root'
MYSQL_PWD = 'bc123'
MYSQL_DB = 'Lianjia'

这是项目文件里面的spider文件里的主文件所有的爬虫流程都在这里:

# -*- coding: utf-8 -*-
import scrapy
from lianjia.items import LianjiaItem

class JiaSpider(scrapy.Spider):
    name = 'jia'
    allowed_domains = ['bj.lianjia.com']
    # for i in range(1,10):
    start_urls = ['https://bj.lianjia.com/ershoufang/pg1/']
    # start_urls = ['https://bj.lianjia.com/ershoufang/']

    def start_requests(self):
        for url in self.start_urls:
            yield scrapy.Request(url, callback=self.custom_parse)

        # def parse(self, response):
        #     pass

    def custom_parse(self, response):
        print(response.status)
        # step1.提取公司列表页中的公司详情地址
        # companies = response.xpath('//div[@class="leftContent"]/ul[@class="sellListContent"]/li')
        companies = response.xpath('//div[@class="info clear"]')

        # print(companies)
        for company in companies:
            url = company.xpath('./div[@class="title"]/a/@href').extract_first()
            # print(url)
            yield scrapy.Request(url, callback=self.parse_company_detail)

        # # step2.获取当前页面中其他分页地址
        # other_page = response.xpath('//*[@id="leftContent"]/div[8]/div[2]/div/a[7]/@href').extract()
        # # print(other_page)
        # for pageurl in other_page:
        #     pageUrl1 = 'https://bj.lianjia.com/' + pageurl
        #     pageUrl1 = response.urljoin(pageurl)
        #     print(pageUrl1)
        #     yield scrapy.Request(pageUrl1, callback=self.custom_parse)


    def parse_company_detail(self, response):
        #
        print(response.status)

        lianItem = LianjiaItem()

        # 公司名称：
        lianItem['companyName'] = response.xpath('/html/body/div[3]/div/div/div[1]/h1/@title').extract_first()


        lianItem['companyStatus'] = response.xpath('/html/body/div[5]/div[2]/div[2]/span[1]/text()').extract_first()

        lianItem['phoneNum'] = ''.join(response.xpath(
            '/html/body/div[5]/div[2]/div[3]/div[3]/div[1]/text()').extract()).replace(
            ' ', '').replace('\n', '')

        lianItem['webUrl'] = response.xpath(
            '/html/body/div[5]/div[2]/div[3]/div[1]/div[1]/text()').extract_first()

        lianItem['email'] = ''.join(response.xpath(
            '/html/body/div[5]/div[2]/div[3]/div[1]/div[2]/text()').extract()).replace(
            ' ', '').replace('\n', '')

        lianItem['adress'] = response.xpath(
            '/html/body/div[5]/div[2]/div[3]/div[3]/div[2]').extract_first()

        lianItem['registerCapital'] = ''.join(
            response.xpath('//*[@id="cartCount"]/text()').extract()).replace(
            ' ', '').replace('\n', '')

        lianItem['active'] = ''.join(
            response.xpath('//*[@id="favCount"]/text()').extract()).replace(
            ' ', '').replace('\n', '')

        lianItem['publishTime'] = ''.join(
            response.xpath('/html/body/div[5]/div[2]/div[2]/div[1]/div[1]/span/text()').extract()).replace(
            ' ', '').replace('\n', '')

        lianItem['socialCode'] = ''.join(
            response.xpath('/html/body/div[7]/div[1]/div[3]/div/div[1]/div[2]/a[1]/text()').extract()).replace(
            ' ', '').replace('\n', '')


        print(lianItem)
        yield lianItem
        #这里代表分页 我们巧妙运用到了for循环 然后回调custom_parse函数
        for i in range(2,5):
            url = 'https://bj.lianjia.com/ershoufang/pg{}/'.format(str(i))
            yield scrapy.Request(url,callback=self.custom_parse)

Item.py文件 负责获取字段 这些字段代表你在网站上要获取的需求 然后用Xpath解析数据用:

# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class LianjiaItem(scrapy.Item):

    # 公司名称：
    companyName = scrapy.Field()

    companyStatus = scrapy.Field()

    phoneNum = scrapy.Field()

    webUrl = scrapy.Field()

    email = scrapy.Field()

    adress = scrapy.Field()

    registerCapital = scrapy.Field()

    active = scrapy.Field()

    publishTime = scrapy.Field()

    socialCode = scrapy.Field()

middlewares.py是下载中间件 一般额外设置 这个我们设置成默认 无需改动:

# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals


class LianjiaSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(self, response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(self, response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(self, response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(self, start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)


class LianjiaDownloaderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the downloader middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_request(self, request, spider):
        # Called for each request that goes through the downloader
        # middleware.

        # Must either:
        # - return None: continue processing this request
        # - or return a Response object
        # - or return a Request object
        # - or raise IgnoreRequest: process_exception() methods of
        #   installed downloader middleware will be called
        return None

    def process_response(self, request, response, spider):
        # Called with the response returned from the downloader.

        # Must either;
        # - return a Response object
        # - return a Request object
        # - or raise IgnoreRequest
        return response

    def process_exception(self, request, exception, spider):
        # Called when a download handler or a process_request()
        # (from other downloader middleware) raises an exception.

        # Must either:
        # - return None: continue processing this exception
        # - return a Response object: stops process_exception() chain
        # - return a Request object: stops process_exception() chain
        pass

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

pipelines.py 负责做数据持久化 这里我们用到了mysql数据库 所以把爬取的数据全部存储到mysql

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html



import  pymysql
from lianjia.items import LianjiaItem
class LianjiaPipeline(object):

    def __init__(self,host,port,db,user,pwd):
        self.mysqlConn = pymysql.Connect(host=host,user=user,password=pwd,database=db,port=port,charset='utf8')
        self.cursor = self.mysqlConn.cursor()

    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            host = crawler.settings.get('MYSQL_HOST'),
            port = crawler.settings.get('MYSQL_PORT'),
            db = crawler.settings.get('MYSQL_DB'),
            user = crawler.settings.get('MYSQL_USER'),
            pwd = crawler.settings.get('MYSQL_PWD'),
        )

    def open_spider(self,spider):
        # 爬虫开启的时候调用，只调用一次
        print('爬虫文件开始执行',spider.name)

    def process_item(self, item, spider):
        print('管道文件我来了')
        sql = """
        INSERT INTO jialist(companyName,companyStatus,phoneNum,webUrl,email,adress,registerCapital,active,publishTime,socialCode)
        VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
        """
        try:
            self.cursor.execute(sql,(item['companyName'],item['companyStatus'],item['phoneNum'],item['webUrl'],item['email'],item['adress'],item['registerCapital'],item['active'],item['publishTime'],item['socialCode']))
            self.mysqlConn.commit()
        except Exception as err:
            print(err)
            self.mysqlConn.rollback()
        return item

    def close_spider(self,spider):
        # 爬虫结束的时候会调用，只调用一次
        print('爬虫文件结束执行',spider.name)
        #关闭游标和关闭连接
        self.cursor.close()
        self.mysqlConn.close()

剩余创建项目之类的 详见本人博客其他文章 谢谢观看!