scrapy爬虫框架学习（二）scrapy爬取多级网页信息

scrapy爬虫框架学习（二）scrapy爬取多级网页信息

---

1爬取目标：

1.1 针对一级页面获取专利详情页的链接信息

1.2 针对专利详情页进行详细信息

2.项目代码实现

2.1 item.py:定义要收集的详情数据结构

import scrapy

#定义要存储的数据项目
class CnkipatentItem(scrapy.Item):
    application_number = scrapy.Field()
    application_date = scrapy.Field()
    public_number = scrapy.Field()
    publication_date = scrapy.Field()
    applicant = scrapy.Field()
    address = scrapy.Field()
    common_applicants = scrapy.Field()
    inventor = scrapy.Field()
    international_application = scrapy.Field()
    international_publishing = scrapy.Field()
    into_the_country_date = scrapy.Field()
    patent_agencies = scrapy.Field()
    agents = scrapy.Field()
    original_application_number = scrapy.Field()
    province_code = scrapy.Field()
    summary = scrapy.Field()
    sovereignty_item = scrapy.Field()
    page = scrapy.Field()
    main_classification_number = scrapy.Field()
    patent_classification_Numbers = scrapy.Field()

2.2 patentSpider.py：完成爬取和解析策略

# -*- coding: utf-8 -*-
import scrapy
import re
#引入定义的item数据结构
from cnkiPatent.items import CnkipatentItem

#继承自spider类
class PatentspiderSpider(scrapy.Spider):
    name = 'patentSpider'#根据这个名字执行，如果项目中有多个爬虫记得名字要唯一
    allowed_domains = ['search.cnki.net']
    start_urls = ['http://search.cnki.net/search.aspx?q=%e5%b0%8f%e6%a0%b8%e9%85%b8&rank=relevant&cluster=zyk&val=SCPD']
    base_url = 'http://search.cnki.net/'

#response是通过引擎、调度器、下载器工作下载到的的爬取内容
    def parse(self, response):
        #从一级页面爬取专利详情链接
        patentTitles = response.xpath('//div[@class="wz_content"]')
        for patentTitle in patentTitles:
            patent_url = patentTitle.xpath('.//h3/a/@href').get()
            #对每一个详情进行解析
            yield scrapy.Request(url=patent_url,callback=self.parse_item,dont_filter=True)
        #获取下一页一级页面
        next_url = response.xpath('//div[@class="articles"]/p[@id="page"]'
                                  '/a[@class="n"]/@href').get()
        print(next_url)
        if next_url is not None:
            # 后续给的链接因为已经在这个域里面所以没有开头的域名，要加上
            next_url = self.base_url + next_url
            yield scrapy.Request(url=next_url,callback=self.parse)

        #返回存储的数据结构
        #item = CnkipatentItem()
        #yield item

    def parse_item(self,response):
        patent_details = response.xpath('//table[@id="box"]//td/text()').extract()
        item = CnkipatentItem()
        item['application_number'] = re.sub(r'\xa0','',patent_details[1])
        item['application_date'] = re.sub(r'\xa0','',patent_details[3])
        item['public_number'] = re.sub(r'\xa0','',patent_details[5])
        item['publication_date'] = re.sub(r'\xa0','',patent_details[7])
        item['applicant'] = re.sub(r'\xa0','',patent_details[9])
        item['address'] = re.sub(r'\xa0','',patent_details[11])
        item['common_applicants'] = re.sub(r'\xa0','',patent_details[13])
        item['inventor'] = re.sub(r'\xa0','',patent_details[15])
        item['international_application'] = re.sub(r'\xa0','',patent_details[17])
        item['international_publishing'] = re.sub(r'\xa0','',patent_details[19])
        item['into_the_country_date'] = re.sub(r'\xa0','',patent_details[21])
        item['patent_agencies'] = re.sub(r'\xa0','',patent_details[23])
        item['agents'] = re.sub(r'\xa0','',patent_details[25])
        item['original_application_number'] = re.sub(r'\xa0','',patent_details[27])
        item['province_code'] = re.sub(r'\xa0','',patent_details[29])
        item['summary'] = re.sub(r'\xa0','',patent_details[31])
        item['sovereignty_item'] = re.sub(r'\xa0','',patent_details[33])
        item['page'] = re.sub(r'\xa0','',patent_details[35])
        item['main_classification_number'] = re.sub(r'\xa0','',patent_details[37])
        item['patent_classification_Numbers'] = re.sub(r'\xa0','',patent_details[39])
        yield item

2.3 pipelines.py：定义存储方法，用json为例

import json
import pandas as pd

class CnkipatentPipeline(object):
    #打开时的操作，例如打开要存储的文件：txt、json等
    def open_spider(self,spider):
        self.file = open('小核酸.json', 'wb')

    #将spider中parse函数yield返回来的内容装进item给到这个函数
    def process_item(self, item, spider):
        data = json.dumps(dict(item), ensure_ascii=False, indent=4) + ','
        #编码
        self.file.write(data.encode('utf-8'))

    #关闭spider时的操作，例如关闭要存储的文件
    def close_spider(self,spider):
        self.file.close()

3.得到的数据

仅以一条数据为例

{
    "application_number": "CN201210437612.8",
    "application_date": "2012-11-06",
    "public_number": "CN102899327A",
    "publication_date": "2013-01-30",
    "applicant": "清华大学深圳研究生院;深圳南粤药业有限公司;苏州吉玛基因股份有限公司",
    "address": "518055 广东省深圳市南山区西丽镇深圳大学城清华校区",
    "common_applicants": "",
    "inventor": "张雅鸥;王纠;何杰;谢伟东;许乃寒;李颖;张佩琢;苏宏瑞;万刚;吕青;卢锦华;柳忠义",
    "international_application": "",
    "international_publishing": "",
    "into_the_country_date": "",
    "patent_agencies": "北京纪凯知识产权代理有限公司 11245",
    "agents": "关畅;王慧凤",
    "original_application_number": "",
    "province_code": "44",
    "summary": "本发明公开了一种小核酸及其在制备抑制单纯疱疹病毒(HSV)和人类乳头瘤病毒(HPV)病毒的药物中的应用。该小核酸是双链核酸,其正义链序列为序列表中序列5,其反义链序列为序列表中序列6。该小核酸可制成温度敏感型凝胶制剂。用所述小核酸温度敏感型凝胶制剂注入(或喷入)阴道或口腔等不规则腔道,能抑制上述腔道和宫颈上皮细胞中宿主B2M基因的表达和或外来的HPV?E7的表达,从而能用于抑制HSV-1病毒的人类乳头瘤病毒(HPV)感染和/或治疗HVP所致宫颈癌前病变。",
    "sovereignty_item": "一种小核酸,是双链核酸,其正义链序列为序列表中序列5,其反义链序列为序列表中序列6。",
    "page": "25",
    "main_classification_number": "C12N15/113",
    "patent_classification_Numbers": "C12N15/113;A61K48/00;A61K9/06;A61P35/00;A61P31/20;A61P31/22"
}