scrapy 使用rule LinkExtractor提取链接

# coding=utf-8
import scrapy
from scrapy.linkextractors import LinkExtractor


class Jianshu(scrapy.Spider):
    name = "jianshu_spider"
    allowed_domains = ["jianshu.com"]
    start_urls = ['https://www.jianshu.com/']

    def parse(self, response):
        link = LinkExtractor(restrict_xpaths='//ul[@class="note-list"]/li/div[1]/a')
        links = link.extract_links(response)
        for link_one in links:
            print(link_one.url, link_one.text)

参考链接: https://www.jianshu.com/p/7c5d41c61ad2

第二版:

# coding=utf-8

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders.crawl import Rule, CrawlSpider
from douban_dushu.items import DoubanDushuItem


class DouBan(CrawlSpider):
    name = "douban"
    start_urls = ['https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4']
    '''获取url'''
    rules = (
        Rule(LinkExtractor(restrict_xpaths='//*[@class="subject-list"]/li/div[2]/h2'), callback='parse_items'),
    )

    def parse_items(self, response):
        item = DoubanDushuItem()
        '''获取详情页name'''
        item['book_name'] = response.xpath('//span[@property="v:itemreviewed"]/text()').extract()
        yield item