使用scrapy中crawlspider爬取csdn文章

生成crawlspider命令：

scrapy genspider -t crawl csdn "csdn.cn"

在csdn_spider.py

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class CsdnSpdierSpider(CrawlSpider):
name = 'csdn_spdier'
allowed_domains = ['blog.csdn.net']
start_urls = ['http://blog.csdn.net/hbblzjy']

rules = (
    #提取专家的博客地址
    Rule(LinkExtractor(allow=r'blog.csdn.net/\w+$'), follow=True),
    #提取博客详情页的地址
    Rule(LinkExtractor(allow=r'/\w+/article/details/\d+$'),callback="parse_item"),
    #提取专家翻页地址
    Rule(LinkExtractor(allow=r'channelid=\d+&page=\d+$'), follow=True),
    #提取博客列表页翻页
    Rule(LinkExtractor(allow=r'/\w+/article/list/\d+$'), follow=True),
)

def parse_item(self, response):
    item = {}
    item["title"] = response.xpath("//h1/text()").extract_first()
    item['update_time'] = response.xpath("//span[@class='time']/text()").extract_first()
    item["tag"] = response.xpath("//ul[@class='article_tags clearfix csdn-tracking-statistics']//text()").extract()
    # item["content"] = response.xpath("//div[@id='article_content']")
    # print(item)
    yield item

在pipelines.py

import re

class CsdnPipeline(object):
def process_item(self, item, spider):
    item["tag"] = [re.sub(r"\s+|/","",i,re.S) for i in item["tag"]]
    item["tag"] =[i for i in item["tag"] if len(i)>0 and i!='标签：']
    print(item)

在settings.py

BOT_NAME = 'csdn'

SPIDER_MODULES = ['csdn.spiders']

NEWSPIDER_MODULE = 'csdn.spiders'

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'

ROBOTSTXT_OBEY = False