根据官方文档做的简单练习,唯一遇到的问题就是爬取返回403.解决方法是在settings.py文件中增加以下参数:
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
下面是spider的内容:
# -*- coding: utf-8 -*-
import scrapy
class MoviesSpider(scrapy.Spider):
name = 'movies'
allowed_domains = ['movie.douban.com']
start_urls = ['https://movie.douban.com/top250']
def parse(self, response):
grid_view=response.css('ol.grid_view')
for li_item in grid_view.css('li'):
yield {
'rank':li_item.css('div.item div.pic em::text').extract_first(),
'url':li_item.css('div.item div.pic a::attr(href)').extract_first(),
'title_zh':li_item.css('div.hd a span:first-child::text').extract_first(),
'title_en':li_item.css('div.hd a span:nth-child(2)::text').extract_first(),
'title_tw':li_item.css('div.hd a span:last-child::text').extract_first(),
'editor':li_item.css('div.bd p:first-child::text').extract_first(),
'star':li_item.css('div.bd div.star span.rating_num::text').extract_first(),
'votes':li_item.css('div.bd div.star span:last-child::text').re(r'(\d+)')[0],
'desc':li_item.css('span.inq::text').extract_first()
}
next_page=response.css('span.next a::attr(href)').extract_first()
if next_page is not None:
yield response.follow(next_page,self.parse)