# -*- coding: utf-8 -*- import scrapy from ..items import QutoutiaoItem import json import re from ..settings import CATEGORY_INFO, LIST_LIMIT class QutoutiaoSpider(scrapy.Spider): name = 'qutoutiao' #allowed_domains = ['qutoutiao.net'] start_urls = [] # 各类小标题 categoryInfo = CATEGORY_INFO limit = LIST_LIMIT for value in categoryInfo: url = BASE_API + "cid=%s&tn=1&page=1&limit=%s" % ( str(value['cid']), str(limit)) start_urls.append(url) def parse(self, response): response_url = response.url # 分类id从url获取一次 searchObj = re.search(r'(.*)cid=(\d+)', response_url) cid = searchObj and searchObj.group(2) or 0 data = json.loads(response.text)['data']['data'] for value in data: # 初始化模型对象 item = QutoutiaoItem() # 来源 item['source_name'] = value['source_name'] # 标题 item['title'] = value['title'] # 详细页url url = item['url'] = value['url'] # url = url[0:url.find('?')] # 简介 item['introduction'] = value['introduction'] # 封面图 item['cover'] = value['cover'] # 发布时间 item['publish_time'] = value['publish_time'] # 分类 item['cid'] = cid # 爬取详情页 yield scrapy.Request(url=item['url'], meta={'meta_item': item}, callback=self.detail_parse) # 详情页 def detail_parse(self, response): # 提取每次Response的meta数据 meta_item = response.meta['meta_item'] # 取内容 content_selector = response.xpath('//div[@class="content"]') meta_item['content_images'] = content_selector.xpath( '//img/@src|//img/@data-src').extract() meta_item['content'] = content_selector.extract()[0] yield meta_item # 列表API BASE_API = 'http://api.1sapp.com/content/outList?' # 爬取地址
# -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # http://doc.scrapy.org/en/latest/topics/items.html import scrapy class QutoutiaoItem(scrapy.Item): # define the fields for your item here like: # 文章id aid = scrapy.Field() # 来源 source_name = scrapy.Field() # 标题 title = scrapy.Field() # 详细页url url = scrapy.Field() # 简介 introduction = scrapy.Field() # 封面图 cover = scrapy.Field() # 发布时间 publish_time = scrapy.Field() # 分类ID cid = scrapy.Field() # 内容 content = scrapy.Field() # 内容-中的图片 content_images = scrapy.Field()
# -*- coding: utf-8 -*- # Define here the models for your spider middleware # # See documentation in: # http://doc.scrapy.org/en/latest/topics/spider-middleware.html from scrapy import signals class QutoutiaoSpiderMiddleware(object): # Not all methods need to be defined. If a method is not defined, # scrapy acts as if the spider middleware does not modify the # passed objects. @classmethod def from_crawler(cls, crawler): # This method is used by Scrapy to create your spiders. s = cls() crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) return s def process_spider_input(self, response, spider): # Called for each response that goes through the spider # middleware and into the spider. # Should return None or raise an exception. return None def process_spider_output(self, response, result, spider): # Called with the results returned from the Spider, after # it has processed the response. # Must return an iterable of Request, dict or Item objects. for i in result: yield i def process_spider_exception(self, response, exception, spider): # Called when a spider or process_spider_input() method # (from other spider middleware) raises an exception. # Should return either None or an iterable of Response, dict # or Item objects. pass def process_start_requests(self, start_requests, spider): # Called with the start requests of the spider, and works # similarly to the process_spider_output() method, except # that it doesn’t have a response associated. # Must return only requests (not items). for r in start_requests: yield r def spider_opened(self, spider): spider.logger.info('Spider opened: %s' % spider.name) from fake_useragent import UserAgent import logging class UserAgent_CookiesMiddleware(object): # 随机更换user-agent def __init__(self, crawler): super(UserAgent_CookiesMiddleware, self).__init__() self.ua = UserAgent() self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random") self.logger = logging.getLogger(__name__) @classmethod def from_crawler(cls, crawler): return cls(crawler) def process_request(self, request, spider): def get_ua(): return getattr(self.ua, self.ua_type) random_agent = get_ua() if random_agent: # 记录 request.headers['User-Agent'] = random_agent request.headers['Accept'] = 'application/json, text/javascript, */*; q=0.01' request.headers['Origin'] = 'http://home.qutoutiao.net' request.headers['Referer'] = 'http://home.qutoutiao.net/pages/home.html' self.logger.debug('Current UserAgent: ' + random_agent)
# -*- coding: utf-8 -*- # Define your item pipelines here # # Don't forget to add your pipeline to the ITEM_PIPELINES setting # See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html import scrapy from scrapy.utils.project import get_project_settings from scrapy.pipelines.images import ImagesPipeline import os from .qttutils import QttUtils # 封面下载 class CoverImagePipeline(ImagesPipeline): # 获取settings中的常量 IMAGES_STORE = get_project_settings().get('IMAGES_STORE') # 下载图片 def get_media_requests(self, item, info): cover_images = item['cover'] if cover_images: for image_url in cover_images: yield scrapy.Request(url=image_url) # 下载完成 def item_completed(self, results, item, info): # print('*'*20,results,item,info) image_path = [x['path'] for ok, x in results if ok] # 获取自定义存储路径 store_path = QttUtils.getStorePath() coverImages = [] # 将图片移动到新的路径 print('------------------image_path-',image_path) print('-----------------type(image_path)--', type(image_path)) if image_path: for image_url in image_path: file_name = os.path.split(str(image_url)) print('------------------file_name-', file_name) print('------------------file_name-', type(file_name)) new_image = store_path + '/' + file_name[1] coverImages.append(new_image) os.rename(self.IMAGES_STORE + '/' + image_url, new_image) item['cover'] = coverImages return item # 内容图片下载 class ContentImagePipeline(ImagesPipeline): # 获取settings中的常量 IMAGE_STORE = get_project_settings().get('IMAGES_STORE') # 下载图片 def get_media_requests(self, item, info): content_images = item['content_images'] if content_images: for image_url in content_images: yield scrapy.Request(image_url) # 下载完成 def item_completed(self, results, item, info): image_path = [x['path'] for ok, x in results if ok] print('---------------------image_path', image_path) # 获取自定义存储路径 store_path = QttUtils.getStorePath() contentImages = [] # 将图片移动到新的路径 if image_path: for base_path in image_path: print('----------------value', base_path) file_name = os.path.split(str(base_path)) new_image = store_path + "/" + file_name[1] contentImages.append(new_image) os.rename(self.IMAGE_STORE + "/" + base_path, new_image) item['content_images'] = contentImages return item # # 下载完成 方法一 # def item_completed(self, results, item, info): # image_info = [(x['path'], x['url']) for ok, x in results if ok] # print('---------------------image_info', image_info) # # 获取自定义存储路径 # store_path = QttUtils.getStorePath() # contentImages = [] # # 将图片移动到新的路径 # if image_info: # for value in image_info: # print('----------------value', value) # image_url = value[0] # image_source = value[1] # # file_name = os.path.split(str(image_url)) # new_image = store_path + "/" + file_name[1] # contentImages.append((new_image, image_source)) # os.rename(self.IMAGE_STORE + "/" + image_url, new_image) # item['content_images'] = contentImages # return item import json from .qttutils import QttUtils class QutoutiaoPipeline(object): def __init__(self): # 获取自定义的存储路径 store_path = QttUtils.getStorePath() json_path = store_path + '/' + 'qutoutiao.json' self.filename = open(json_path, 'wb') def process_item(self, item, spider): text = json.dumps(dict(item), ensure_ascii=False) + '\n' self.filename.write(text.encode('utf-8')) return item def close_spider(self, spider): self.filename.close()
# -*- coding: utf-8 -*- # @Time : 2018-6-1 11:01 # @Author : Amir # @Site : # @File : qttutils.py # @Software: PyCharm ''' 趣头条工具类 ''' import time import os import shutil from .settings import DATA_PATH class QttUtils: # 获取存储路径 # # @param [string] action [remove删除目录，默认create] # @return [string] path/year/month/day/* @staticmethod def getStorePath(action='create'): localtimes = time.localtime() year = time.strftime("%Y", localtimes) month = time.strftime('%m', localtimes) day = time.strftime('%d', localtimes) store_path = DATA_PATH + "/%s/%s/%s"%(year,month,day) # 删除目录 if os.path.exists(store_path) and action == 'remove': shutil.rmtree(store_path) # 创建多级目录 if not os.path.exists(store_path) and action == 'create': os.makedirs(store_path) return store_path
# -*- coding: utf-8 -*- # Scrapy settings for QuTouTiao project # # For simplicity, this file contains only settings considered important or # commonly used. You can find more settings consulting the documentation: # # http://doc.scrapy.org/en/latest/topics/settings.html # http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html # http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html BOT_NAME = 'QuTouTiao' SPIDER_MODULES = ['QuTouTiao.spiders'] NEWSPIDER_MODULE = 'QuTouTiao.spiders' #列表记录数 LIST_LIMIT = 10 # 储存路径 DATA_PATH = r'./data' IMAGES_STORE = r'./image' #分类 CATEGORY_INFO = [ {"cid":255,"name":"推荐"}, {"cid":1,"name":"热点"}, {"cid":6,"name":"娱乐"}, {"cid":5,"name":"养生"}, {"cid":2,"name":"搞笑"}, {"cid":7,"name":"科技"}, {"cid":8,"name":"生活"}, {"cid":10,"name":"财经"}, {"cid":9,"name":"汽车"}, ] # Crawl responsibly by identifying yourself (and your website) on the user-agent #USER_AGENT = 'QuTouTiao (+http://www.yourdomain.com)' # Obey robots.txt rules ROBOTSTXT_OBEY = False # Configure maximum concurrent requests performed by Scrapy (default: 16) #CONCURRENT_REQUESTS = 32 # Configure a delay for requests for the same website (default: 0) # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay # See also autothrottle settings and docs DOWNLOAD_DELAY = 3 # The download delay setting will honor only one of: #CONCURRENT_REQUESTS_PER_DOMAIN = 16 #CONCURRENT_REQUESTS_PER_IP = 16 # Disable cookies (enabled by default) COOKIES_ENABLED = False # Disable Telnet Console (enabled by default) #TELNETCONSOLE_ENABLED = False # Override the default request headers: #DEFAULT_REQUEST_HEADERS = { # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', # 'Accept-Language': 'en', #} # Enable or disable spider middlewares # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html #SPIDER_MIDDLEWARES = { # 'QuTouTiao.middlewares.QutoutiaoSpiderMiddleware': 543, #} # Enable or disable downloader middlewares # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html DOWNLOADER_MIDDLEWARES = { 'QuTouTiao.middlewares.UserAgent_CookiesMiddleware': 299, } # Enable or disable extensions # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html #EXTENSIONS = { # 'scrapy.extensions.telnet.TelnetConsole': None, #} # Configure item pipelines # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html ITEM_PIPELINES = { 'QuTouTiao.pipelines.QutoutiaoPipeline': 300, 'QuTouTiao.pipelines.ContentImagePipeline': 301, 'QuTouTiao.pipelines.CoverImagePipeline': 302 } # Enable and configure the AutoThrottle extension (disabled by default) # See http://doc.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'