# 用框架爬取腾讯招聘首页
# tencent.py
# -*- coding: utf-8 -*-
import scrapy
from ..items import JobItem
from datetime import datetime
class TencentSpider(scrapy.Spider):
name = 'tencent'
allowed_domains = ['tencent.com']
start_urls = ['https://hr.tencent.com/position.php/']
def parse(self, response):
job_list = response.xpath('.//tr')[1:-2]
for job in job_list:
item = JobItem()
p_name = job.xpath('.//td[1]/a/text()').extract()[0]
p_type = job.xpath('.//td[2]/text()').extract()
if p_type:
p_type = p_type[0]
else:
p_type = ''
p_number = job.xpath('.//td[3]/text()').extract()[0]
p_location = job.xpath('.//td[4]/text()').extract()[0]
p_date = job.xpath('.//td[5]/text()').extract()[0]
item['p_name'] = p_name
item['p_type'] = p_type
item['p_number'] = p_number
item['p_location'] = p_location
item['p_date'] = p_date
item['crawl_time'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
item['spider_name'] = self.name
yield item
print(p_name,p_type,p_number,p_location,p_date)
# 构造请求对象
next_cls = response.xpath('.//a[@id="next"]/@class').extract_first()
if next_cls is None:
next_url = response.xpath('.//a[@id="next"]/@href').extract_first()
next_url = 'https://hr.tencent.com/' + next_url
# callback 回调函数 指定解析函数
yield scrapy.Request(next_url,callback=self.parse)
# items.py
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class Day11Item(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
pass
# 明确爬取数据项 类似django models
class JobItem(scrapy.Item):
p_name = scrapy.Field()
p_type = scrapy.Field()
p_number = scrapy.Field()
p_location = scrapy.Field()
p_date = scrapy.Field()
crawl_time = scrapy.Field()
# 数据来源
spider_name = scrapy.Field()
# pipelines.py
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
class Day11Pipeline(object):
def process_item(self, item, spider):
return item
class JobPipeline(object):
def __init__(self):
self.f = open('job.json','w',encoding='utf-8')
def process_item(self,item, spider):
self.f.write(json.dumps(dict(item),ensure_ascii=False) + '\n')
return item
# 爬虫结束调用
def close_spider(self,spider):
self.f.close()
# settings.py
# -*- coding: utf-8 -*-
# Scrapy settings for day11 project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
# http://doc.scrapy.org/en/latest/topics/settings.html
# http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
# http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
BOT_NAME = 'day11'
SPIDER_MODULES = ['day11.spiders']
NEWSPIDER_MODULE = 'day11.spiders'
# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = 'day11 (+http://www.yourdomain.com)'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32
# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
# DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16
# Disable cookies (enabled by default)
# 默认是True
# COOKIES_ENABLED = False
# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False
# Override the default request headers:
# 默认请求头 生效范围 所有爬虫
DEFAULT_REQUEST_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en'
}
# Enable or disable spider middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
# 'day11.middlewares.Day11SpiderMiddleware': 543,
#}
# Enable or disable downloader middlewares
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
DOWNLOADER_MIDDLEWARES = {
# 数字越小 越先经过 范围 1-999
# 'day11.middlewares.MyCustomDownloaderMiddleware': 543,
}
# Enable or disable extensions
# See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
#EXTENSIONS = {
# 'scrapy.extensions.telnet.TelnetConsole': None,
#}
# Configure item pipelines
# See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
'day11.pipelines.JobPipeline': 1,
}
# Enable and configure the AutoThrottle extension (disabled by default)
# See http://doc.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False
# Enable and configure HTTP caching (disabled by default)
# See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
# main.py
from scrapy import cmdline
cmdline.execute('scrapy crawl tencent'.split())
# 爬取部分结果如下：
2018-04-10 14:52:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://hr.tencent.com/position.php?&start=310>
{'crawl_time': '2018-04-10 14:52:45',
'p_date': '2018-04-10',
'p_location': '深圳',
'p_name': '26564-游戏内容编辑(深圳）',
'p_number': '2',
'p_type': '内容编辑类',
'spider_name': 'tencent'}
2018-04-10 14:52:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://hr.tencent.com/position.php?&start=310>
{'crawl_time': '2018-04-10 14:52:45',
'p_date': '2018-04-10',
'p_location': '深圳',
'p_name': '15570-御龙在天端游运营商业化策划（深圳）',
'p_number': '1',
'p_type': '产品/项目类',
'spider_name': 'tencent'}
15614-天刀3D角色设计师(上海） 设计类 1 上海 2018-04-10
26564-游戏内容编辑(深圳） 内容编辑类 2 深圳 2018-04-10
15570-御龙在天端游运营商业化策划（深圳） 产品/项目类 1 深圳 2018-04-10
2018-04-10 14:52:45 [scrapy.core.engine] DEBUG: Crawled (200) <GET https://hr.tencent.com/position.php?&start=320#a> (referer: https://hr.tencent.com/position.php?&start=310)
MIG08-产品经理（上海） 产品/项目类 1 上海 2018-04-10
25663-腾讯云AI行业方案专家（北京） 产品/项目类 1 北京 2018-04-10
2018-04-10 14:52:45 [scrapy.core.scraper] DEBUG: Scraped from <200 https://hr.tencent.com/position.php?&start=320>
兄弟连学python
Python学习交流、资源共享群：563626388 QQ