# -*- coding: utf-8 -*-
import scrapy
from demo1.items import GameItem
class Top17173Spider(scrapy.Spider):
name = 'top17173'
allowed_domains = ['top17173.com']
start_urls = ['http://top.17173.com/list-0-0-0-0-0-0-0-0-0-0-1.html','http://top.17173.com/list-0-0-4-0-0-0-0-0-0-0-1.html','http://top.17173.com/vr-index.shtml']
def parse(self, response):
name = response.xpath('//div[@class="item-in"]/div[@class="c2"]//a/text()').extract()
poll = response.xpath('//div[@class="item-in"]/div[@class="c3"]/text()').extract()
polls = []
names = []
for p in poll:
p = ''.join(p.split())
polls.append(p)
for n in name:
n = ''.join(n.split())
names.append(n)
# print(polls)
# print(names)
item = GameItem()
for name,poll in zip(names,polls):
item['name']=name
item['poll']=poll
if response.url.find('list-0-0-0-0-0-0-0-0-0-0-1.html')!=-1:
item['type']='网游'
elif response.url.find('list-0-0-4-0-0-0-0-0-0-0-1.html')!=-1:
item['type']='手游'
else:
item['type']='VR'
yield item
#创建item对象类
items.py
# -*- coding: utf-8 -*-
# Define here the models for your scraped items
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class GameItem(scrapy.Item):
# define the fields for your item here like:
name = scrapy.Field()
poll = scrapy.Field()
type = scrapy.Field()
#保存文件
pipelines.py
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
class Demo1Pipeline(object):
def open_spider(self,spider):
self.file = open('game.txt','a',encoding='utf-8')
def process_item(self, item, spider):
# if item['type']=='网游':
# with open('wangyou.txt','w',encoding='utf-8')
self.file.write(json.dumps(dict(item),ensure_ascii=False)+'\n')
print(item)
self.file.flush()
return item
def close_spider(self,spider):
self.file.close()