0.前言
最近有一段时间没爬虫了，现在来学习一下爬虫框架及数据库操作！
看完这篇文章，你可以学会如下操作！
-
scrapy框架
-
BeautifulSoup
-
lxml
-
selenium
-
pyecharts
-
pymysql
1.建立项目
scrapy startproject CQJob
scrapy genspider cqjobs
2.spider+selenium
start_urls配置
start_urls = ['https://www.lagou.com/jobs/list_?px=default&hy=%E5%8C%BB%E7%96%97%E5%81%A5%E5%BA%B7&city=%E9%87%8D%E5%BA%86'] # 配置url
chromedriver配置
# 开启chromedriver
browser = webdriver.Chrome()
browser.get(self.start_urls[0])
browser.implicitly_wait(10)
#
# 写入源html
# f = open('./wr.txt', 'w', encoding='utf8')
# raw_html = browser.page_source
# f.write(raw_html)
# f.close()
.......
BeautifulSoup及xpath使用对多页面处理代码
............
browser.close() # 关闭浏览器
BeautifulSoup及xpath使用对多页面处理
# 使用BeautifulSoup定位
'''
pager_next
pager_next pager_next_disabled
'''
for i in range(11):
selector = etree.HTML(browser.page_source) # 获取源码
soup = BeautifulSoup(browser.page_source, features='lxml')
# a = soup.find_all("div", class_="pager_container")
span = soup.find("div", attrs={"class": "pager_container"}).find("span", attrs={"action": "next"})
# f = open('./new.txt', 'w', encoding='utf8')
# f.write(str(span))
# f.close()
classSpan = span['class']
print('----------------------------------------------')
print(classSpan) # 输出内容为 -> ['pager_next', 'pager_next_disabled']
next_flag = classSpan[1]
self.parsedata(selector)
if next_flag == "pager_next_disabled":
print("已经爬到最后一页，爬虫结束")
break
else:
print("还有下一页，爬虫继续")
# 这里一定要注意不能直接copy源代码的xpath，因为每个页面元素标签有可能不一样！
browser.find_element_by_xpath('//*[@id="s_position_list"]/div[2]/div/span[@action="next"]').click() # 点击下一页
time.sleep(5)
print('第{}页抓取完毕'.format(i + 1))
数据定义及封装
items.py
class CqjobItem(scrapy.Item):
# define the fields for your item here like:
# name = scrapy.Field()
name = scrapy.Field() # 职位
company = scrapy.Field() # 公司名称
location = scrapy.Field() # 公司地点
welfare = scrapy.Field() # 福利
salaryMin = scrapy.Field() # 工资下限
salaryMax = scrapy.Field() # 工资上限
salaryMid = scrapy.Field() # 平均工资
experience = scrapy.Field() # 工作经验
education = scrapy.Field() # 教育程度
companyType = scrapy.Field() # 公司类型
companyLevel = scrapy.Field() # 公司级别
companySize = scrapy.Field() # 公司人数规模
cqjobs.py(spiders文件)
name_list = []
location_list = []
company_list = []
welfare_list = []
salaryMin_list = []
salaryMax_list = []
salaryMid_list = []
experience_list = []
education_list = []
companyType_list = []
companyLevel_list = []
companySize_list = []
items = CqjobItem()
items['name'] = self.name_list
items['company'] = self.company_list
items['location'] = self.location_list
items['welfare'] = self.welfare_list
items['salaryMin'] = self.salaryMin_list
items['salaryMax'] = self.salaryMax_list
items['salaryMid'] = self.salaryMid_list
items['experience'] = self.experience_list
items['education'] = self.education_list
items['companyType'] = self.companyType_list
items['companyLevel'] = self.companyLevel_list
items['companySize'] = self.companySize_list
print(items)
xpath爬取特定数据
def parsedata(self, selector):
sel_list = selector.xpath('//*[@id="s_position_list"]/ul/li')
for item in sel_list:
name = item.xpath('div[1]/div[1]/div[1]/a/h3/text()')[0]
self.name_list.append(name)
location = item.xpath('div[1]/div[1]/div[1]/a/span/em/text()')[0]
self.location_list.append(location)
company = item.xpath('div[1]/div[2]/div[1]/a/text()')[0]
self.company_list.append(company)
welfare = item.xpath('div[2]/div[2]/text()')[0]
self.welfare_list.append(welfare)
salaryList = item.xpath('div[1]/div[1]/div[2]/div/span/text()')[0].strip().split("-")
# print(salaryList) # [10k-15k]
salaryMin = salaryList[0][:len(salaryList[0]) - 1] # 10 去除k,只留数字
self.salaryMin_list.append(salaryMin)
salaryMax = salaryList[1][:len(salaryList[1]) - 1] # 15
self.salaryMax_list.append(salaryMax)
salaryMid = (int(salaryMin) + int(salaryMax)) / 2
self.salaryMid_list.append(salaryMid)
educationArray = item.xpath('div[1]/div[1]/div[2]/div//text()')[3].strip().split("/")
# print(educationArray)
experience = educationArray[0].strip()
self.experience_list.append(experience)
education = educationArray[1].strip()
self.education_list.append(education)
# conmpanyMsgArray = item.xpath('div[1]/div[2]/div[2]/text()')[0].strip().split("/")
conmpanyMsgList = item.xpath('div[1]/div[2]/div[2]/text()')[0].strip().split("/")
companyType = conmpanyMsgList[0].strip()
self.companyType_list.append(companyType)
companyLevel = conmpanyMsgList[1].strip()
self.companyLevel_list.append(companyLevel)
companySize = conmpanyMsgList[2].strip()
self.companySize_list.append(companySize)
数据分析
# TreeMap数据格式
def getTreeData(self, treedata):
treemap_data = []
for key in treedata:
if key != '重庆':
treemap_data.append({"value": treedata[key], "name": key})
return treemap_data
# 位置数据分析
def LocAnalysis(self, items):
loca_data = items['location']
list_data = set(loca_data)
treemap_data = {}
for item in list_data:
treemap_data[item] = loca_data.count(item)
print(treemap_data)
data = self.getTreeData(treemap_data) # 转换为相应的TreeMap数据
print(data)
treemap = TreeMap("重庆医疗健康位置分布图", width=1200, height=600, title_pos="center")
treemap.add("位置数据", data, is_label_show=True, label_pos='inside', label_text_color='#000', is_legend_show=False)
treemap.render()
def educaAnalysis(self, items):
educa_data = items['education']
educalist_data = set(educa_data)
print(educalist_data)
edupie_list = []
edupie_data = []
for item in educalist_data:
edupie_list.append(item)
edupie_data.append(educa_data.count(item))
print(edupie_list)
print(edupie_data)
pie = Pie("重庆医疗健康招聘学历要求", title_pos='center')
pie.add(
"学历",
edupie_list,
edupie_data,
center=[50, 50],
is_random=True,
radius=[30, 75],
rosetype="area",
is_legend_show=False,
is_label_show=True,
)
pie.render()
# 位置数据分析
def CompanyAnalysis(self, items):
loca_data = items['company']
list_data = set(loca_data)
treemap_data = {}
for item in list_data:
treemap_data[item] = loca_data.count(item)
print(treemap_data)
data = self.getTreeData(treemap_data) # 转换为相应的TreeMap数据
print(data)
treemap = TreeMap("重庆医疗相关公司分布图", width=1500, height=900, title_pos="center")
treemap.add("公司数据", data, is_label_show=True, label_pos='inside', label_text_color='#000', is_legend_show=False)
treemap.render()
# 工资数据分析
def SalaryAnalysis(self, items):
axis_data = items['name']
print(axis_data)
ayis_data = items['salaryMid']
print(ayis_data)
bar = Bar("重庆医疗职位平均工资图", width=1500, height=450, title_pos="center")
bar.add("工资数据", axis_data, ayis_data, mark_line=["average"], mark_point=["max", "min"], legend_pos='right',
is_xaxis_show=False)
bar.render()
def SalaryTreeAnalysis(self, items):
salary_name = items['name']
salary_data = items['salaryMid']
salary_set = set(salary_name)
treemap_data = {}
for item in salary_set:
treemap_data[item] = salary_data
print(treemap_data)
data = self.getTreeData(treemap_data) # 转换为相应的TreeMap数据
print(data)
treemap = TreeMap("重庆医疗职位工资分布图", width=1500, height=900, title_pos="center")
treemap.add("职位数据", data, is_label_show=True, label_pos='inside', label_text_color='#000', is_legend_show=False)
treemap.render()
数据分析调用
self.LocAnalysis(items)
self.educaAnalysis(items)
self.CompanyAnalysis(items)
self.SalaryAnalysis(items)
self.SalaryTreeAnalysis(items)
3.数据存储
settings.py
# Crawl responsibly by identifying yourself (and your website) on the user-agent
USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
# Obey robots.txt rules
ROBOTSTXT_OBEY = False
ITEM_PIPELINES = {
'CQJob.pipelines.CqjobPipeline': 300,
}
pipelines.py
import pymysql
class CqjobPipeline(object):
def process_item(self, item, spider):
'''
将爬取的信息保存到mysql
'''
connection = pymysql.connect(host='localhost', user='root', password='xxxx', db='scrapydb', charset='utf8mb4')
try:
with connection.cursor() as cursor:
for i in range(len(item['name'])):
sql = "insert into `cqjobs`(`name`,`company`,`location`,`welfare`,`salaryMin`,`salaryMax`,`salaryMid`,`experience`,`education`,`companyType`,`companyLevel`,`companySize`)values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cursor.execute(sql, (
item['name'][i], item['company'][i], item['location'][i], item['welfare'][i],
item['salaryMin'][i], item['salaryMax'][i], item['salaryMid'][i], item['experience'][i],
item['education'][i], item['companyType'][i], item['companyLevel'][i], item['companySize'][i]))
connection.commit()
# except pymysql.err.IntegrityError as e:
# print('重复数据，勿再次插入!')
finally:
connection.close()
return item
