python爬虫——智联招聘（上）

开发环境

win7+，python3.4+

pymysql库，安装：pip3 install pymysql

selenium库，火狐浏览器56.0版本，geckodriver.exe，selenium知识点

MySQL5.5数据库，Navicat图形化界面

爬取步骤

1.分析智联招聘网，获取网页信息

    打开“https://www.zhaopin.com/”选择城市“北京”，输入“GIS”点击“搜工作”网页将显示与“GIS”相关的北京地区的招聘信息

   F12进去开发者后台“城市”“工作输入”“搜工作按钮”的html元素分别为“id=JobLocation”，“id=KeyWord_kw2”，“class=dosearch”（selenium知识点）。根据这些可以自动转入下个页面：

代码一：

def get_main_page(keyword, city):
    fox = webdriver.Firefox()
    url = 'https://www.zhaopin.com/'
    fox.get(url)
    time.sleep(1)

    jl = fox.find_element_by_id('JobLocation')
    jl.clear()
    jl.send_keys(city)

    zl = fox.find_element_by_id('KeyWord_kw2')
    zl.clear()
    zl.send_keys(keyword)

    sj = fox.find_element_by_class_name('doSearch').click()
    time.sleep(3)

2.分析招聘信息，获取信息

    查看源代码找到各个部分的信息具体如下

def get_everypage_info(fox, keyword, city):
    fox.switch_to_window(fox.window_handles[-1])
    tables = fox.find_elements_by_tag_name('table')

for i in range(0, len(tables)):
    if i == 0:
        '''
        row = ['职位名称', '公司名称', '工作地点', '公司规模', '工作经验', '平均月薪', '学历要求', '职位描述']
        information.append(row)
        '''
    else:
        address, develop, jingyan, graduate, require = " ", " ", " ", " ", " "
        job = tables[i].find_element_by_tag_name('a').text
        company = tables[i].find_element_by_css_selector('.gsmc a').text
        salary = tables[i].find_element_by_css_selector('.zwyx').text

        spans = tables[i].find_elements_by_css_selector('.newlist_deatil_two span')
        for j in range(0, len(spans)):
            if "地点" in spans[j].get_attribute('textContent'):
                address = (spans[j].get_attribute('textContent'))[3:]
            elif "公司规模" in spans[j].get_attribute('textContent'):
                develop = (spans[j].get_attribute('textContent'))[5:]
            elif "经验" in spans[j].get_attribute('textContent'):
                jingyan = (spans[j].get_attribute('textContent'))[3:]
            elif "学历" in spans[j].get_attribute('textContent'):
                graduate = (spans[j].get_attribute('textContent'))[3:]

        require = (tables[i].find_element_by_css_selector('.newlist_deatil_last').get_attribute('textContent'))[8:]

以上代码得到每一页的每个招聘公司的信息：职位名称', '公司名称', '工作地点', '公司规模', '工作经验', '平均月薪', '学历要求', '职位描述'

3.信息存入MySQL数据库

    连接mysql并且创建新表，将数据逐行写入数据库，同时将“职位描述”写入一个txt文件

连接mysql：

table_name = city + '_' + keyword
conn = pymysql.Connect(host='127.0.0.1', port=3306, user='root', passwd='', db='python', charset='utf8')
cursor = conn.cursor()

创建新表：

sql = """CREATE TABLE IF NOT EXISTS %s(
            职位名称 CHAR(100),
            公司名称 CHAR(100),
            工作地点 CHAR(100),
            公司规模 CHAR(100),
            工作经验 CHAR(100),
            平均月薪 CHAR(100),
            学历要求 CHAR(100)
             )default charset=UTF8""" % (table_name)
cursor.execute(sql)

将信息分别写入mysql和txt：

insert_row = ('insert into {0}(职位名称,公司名称,工作地点,公司规模,工作经验,平均月薪,学历要求) VALUES(%s,%s,%s,%s,%s,%s,%s)'.format(table_name))
insert_data = (job, company, address, develop, jingyan, salary, graduate)
cursor.execute(insert_row, insert_data)
conn.commit()
with open('%s职位描述.txt' % (table_name), 'a', encoding='utf-8') as f:
    f.write(require)

4.招聘信息页面跳转

“下一页”按钮的html元素通过下面代码找到并跳转：

count = 0
while count <= 10:
    try:
        next_page = fox.find_element_by_class_name('pagesDown-pos').click()
        break
    except:
        time.sleep(8)
        count += 1
        continue

if count > 10:
    fox.close()
else:
    time.sleep(1)
    get_everypage_info(fox, keyword, city)

注意：此处十分重要，while循环用于判断是否到了最后一页，如果进行10次“next_page = fox.find_element_by_class_name('pagesDown-pos').click()”仍然没反应，就会跳出循环进去下面的if，关闭浏览器；如果“next_page = fox.find_element_by_class_name('pagesDown-pos').click()”有反应break也会跳出while进入下面“else”进而跳转到下一页

5.“main”设置进行城市循环

if __name__ == "__main__":
    citys = ['上海', '深圳', '广州', '武汉', '杭州', '南京', '成都', '青岛']   # '北京',   已爬取
    job = '数据挖掘分析'
    for city in citys:
        print(" ")
        get_main_page(job, city)

每个城市的job信息爬取完了自动进行列表中下个城市信息爬取

6.注意和问题

（1）创建mysql表问题一：定义表的编码形式“default charset=UTF8”，不然输入写入时报错

（2）数据写入mysql表问题二：'insert into {0}(职位名称,公司名称,工作地点,公司规模,工作经验,平均月薪,学历要求) VALUES(%s,%s,%s,%s,%s,%s,%s)'.format(table_name)处要先将表名带入，insert 语句中表名和列名都不能带单引号和双引号，提前写入可以避免。和值一起写入时默认代了引号；

 insert_row = ('insert into {0}(职位名称,公司名称,工作地点,公司规模,工作经验,平均月薪,学历要求) VALUES(%s,%s,%s,%s,%s,%s,%s)'.format(table_name))

            insert_data = (job, company, address, develop, jingyan, salary, graduate)

            cursor.execute(insert_row, insert_data)

（3）time.sleep（）根据网速和电脑性能而定，上佳的时间可以设置短；不佳的就要适当延长时间设置，不让代码将捕捉不到html元素

完整代码：

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time
import pymysql

def get_main_page(keyword, city):
    fox = webdriver.Firefox()
    url = 'https://www.zhaopin.com/'
    fox.get(url)
    time.sleep(1)

    jl = fox.find_element_by_id('JobLocation')
    jl.clear()
    jl.send_keys(city)

    zl = fox.find_element_by_id('KeyWord_kw2')
    zl.clear()
    zl.send_keys(keyword)

    sj = fox.find_element_by_class_name('doSearch').click()
    time.sleep(3)
    get_everypage_info(fox, keyword, city)

def get_everypage_info(fox, keyword, city):
    fox.switch_to_window(fox.window_handles[-1])
    tables = fox.find_elements_by_tag_name('table')

    table_name = city + '_' + keyword
    conn = pymysql.Connect(host='127.0.0.1', port=3306, user='root', passwd='', db='python', charset='utf8')
    cursor = conn.cursor()

    sql = """CREATE TABLE IF NOT EXISTS %s(
                职位名称 CHAR(100),
                公司名称 CHAR(100),
                工作地点 CHAR(100),
                公司规模 CHAR(100),
                工作经验 CHAR(100),
                平均月薪 CHAR(100),
                学历要求 CHAR(100)
                 )default charset=UTF8""" % (table_name)
    cursor.execute(sql)

    for i in range(0, len(tables)):
        if i == 0:
            '''
            row = ['职位名称', '公司名称', '工作地点', '公司规模', '工作经验', '平均月薪', '学历要求', '职位描述']
            information.append(row)
            '''
        else:
            address, develop, jingyan, graduate, require = " ", " ", " ", " ", " "
            job = tables[i].find_element_by_tag_name('a').text
            company = tables[i].find_element_by_css_selector('.gsmc a').text
            salary = tables[i].find_element_by_css_selector('.zwyx').text

            spans = tables[i].find_elements_by_css_selector('.newlist_deatil_two span')
            for j in range(0, len(spans)):
                if "地点" in spans[j].get_attribute('textContent'):
                    address = (spans[j].get_attribute('textContent'))[3:]
                elif "公司规模" in spans[j].get_attribute('textContent'):
                    develop = (spans[j].get_attribute('textContent'))[5:]
                elif "经验" in spans[j].get_attribute('textContent'):
                    jingyan = (spans[j].get_attribute('textContent'))[3:]
                elif "学历" in spans[j].get_attribute('textContent'):
                    graduate = (spans[j].get_attribute('textContent'))[3:]

            require = (tables[i].find_element_by_css_selector('.newlist_deatil_last').get_attribute('textContent'))[8:]

            row = [job, company, address, develop, jingyan, salary, graduate, require]
            insert_row = ('insert into {0}(职位名称,公司名称,工作地点,公司规模,工作经验,平均月薪,学历要求) VALUES(%s,%s,%s,%s,%s,%s,%s)'.format(table_name))
            insert_data = (job, company, address, develop, jingyan, salary, graduate)
            cursor.execute(insert_row, insert_data)
            conn.commit()
            with open('%s职位描述.txt' % (table_name), 'a', encoding='utf-8') as f:
                f.write(require)

    print('此页已抓取···')
    conn.close()

    count = 0
    while count <= 10:
        try:
            next_page = fox.find_element_by_class_name('pagesDown-pos').click()
            break
        except:
            time.sleep(8)
            count += 1
            continue

    if count > 10:
        fox.close()
    else:
        time.sleep(1)
        get_everypage_info(fox, keyword, city)


if __name__ == "__main__":
    citys = ['上海', '深圳', '广州', '武汉', '杭州', '南京', '成都', '青岛']   # '北京',   已爬取
    job = '数据挖掘分析'
    for city in citys:
        print(" ")
        get_main_page(job, city)

最后获取的输入如图