python3爬取教务系统的个人学期课程表（无头谷歌浏览模拟登录）

前言

今天带来的是与上次爬取教务系统获取成绩单的姐妹版——爬取教务个人的学期课程表。

工具

使用pycharm编辑器，安装selenium库，beautifulsoup库，csv库，当然需要下载对应的chromedriver 版本对应下载链接点击下载

接入需要的接口

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.select import Select
from bs4 import BeautifulSoup
import time
import csv

设置参数

#模拟登陆终端文件路径
driver_path = r'E:\py\chromedriver\chromedriver.exe'

#生成csv文件路径
csv_file_path = r'E://py//个人学期课程表.csv'

#登录教务系统的账号与密码
UserId = '账号' 
PassWord = '密码' 

#实现后台登陆浏览器
chrome_options = Options()
chrome_options.add_argument('--disable-gpu') #关闭GPU对网页的加速，防止网页过多卡住
chrome_options.add_argument('--headless')#进行无头后台运行浏览器
#更改代理ip防反爬虫
#chrome_options.add_argument(f"--proxy-server=121.52.208.200:808")

driver = webdriver.Chrome(executable_path = driver_path, chrome_options=chrome_options)

模拟登录

模拟登录，然后用driver的find_element_by_× 寻找对应输入框的位置输入账号密码，点击登录，页面转换一系列操作，driver.implicitly_wait(1)是防止页面加载过慢，导致直接爬取下一步出错，具体可自行百度

driver.get('http://jwgln.zsc.edu.cn/jsxsd/')
driver.implicitly_wait(1)

#输入登录账号
try:
    driver.find_element_by_id("userAccount").send_keys(UserId)
    print('输入账号成功!')
except:
    print('输入账号失败!')
# 输入登录密码
try:
    driver.find_element_by_id("userPassword").send_keys(PassWord)
    print('输入密码成功!')
except:
    print('输入密码失败!')
# 点击登录
try:
    driver.find_element_by_xpath('//*[@id="btnSubmit"]').click()  # 用click模拟浏览器点击
    print('正在登录...')
except:
    print('登录失败!')
driver.implicitly_wait(1)
if '用户名或密码错误' in driver.page_source:
    print('登录失败，用户名或密码错误，请查证账号密码是否准确。')
    exit(0)
else:
    print('登录成功！')
# 点击学业情况
try:
    driver.find_element_by_xpath('//*[@class="block4"]').click()
    print('点击培养方案成功!')
except:
    print('点击培养方案失败!')
driver.implicitly_wait(1)

#点击课程成绩查询
try:
    driver.find_element_by_xpath('//*[@href="/jsxsd/xskb/xskb_list.do"]').click()
    time.sleep(1)
    print('我的学期课表点击成功!')
except:
    print('我的学期课表点击失败!')

接着是难点

点击学期课表 默认是没有课表的日期 会出现alert弹窗 需要模拟点击

#页面弹窗点击确认
try:
    alt = driver.switch_to_alert()
    alt.accept()
except:
    pass

接着是下拉框选择日期

检查页面元素

使用Select选择到自己所需的日期，点击，代码如下

#选择准确学期
#driver.find_element_by_xpath(".//*[@id='xnxq01id']/option[4]").click()
# 若此时点击后，下拉选项未收回，可点击整个下拉框，收回下拉选项
#driver.find_element_by_xpath(".//*[@id='nr']").click()
try:
    Select(driver.find_element_by_id("xnxq01id")).select_by_value("2019-2020-2").click()
except:
    pass

数据整理

使用的是beautifulsoup来筛选所需要的数据

soup = BeautifulSoup(driver.page_source,'lxml')
page = soup.find_all('div',attrs={'class': "kbcontent"})
teachers1,teachers2 = [],[]
weeks1,weeks2= [],[]
classrooms1,classrooms2= [],[]
for i in page:
    teachers1.append(i.find('font',attrs={'title':'老师'}))
    weeks1.append(i.find('font',attrs={'title':'周次(节次)'}))
    classrooms1.append(i.find('font', attrs={'title': '教室'}))
my_detail = list(page)
for i in teachers1:
    if i == None:
        teachers2.append('\n')
    else:
        teachers2.append(i.string)
for i in weeks1:
    if i == None:
        weeks2.append('\n')
    else:
        weeks2.append('\n'+i.string)
for i in classrooms1:
    if i == None:
        classrooms2.append('\n')
    else:
        classrooms2.append('\n'+i.string)
all_data = []
pitch_number = ['（上午）\n第1，2节\n(08:00-08:45)\n(08:55-09:40)','第3，4节\n(10:00-10:45)\n(10:55-11:40)',
                '（下午）\n第5，6节\n(14:30-15:15)\n(15:25-16:10)','第7，8节\n(16:20-16:05)\n(17:15-18:00)',
                '（晚上）\n第9，10节\n(19:30-20:15)\n(20:25-21:10)','第11，12节','第13，14节']

temp = []
temp.append(pitch_number[0])
num = 0
pnum = 0
for i in range(len(my_detail)):
    if my_detail[i].text == '\xa0':
        temp.append('\n\n\n')
    else:
       temp.append(my_detail[i].text.split(teachers2[i])[0]+'\n'+teachers2[i]+weeks2[i]+classrooms2[i])
    num = num + 1
    if num == 7:
        all_data.append(temp)
        temp = []
        pnum = pnum + 1
        temp.append(pitch_number[pnum])
        num = 0
page2 = soup.find('td',attrs={'colspan':"7"})
BZ = ['备注:'+page2.text,'\n','\n','\n','\n','\n','\n','\n']
all_data.append(BZ)

生成文件

f = open(csv_file_path, 'w', newline='')
csv_write = csv.writer(f)
csv_write.writerow(['课程时间','星期一','星期二','星期三','星期四','星期五','星期六','星期日'])
#设置列表的头部，就是各个列的列名
#再按列的长度每次输入一个列表进去生成一行
for i in range(len(all_data)):
    csv_write.writerow(all_data[i])
f.close()
print('生成csv文件成功')

全部代码

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.select import Select
from bs4 import BeautifulSoup
import time
import csv
'''
@author Himit_ZH
Date:2020.01.20
'''
#模拟登陆终端文件路径
driver_path = r'E:\py\chromedriver\chromedriver.exe'

#生成csv文件路径
csv_file_path = r'E://py//个人学期课程表.csv'

#登录教务系统的账号与密码
UserId = '账号' 
PassWord = '密码' 

#实现后台登陆浏览器
chrome_options = Options()
chrome_options.add_argument('--disable-gpu') #关闭GPU对网页的加速，防止网页过多卡住
chrome_options.add_argument('--headless')#进行无头后台运行浏览器
#更改代理ip防反爬虫
#chrome_options.add_argument(f"--proxy-server=121.52.208.200:808")

driver = webdriver.Chrome(executable_path = driver_path, chrome_options=chrome_options)

driver.get('http://jwgln.zsc.edu.cn/jsxsd/')
driver.implicitly_wait(1)

#输入登录账号
try:
    driver.find_element_by_id("userAccount").send_keys(UserId)
    print('输入账号成功!')
except:
    print('输入账号失败!')
# 输入登录密码
try:
    driver.find_element_by_id("userPassword").send_keys(PassWord)
    print('输入密码成功!')
except:
    print('输入密码失败!')
# 点击登录
try:
    driver.find_element_by_xpath('//*[@id="btnSubmit"]').click()  # 用click模拟浏览器点击
    print('正在登录...')
except:
    print('登录失败!')
driver.implicitly_wait(1)
if '用户名或密码错误' in driver.page_source:
    print('登录失败，用户名或密码错误，请查证账号密码是否准确。')
    exit(0)
else:
    print('登录成功！')
# 点击学业情况
try:
    driver.find_element_by_xpath('//*[@class="block4"]').click()
    print('点击培养方案成功!')
except:
    print('点击培养方案失败!')
driver.implicitly_wait(1)

#点击课程成绩查询
try:
    driver.find_element_by_xpath('//*[@href="/jsxsd/xskb/xskb_list.do"]').click()
    time.sleep(1)
    print('我的学期课表点击成功!')
except:
    print('我的学期课表点击失败!')

#页面弹窗点击确认
try:
    alt = driver.switch_to_alert()
    alt.accept()
except:
    pass
#选择准确学期
#driver.find_element_by_xpath(".//*[@id='xnxq01id']/option[4]").click()
# 若此时点击后，下拉选项未收回，可点击整个下拉框，收回下拉选项
#driver.find_element_by_xpath(".//*[@id='nr']").click()
try:
    Select(driver.find_element_by_id("xnxq01id")).select_by_value("2019-2020-2").click()
except:
    pass
print('开始进行数据整理')
#对获取的数据进行整理
soup = BeautifulSoup(driver.page_source,'lxml')
page = soup.find_all('div',attrs={'class': "kbcontent"})
teachers1,teachers2 = [],[]
weeks1,weeks2= [],[]
classrooms1,classrooms2= [],[]
for i in page:
    teachers1.append(i.find('font',attrs={'title':'老师'}))
    weeks1.append(i.find('font',attrs={'title':'周次(节次)'}))
    classrooms1.append(i.find('font', attrs={'title': '教室'}))
my_detail = list(page)
for i in teachers1:
    if i == None:
        teachers2.append('\n')
    else:
        teachers2.append(i.string)
for i in weeks1:
    if i == None:
        weeks2.append('\n')
    else:
        weeks2.append('\n'+i.string)
for i in classrooms1:
    if i == None:
        classrooms2.append('\n')
    else:
        classrooms2.append('\n'+i.string)
all_data = []
pitch_number = ['（上午）\n第1，2节\n(08:00-08:45)\n(08:55-09:40)','第3，4节\n(10:00-10:45)\n(10:55-11:40)',
                '（下午）\n第5，6节\n(14:30-15:15)\n(15:25-16:10)','第7，8节\n(16:20-16:05)\n(17:15-18:00)',
                '（晚上）\n第9，10节\n(19:30-20:15)\n(20:25-21:10)','第11，12节','第13，14节']

temp = []
temp.append(pitch_number[0])
num = 0
pnum = 0
for i in range(len(my_detail)):
    if my_detail[i].text == '\xa0':
        temp.append('\n\n\n')
    else:
       temp.append(my_detail[i].text.split(teachers2[i])[0]+'\n'+teachers2[i]+weeks2[i]+classrooms2[i])
    num = num + 1
    if num == 7:
        all_data.append(temp)
        temp = []
        pnum = pnum + 1
        temp.append(pitch_number[pnum])
        num = 0
page2 = soup.find('td',attrs={'colspan':"7"})
BZ = ['备注:'+page2.text,'\n','\n','\n','\n','\n','\n','\n']
all_data.append(BZ)
f = open(csv_file_path, 'w', newline='')
csv_write = csv.writer(f)
csv_write.writerow(['课程时间','星期一','星期二','星期三','星期四','星期五','星期六','星期日'])
for i in range(len(all_data)):
    csv_write.writerow(all_data[i])
f.close()
print('生成csv文件成功')
driver.close()
driver.quit()