Rastreador de Python para obtener información académica de la Academia China de Ingeniería

import re
import os
import os.path
import time
from urllib.request import urlopen

dstDir = 'YuanShi'
if not os.path.isdir(dstDir):
    os.mkdir(dstDir)

startUrl = r'http://www.cae.cn/cae/html/main/col48/column_48_1.html'
with urlopen(startUrl) as fp:
    content = fp.read().decode()

# 提取并遍历每位大牛链接
pattern = r'<li class="name_list"><a href="(.+)" target="_blank">(.+)</a></li>'
result = re.findall(pattern, content)
for item in result:
    perUrl, name = item
    # 测试是否获取信息
    print(perUrl)
    # 这里根据初爬结果进行改进
    name = name.replace('<h3>', '').replace('</h3>', '')
    name = os.path.join(dstDir, name)
    perUrl = r'http://www.cae.cn/' + perUrl
    with urlopen(perUrl) as fp:
        content = fp.read().decode()

    # 抓取简介
    pattern = r'<p>(.+?)</p>'
    result = re.findall(pattern, content)  # 返回string中所有与pattern匹配的全部字符串,返回形式为数组。
    if result:
        intro = re.sub('(<a.+</a>)|(&ensp;)|(&nbsp);','','\n'.join(result))
        with open(name+'.txt', 'w', encoding='utf8') as fp:
            fp.write(intro)