Python爬取网站上的内链和外链

#%%
# -*- coding: utf-8 -*  
#__author__ = 'Administrat  
#coding=utf-8  
from urllib.request import urlopen  
from urllib.parse import urlparse  
from bs4 import BeautifulSoup  
import re  
import datetime  
import random  
from urllib  import request  
  
pages = set()  
random.seed(datetime.datetime.now())  
  
#sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='gb18030')  
headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}  
  
  
#获取页面所有内链的列表  
def getInternalLinks(bsObj, includeUrl):  
    includeUrl = urlparse(includeUrl).scheme+"://"+urlparse(includeUrl).netloc  
    internalLinks = []  
    #找出所有以“/”开头的链接  
    for link in bsObj.findAll("a", href=re.compile("^(/|.*"+includeUrl+")")):  
        if link.attrs['href'] is not None:  
            if link.attrs['href'] not in internalLinks:  
                if(link.attrs['href'].startswith("/")):  
                    internalLinks.append(includeUrl+link.attrs['href'])  
                else:  
                    internalLinks.append(link.attrs['href'])  
    return internalLinks  
  
#获取页面所有外链的列表  
def getExternalLinks(bsObj, excludeUrl):  
    externalLinks = []  
    #找出所有以“http”或者“www”开头且不包含当前URL的链接  
    for link in bsObj.findAll("a", href=re.compile("^(http|www)((?!"+excludeUrl+").)*$")):  
        if link.attrs['href'] is not None:  
            if link.attrs['href'] not in externalLinks:  
                externalLinks.append(link.attrs['href'])  
    return externalLinks  
  
  
def getRandomExternalLink(startingPage):  
    req=request.Request(startingPage,headers=headers)  
    html=urlopen(req)  
    bsObj=BeautifulSoup(html.read(),"html.parser")  
    externalLinks = getExternalLinks(bsObj, urlparse(startingPage).netloc)  
    if len(externalLinks) == 0:  
        print("没有外部链接，准备遍历整个网站")  
        domain = urlparse(startingPage).scheme+"://"+urlparse(startingPage).netloc  
        internalLinks = getInternalLinks(bsObj, domain)  
        return getRandomExternalLink(internalLinks[random.randint(0,len(internalLinks)-1)])  
    else:  
        return externalLinks[random.randint(0, len(externalLinks)-1)]  
  
def followExternalOnly(startingSite):  
    externalLink = getRandomExternalLink(startingSite)  
    print("随机外链是: "+externalLink)  
    followExternalOnly(externalLink)  
  
#收集网站上发现的所有外链列表  
allExtLinks = set()  
allIntLinks = set()  
def getAllExternalLinks(siteUrl):  
    domain = urlparse(siteUrl).scheme+"://"+urlparse(siteUrl).netloc  
    internalLinks = getInternalLinks(bsObj,domain) 
    externalLinks = getExternalLinks(bsObj,domain)  
    
    f = open('result.txt','w')
    #收集外链  
    for link in externalLinks:  
        if link not in allExtLinks:  
            allExtLinks.add(link)  
            #print(link)
            f.writelines(link+'\n')
            print("即将获取的外部链接的URL是："+link)  
    #收集内链  
    for link in internalLinks:  
        if link not in allIntLinks:  
            print("即将获取内部链接的URL是："+link)
            allIntLinks.add(link)  
            getAllExternalLinks(link)
            f.writelines(link+'\n')
  
#followExternalOnly("http://bbs.3s001.com/forum-36-1.html")  
#allIntLinks.add("http://bbs.3s001.com/forum-36-1.html")  
getAllExternalLinks("http://www.zzu.edu.cn/")