python爬取b站排行榜

import requests
import re
from bs4 import BeautifulSoup
import pymysql
import traceback

class Spider:#常用的爬取方法的简单封装
    def __init__(self,url):
        self.url=url
        
    def getHTML(self):#获取html的对应代码
        headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.103 Safari/537.36'}
        try:
            response=requests.get(url=self.url,headers=headers,timeout=20)
            response.raise_for_status()
            response.encoding=response.apparent_encoding
            return response.text
        except:
            return "网页访问失败"
        
        
    def setSoup(self):#获取soup对象
        html=self.getHTML()
        self.soup=BeautifulSoup(html,'html.parser')
    
    def findTag(self,tagName):#按照标签名查找标签
        return self.soup.find_all(tagName)
    
    def findTagByAttrs(self,tagName,attrs):
        return self.soup.find_all(tagName,attrs)

    
    def getBeautifyHTML(self):
        return self.soup.prettify()


def getPage(url):#爬取单个页面,核心代码
    spider=Spider(url)
    spider.setSoup()
    itemList=spider.findTagByAttrs('li','rank-item')
    pageContentList=[]
    for item in itemList:
        pageContentItem=[]
        for title in item.find_all('a','title'):
            pageContentItem.append(title.string)
            # print(title.string)
            
        for playnum in item.find_all('span','data-box'):
            pattern=r">([^<]+)<"
            n=re.findall(pattern,playnum.__str__())[0]
            pageContentItem.append(n)
            # print(n)
        
            
        # print(item.find_all('div','pts')[0].div.string)
        pageContentItem.append(item.find_all('div','pts')[0].div.string)
        pageContentList.append(pageContentItem)
    return pageContentList     


def getURLFormBilibili():# 获取各种各样排行的榜单的信息
    date={
        1:'日排行',
        3:'三日排行',
        7:'周排行',
        30:'月排行'
    }
    areatype={
        0:'全站',
        1:'动画',
        168:'国漫相关',
        3:'音乐',
        129:'舞蹈',
        4:'游戏',
        36:'科技',
        188:'数码',
        160:'生活',
        119:'鬼畜',
        155:'时尚',
        5:'娱乐',
        181:'影视'
    }
    ranktype={
       'all':'全站',
       'origin':'原创'
    }
    submit={
        '0':'全部投稿',
        '1':'近期投稿'
    }
    urlDict={}#存放相应url的字典
    for ranktypeItem in ranktype.keys():
        for areatypeItem in areatype.keys():
            for submitItem in submit.keys():
                for dateTypeItem in date.keys():
                    title=ranktype[ranktypeItem]+'_'+areatype[areatypeItem]+'_'+submit[submitItem]+'_'+date[dateTypeItem]
                    destinaTionUrl='https://www.bilibili.com/ranking/{}/{}/{}/{}'.format(ranktypeItem,areatypeItem,submitItem,dateTypeItem)
                    urlDict[title]=destinaTionUrl
    return urlDict

    
    
class MysqlConnect:#数据库的连接类
    def __init__(self):
        pass
    
    def getConnect(self):
        db=coon = pymysql.connect(
            host = 'localhost',user = 'root',passwd = '19990614kang',
            port = 3306,db = 'bilibilirank',charset = 'utf8'
             #port必须写int类型
             #charset必须写utf8，不能写utf-8
        )
        return db
        
    def insertInfo(self,sql):
        db=self.getConnect()
        cursor=db.cursor()
        try:
            cursor.execute(sql)
            db.commit()
            print("sucessed...")
        except:
            print("failed...")
            db.rollback()
            
    
    def queryOutCome(self,sql):
         # 获取数据库连接  
        db = self.getConnect()  
        # 使用cursor() 方法创建一个游标对象 cursor  
        cursor = db.cursor()  
        try:  
            # 执行sql语句  
            cursor.execute(sql)  
            result = cursor.fetchone()  
        except: #方法二：采用traceback模块查看异常  
            #输出异常信息  
            traceback.print_exc()  
            # 如果发生异常，则回滚  
            db.rollback()  
        finally:  
            # 最终关闭数据库连接  
            db.close()  
        return result  
    
    def getCreateTableSql(self,tableName):#获取创建表的sql语句
        sql='''
        create table `{}` (
            id int(11) auto_increment primary key,
            title char(100) NOT NULL UNIQUE,
            playnum char(100) NOT NULL,
            commentnum char(100) NOT NULL,
            author char(100) NOT NULL,
            score char(100) NOT NULL
        )ENGINE=innodb DEFAULT CHARSET=utf8;
        '''.format(tableName)
        return sql
        
    def getInsertToTableSql(self,tableName,title,playnum,commentnum,author,score):
        sql='''
        insert into `{}` values(null,'{}','{}','{}','{}','{}');
        '''.format(tableName,title,playnum,commentnum,author,score)
        return sql
        
        
    
    def createTable(self,tableName,sql):
        db=self.getConnect()
        cursor=db.cursor()
        cursor.execute("drop table if exists %s" %(tableName)) 
        cursor.execute(sql)
        db.close()


if __name__ == "__main__":
    #开始爬取数据
    urlDict=getURLFormBilibili()#获取对应的URL信息
    mysqlconnect=MysqlConnect()#用于连接数据库
    
    for urlName in urlDict:
        print("正在处理"+urlName+"页面...")
        url=urlDict[urlName]
        tableName=urlName
        createsql=mysqlconnect.getCreateTableSql(tableName)
        mysqlconnect.createTable(tableName,createsql)
        pageList=getPage(url)
        for contentItem in pageList:
            insertsql=mysqlconnect.getInsertToTableSql(tableName,contentItem[0],contentItem[1],contentItem[2],contentItem[3],contentItem[4])
            print(insertsql)
            mysqlconnect.insertInfo(insertsql)