Python爬取网站数据并生成Excel

# -*- coding: utf-8 -*-
import os
import re
import sys
import ssl
import json
import time
import xlwt
import urllib
import urllib2
import shutil

from pyExcelerator import *

# 取消证书验证
context = ssl._create_unverified_context()

# 每页动态数量（最多60条）
perPage = 60;

# 文件存放路径
filePath = 'd:/Reptilian/health/xxxx/'

# 记录条数
count = 1

# 创建工作簿
import xlwt

filePath = 'd:/Reptilian/health/xxxx/'

workbook = xlwt.Workbook()
worksheet = workbook.add_sheet('sheet1')

# 爬虫基础地址，某站热门热门动态接口，打开网页通过开发者工具分析得来
xxxxUrl = 'https://api.xxxxx.com/social/v2/timeline/hot?startPage={}&perPage='+ str(perPage) +'&lastId=5aee63773c549f58fa1c3bb1'

# 请求头定义
headers = {
       'Connection': 'keep-alive',
       'Accept-Language': 'zh-CN,zh;q=0.9',
       'Accept':'text/html,application/xhtml+xml,application/xml;\
       q=0.9,image/webp,image/apng,*/*;q=0.8',
       'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 \
       (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36',
   }

# 获取用户信息
def getUserInfo(page):
   url = xxxxUrl.format(str(page))
   print(('爬取内容(每次' + str(perPage) + '条)地址：' + url).decode('utf-8').encode('gbk'))
   req = urllib2.Request(url, headers = headers)
   resp = urllib2.urlopen(req, context = context)
   result = resp.read().decode('utf-8');
   jsonData = json.loads(result)
   if jsonData['ok']:
       # 接口调用成功
       users = jsonData['data']
   else:
       # 接口调用失败
       print('接口调用失败！'.decode('utf-8').encode('gbk'));
       return
   global count
   if len(users) > 0:
       for user in users:
           if (getCorrectName(user['author']['username']) == ''):
               break
           # 拿上传图片列表
           if ('images' in user.keys()):
               imgs = user['images']
               imgNum = len(imgs)
               if imgNum > 1:
                   worksheet.write_merge(count, count + imgNum - 1, 0, 0, getCorrectName(user['author']['username']))
                   worksheet.write_merge(count, count + imgNum - 1 , 1, 1, user['author']['gender'])
                   worksheet.write_merge(count, count + imgNum - 1 , 2, 2, user['author']['avatar'])
                   worksheet.write_merge(count, count + imgNum - 1, 4, 4, getCorrectDate(str(user['created'])))
                   worksheet.write_merge(count, count + imgNum - 1, 5, 5, getCorrectContent(user['content']))
                   for img in imgs:
                       worksheet.write (count, 3, img)
                       count += 1
               elif imgNum == 1:
                   worksheet.write(count, 0, getCorrectName(user['author']['username']))
                   worksheet.write(count, 1, user['author']['gender'])
                   worksheet.write(count, 2, user['author']['avatar'])
                   worksheet.write(count, 3, imgs[0])
                   worksheet.write(count, 4, getCorrectDate(str(user['created'])))
                   worksheet.write(count, 5, getCorrectContent(user['content']))
                   count += 1
               else:
                   worksheet.write(count, 0, getCorrectName(user['author']['username']))
                   worksheet.write(count, 1, user['author']['gender'])
                   worksheet.write(count, 2, user['author']['avatar'])
                   worksheet.write(count, 3, user['photo'])
                   worksheet.write(count, 4, getCorrectDate(str(user['created'])))
                   worksheet.write(count, 5, getCorrectContent(user['content']))
                   count += 1
           else:
               worksheet.write(count, 0, getCorrectName(user['author']['username']))
               worksheet.write(count, 1, user['author']['gender'])
               worksheet.write(count, 2, user['author']['avatar'])
               worksheet.write(count, 3, '')
               worksheet.write(count, 4, getCorrectDate(str(user['created'])))
               worksheet.write(count, 5, getCorrectContent(user['content']))
               count += 1

# 创建制定目录
def mkDir(path):
   path = path.strip()
   path = path.rstrip("\\")
   isExist = os.path.exists(path)

   if not isExist:
       os.makedirs(path)
   else:
       print('目录已存在，不需要重复创建！'.decode('utf-8').encode('gbk'))

# 验证名称是否合法(只保留中文和英文)
def getCorrectName(name):
   temp = re.sub(r'([\u4e00-\u9fa5\u0030-\u0039\u0041-\u005a\u0061-\u007a])', '', name)
   #re.sub(r'(\\-)', '', temp)
   return temp

# 替换动态中的特殊字符
def getCorrectContent(content):
return re.sub(r'#|xxxx|XXXX|@', '', content)

# 获取正确格式时间
def getCorrectDate(time):
t = re.sub(r'T', ' ', time)
return re.sub(r'(.[0-9]*Z$)', '', t)

if __name__ == '__main__':
   # 创建目录
   mkDir(filePath)
   # 创建表格头部
   worksheet.write (0, 0, '用户昵称'.decode('utf-8').encode('gbk').decode('gbk'))
   worksheet.write (0, 1, '用户性别'.decode('utf-8').encode('gbk').decode('gbk'))
   worksheet.write (0, 2, '用户头像'.decode('utf-8').encode('gbk').decode('gbk'))
   worksheet.write (0, 3, '用户上传图片'.decode('utf-8').encode('gbk').decode('gbk'))
   worksheet.write (0, 4, '动态创建时间'.decode('utf-8').encode('gbk').decode('gbk'))
   worksheet.write (0, 5, '用户动态'.decode('utf-8').encode('gbk').decode('gbk'))
   # 获取用户信息并写入表格
   for i in range(0, 200):
       getUserInfo(i)
   # 保存xls文件
   workbook.save(filePath + 'content.xls')

Python爬取网站数据并生成Excel

猜你喜欢