import numpy as np import pandas as pd import xlwt import xlrd from excelTools import ExcelT from pandas import DataFrame # from pandas.core.frame import DataFrame def makeExcel(ta_one,sh_one_li,ta_two,sh_two_li): for x in range(len(sh_one_li)): data_x = pd.read_excel(ta_one,sheet_name=sh_one_li[x])#pd.dataframe data_y = pd.read_excel(ta_two,sheet_name=sh_two_li[x]) train_data = np.array(data_x)#np.ndarray() train_data_list=train_data.tolist()#list train_company = np.array(data_y)#np.ndarray() train_company_list=train_company.tolist()#list fee = [] money = [] company = [] for name in train_company_list: company.append(name[0]) sum = 0 for li in train_data_list: # print(li[1],name) if li[1] == name[0]: sum = sum+li[4] fee.append(li[4]) # s = 0 # for t in fee: # s = s + t # print(s) # print(name[0],len(fee)) money.append(sum) dict_data = { 'company':company, 'money':money } data=DataFrame(dict_data)#将字典转换成为数据框 # DataFrame(data).to_excel('one(一).xlsx',sheet_name='1901') DataFrame(data).to_excel(sh_two_li[x]+'.xlsx',sheet_name=sh_two_li[x]) def dictChangeDataF(): a = [1, 2, 3, 4] # 列表a b = [5, 6, 7, 8] # 列表b c = {"a": a, "b": b} # 将列表a,b转换成字典 data = DataFrame(c) # 将字典转换成为数据框 DataFrame(data).to_excel('b.xlsx', sheet_name='测试') def endTable(): totalData = xlrd.open_workbook('final.xlsx') table = totalData.sheets()[0] data = table.col_values(0) workbook = xlwt.Workbook(encoding='utf-8') sheet = workbook.add_sheet('总表') sheet.write(0, 0, '公司名字') sheet.write(0, 1, '') sheet.write(0, 2, '') sheet.write(0, 3, '') sheet.write(0, 4, '') sheet.write(0, 5, '') sheet.write(0, 6, '') sheet.write(0, 7, '') sheet.write(0, 8, '') sheet.write(0, 9, '') sheet.write(0, 10, '') ta_name = ['four', 'five', 'six(二)', 'seven', 'eight', 'nine', 'ten', 'eleven', 'twelve', 'one(一)'] num = 0 company_count = 0 for na in ta_name: company_count += 1 count = 0 num += 1 for comname in data[1:]: count += 1 if num == 1: tableData = xlrd.open_workbook(na + '.xlsx') table = tableData.sheets()[0] nrows = table.nrows for x in range(1, nrows): data_row = table.row_values(x) if data_row[1] == comname: sheet.write(count, 0, comname) sheet.write(count, company_count, data_row[2]) data_col = table.col_values(1) if comname not in data_col: sheet.write(count, 0, comname) sheet.write(count, company_count, '') else: tableData = xlrd.open_workbook(na + '.xlsx') table = tableData.sheets()[0] nrows = table.nrows for x in range(1, nrows): data_row = table.row_values(x) if data_row[1] == comname: sheet.write(count, company_count, data_row[2]) workbook.save('结果(三).xls') if __name__ == '__main__': ta_one = '' ta_two = '' sh_one_li = [] sh_two_li = [] makeExcel('one(一)',1901) ===============================================================
# -*- coding:utf-8*- import time import requests time1=time.time() import pandas as pd import json import xlwt # workbook = xlwt.Workbook(encoding='utf-8') # sheet = workbook.add_sheet('黑名单') # sheet.write(0,0,'姓名') # sheet.write(0,1,'身份证号') ################定义数据结构列表存储数据#################### def getName(): iname=[] icard=[] ################循环发送请求解析数据####################### count = 0 for i in range(1,2): time.sleep(10) print('正在抓取第'+str(i)+"页.................................") url="https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=6899&query=%E8%80%81%E8%B5%96&pn="+str(i*10)+"&ie=utf-8&oe=utf-8&format=json" head={ "Host": "sp0.baidu.com", "Connection": "keep-alive", "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36", "Accept": "*/*", "Referer": "https://www.baidu.com/s?ie=utf-8&f=3&rsv_bp=1&tn=95943715_hao_pg&wd=%E8%80%81%E8%B5%96&oq=%25E8%2580%2581%25E8%25B5%2596&rsv_pq=ec5e631d0003d8eb&rsv_t=b295wWZB5DEWWt%2FICZvMsf2TZJVPmof2YpTR0MpCszb28dLtEQmdjyBEidZohtPIr%2FBmMrB3&rqlang=cn&rsv_enter=0&prefixsug=%25E8%2580%2581%25E8%25B5%2596&rsp=0&rsv_sug9=es_0_1&rsv_sug=9", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "zh-CN,zh;q=0.8" } html=requests.get(url,headers=head).content html_json=json.loads(html) html_data=html_json['data'] for each in html_data: count+=1 k=each['result'] for each in k: print(each['iname'],each['cardNum']) iname.append(each['iname']) icard.append(each['cardNum']) # sheet.write(count,0,each['iname']) # sheet.write(count,1,each['cardNum']) #####################将数据组织成数据框########################### data=pd.DataFrame({"name":iname,"IDCard":icard}) #################数据框去重#################################### data1=data.drop_duplicates() print(data1) # #########################写出数据到excel######################################### pd.DataFrame.to_excel(data1,"c.xlsx",header=True,encoding='gbk',index=False) # time2=time.time() # print u'ok,爬虫结束!' # print u'总共耗时:'+str(time2-time1)+'s' def getTel(): url = 'http://api.fxhyd.cn/appapi.aspx?callback=jQuery22308425623862046725_1550671913748&jsonp=MobileSeachJsonCallback&actionid=getmobile&token=01126710166b7d58cbf047af98f36a729bff30b6f801&itemid=352&province=0&city=0&isp=0&' res = requests.get(url).text print(res)
========================================================================
class ExcelT(object): def __init__(self): self.workbook = xlwt.Workbook(encoding='utf-8') self.sheet = self.workbook.add_sheet('sheet1') self.sheet.write(0, 0, 'A') self.sheet.write(0, 1, 'B') self.sheet.write(0, 2, 'C') self.sheet.write(0, 3, 'D') self.sheet.write(0, 4, 'E') self.sheet.write(0, 5, 'F') self.sheet.write(0, 6, 'G') self.sheet.write(0, 7, 'H') self.sheet.write(0, 8, 'I') self.sheet.write(0, 9, 'J') self.sheet.write(0, 10, 'K') def writeExcel(self,data_lis,*args): for idx,data_li in enumerate(data_lis): count = idx+1 self.sheet.write(count, 0, data_li[0]) self.sheet.write(count, 1, data_li[1]) self.sheet.write(count, 2, data_li[2]) self.sheet.write(count, 3, data_li[3]) self.sheet.write(count, 4, data_li[4]) self.sheet.write(count, 5, data_li[5]) self.sheet.write(count, 6, data_li[6]) self.sheet.write(count, 7, data_li[7]) self.sheet.write(count, 8, data_li[8]) self.sheet.write(count, 9, data_li[9]) self.sheet.write(count, 10, data_li[10]) def saveExcel(self,tableN): self.workbook.save(tableN+'.xls')