1 #!/usr/bin/python3 2 3 import pymysql 4 import datetime 5 import json 6 import random 7 import logging 8 9 logging.basicConfig(filename="test_ods_incre_hour.log",filemode="a", 10 format="%(asctime)s-%(funcName)s-%(lineno)d-%(levelname)s:%(message)s",level=logging.INFO) 11 console = logging.StreamHandler() 12 logging.getLogger().addHandler(console) 13 14 table_name = "liuyue_test_large_data" 15 cur_date = datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S") 16 17 def gen_rows(rowsnum, onenum=10,id1=0): 18 id1 = id1 if(id1) else 0 19 name1 = "liuyue_" 20 address1 = "xiangyang_" 21 band1 = "15_" 22 mony1 = 0.234 23 isnative = 1 24 secretkey = 0 25 createdate = cur_date 26 update_date = cur_date 27 special_char1 = r"P01~·！@#￥%……&*（）——+=-{}【】、|‘’“”；：？《》，。{}\n\r\n\t后面还有 数据的_" 28 description1 = "甄子丹版陈真,李小龙版陈真,都不如最后一版陈真最经典_" 29 test_null = 'null' 30 rows = [] 31 32 for i in range(1,rowsnum+1): 33 id = id1 + i 34 name = name1 + str(id) 35 address = address1+ str(id) 36 band = band1 + str(id) 37 mony = mony1 + id 38 isnative = random.choice([0,1]) 39 secretkey = random.randint(100000,999999) 40 special_char = special_char1 + str(id) 41 description = description1 + str(id) 42 row = (id,name,address,band,mony,isnative,secretkey,createdate,update_date,special_char,description,test_null) 43 # row = (id,name) 44 rows.append(row) 45 if(i % onenum == 0): 46 yield rows 47 rows = [] 48 yield rows 49 50 51 def execute_sql(sqlstr): 52 db = pymysql.connect("10.136.142.111","liumin1","liumin1","liumin1" ) 53 cursor = db.cursor() 54 data = None 55 try: 56 cursor.execute(sqlstr) 57 if(sqlstr.lower().startswith("select")): 58 data = cursor.fetchone() 59 db.commit() 60 except Exception as e: 61 logging.info(e) 62 logging.info("执行失败的SQL是:%s" % sqlstr) 63 db.rollback() 64 db.close() 65 return data if(data) else None 66 67 def get_mix_id(): 68 sqlstr = "select max(id) from %s"%table_name 69 res = execute_sql(sqlstr) 70 logging.info("当前数据库中最大id为:%s" % res[0]) 71 return res[0] 72 73 def insert_large_data(rowsnum, onenum=10): 74 db = pymysql.connect("10.136.142.111","liumin1","liumin1","liumin1" ) 75 cursor = db.cursor() 76 77 sqlstr = "insert into %s values " % table_name 78 id1 = get_mix_id() 79 n = 0 80 for i in gen_rows(rowsnum, onenum, id1): 81 if(not i): 82 continue 83 for j in i: 84 sql_tmp = json.dumps(j,ensure_ascii=False)[1:-1] 85 sqlstr = sqlstr + '(' + sql_tmp + ')' +',' 86 87 sqlstr = sqlstr[:-1] 88 # logging.info("生成的SQL是:%s" % sqlstr) 89 try: 90 cursor.execute(sqlstr) 91 except Exception as e: 92 logging.info(e) 93 # logging.info("执行失败的SQL是:%s" % sqlstr) 94 # db.rollback() 95 n = n +1 96 if(n % 10 == 0): 97 db.commit() 98 logging.info("提交第%d次成功，已经新增数据量:%d条" % (n//10, onenum*n)) 99 100 sqlstr = "insert into %s values " % table_name 101 db.commit() 102 db.close() 103 logging.info("插入sql成功，插入的记录数是:%d"%rowsnum) 104 105 106 def update_data(change_num): 107 max_id = get_mix_id() 108 for i in range(change_num): 109 random_id = random.randint(1,max_id) 110 sqlstr = 'update %s set description="update by autotest,update time is %s",update_date="%s" where id =%d;' %(table_name,cur_date,cur_date,random_id) 111 execute_sql(sqlstr) 112 logging.info("修改的表是:%s，修改的记录ID是:%d"%(table_name,random_id)) 113 114 115 # def main(): 116 # insert_large_data(3,2) 117 # update_data(3) 118 119 logging.info("开始任务...") 120 #每次更新记录数 121 # update_data(3) 122 #每新增记录数 123 insert_large_data(30000,5000) 124 logging.info("结束任务...\n")
不同插入数据库的方式的性能比较：
1、23秒，30000，一次插入5000条，插入多次（10），提交一次
2、37秒，30000，一次插入5000条，插入一次，提交一次
3、33分0秒，30000，一次插入1条，插入一次，提交一次