To the database insert, update large amounts of data

 

  1 #!/usr/bin/python3
  2  
  3 import pymysql
  4 import datetime
  5 import json
  6 import random
  7 import logging
  8  
  9 logging.basicConfig(filename="test_ods_incre_hour.log",filemode="a",
 10                     format="%(asctime)s-%(funcName)s-%(lineno)d-%(levelname)s:%(message)s",level=logging.INFO)
 11 console = logging.StreamHandler()
 12 logging.getLogger().addHandler(console)
 13 
 14 table_name = "liuyue_test_large_data"
 15 cur_date = datetime.datetime.today().strftime("%Y-%m-%d %H:%M:%S")
 16 
 17 def gen_rows(rowsnum, onenum=10,id1=0):
 18     id1 = id1 if(id1) else 0
 19     name1 = "liuyue_"
 20     address1 = "xiangyang_"
 21     band1 = "15_"
 22     = 0.234 mony1
 23 is      isNative. 1 =
 24      SecretKey = 0
 25      CreateDate = cur_date
 26 is      UPDATE_DATE = cur_date
 27      special_char1 R & lt = " P01 ~ · @ # ¥% ...... & * () - + = - {} [], |! ' ' "";:? ". ", {} \ n \ r n \ t there _ \ following data " 
28      description1 = " Yen version of Chen, Lee version of Chen, not as the final version of Chen most classic _ " 
29      = test_null ' null ' 
30      rows = []
 31 is      
32      for I in Range (. 1, rowsnum +. 1 ):
 33 is         id = id1 + i
 34         name = name1 + str(id)
 35         address = address1+ str(id)
 36         band = band1 + str(id)
 37         mony = mony1 + id
 38         isnative = random.choice([0,1])
 39         secretkey = random.randint(100000,999999)
 40         special_char = special_char1 + str(id)
 41         description = description1 + str(id)
 42         row = (id,name,address,band,mony,isnative,secretkey,createdate,update_date,special_char,description,test_null)
 43         # row = (id,name)
 44         rows.append(row)
 45         if(i % onenum == 0):
 46             yield rows
 47             rows = []
 48     yield rows
 49 
 50 
 51 def execute_sql(sqlstr):
 52     db = pymysql.connect("10.136.142.111","liumin1","liumin1","liumin1" )
 53     cursor = db.cursor()
 54     data = None
 55     try:
 56         cursor.execute(sqlstr)
 57         if(sqlstr.lower().startswith("select")):
 58             data = cursor.fetchone()
 59         db.commit()
 60     except Exception as e:
 61         logging.info(e)
 62         logging.info("执行失败的SQL是:%s" % sqlstr)
 63         db.rollback()
 64     db.close()
 65     return data if(data) else None
 66 
 67 def get_mix_id():
 68     sqlstr = "select max(id) from %s"%table_name
 69     res = execute_sql(sqlstr)
 70     logging.info("当前数据库中最大id为:%s" % res[0])
 71     return res[0]
 72 
 73 def insert_large_data(rowsnum, onenum=10):
 74     db = pymysql.connect("10.136.142.111","liumin1","liumin1","liumin1" )
 75     cursor = db.cursor()
 76 
 77     sqlstr = "insert into %s values " % table_name
 78     id1 = get_mix_id()
 79     n = 0
 80     for i in gen_rows(rowsnum, onenum, id1):
 81         if(not i):
 82             continue
 83         for j in i:
 84             sql_tmp = json.dumps(j,ensure_ascii=False)[1:-1]
85              sqlstr sqlstr + = ' ( ' + sql_tmp + ' ) ' + ' , ' 
86          
87          sqlstr sqlstr = [: -. 1 ]
 88          # logging.info ( "SQL is generated:% S" sqlstr%) 
89          the try :
 90              the cursor.execute (sqlstr)    
 91 is          the except Exception AS E:
 92              logging.info (E)
 93              # logging.info ( "SQL execution is failed:% S" sqlstr%) 
94              # db.rollback () 
95          n-n-+ = 1
 96          IF(n-% 10 == 0):
 97              the db.commit ()
 98              logging.info ( " submitted% d successful, the amount of data has been added:% d article " % (n-10 //, onenum * n-))
 99          
100          sqlstr = " iNSERT INTO% S values " % table_name
 101      the db.commit ()
 102      db.Close ()
 103      logging.info ( " insert sql success, the number of records are inserted: D% " % rowsnum)
 104  
105  
106  DEF update_data (change_num):
 107      max_id = get_mix_id ()
 108      for i in range(change_num):
109         random_id = random.randint(1,max_id)
110         sqlstr =  'update %s set description="update by autotest,update time is %s",update_date="%s"  where id =%d;' %(table_name,cur_date,cur_date,random_id)
111         execute_sql(sqlstr)
112         logging.info("修改的表是:%s，修改的记录ID是:%d"%(table_name,random_id))
113     
114 
115 # def main():
116     # insert_large_data(3,2)
117     # update_data(3)
118 
119 logging.info ( " Start Task ... " )
 120  # each update record number 
121  # update_data (. 3) 
122  # each additional record number 
123 insert_large_data (30000,5000 )
 124 logging.info ( " end task .. . \ the n- " ) 

performance into the database of the different ways of comparison:

1,23 seconds, 30,000, the first insert 5000, inserting a plurality of times (10), submit a
2,37 seconds, 30,000, the first insert 5000, inserting a submission to the next
3, 33 minutes, 0 seconds, 30,000, the first insert 1 strip, insert a submission to the next