数据归档主要用于历史数据的备份
1 #!/usr/bin/env python 2 # -*- encoding: utf8 -*- 3 4 import os, subprocess 5 import datetime 6 import dbconfig 7 from dbhelper import DBHelper 8 from dateutil.relativedelta import relativedelta 9 10 # 格式化时间 11 def dateformat(datestr, delmonth): 12 date = datetime.datetime.strptime(datestr, '%Y-%m-%d') 13 date = date + relativedelta(months=-int(delmonth)) 14 partition_name = 'p' + str(date.strftime('%Y%m')) 15 date = str(date.strftime('%Y-%m-01')) 16 return partition_name, date 17 18 # 写入日志 19 def archivelog(server_dns, db_name, table_name, delpartition, del_num, loginfo, status): 20 logsql = u"""insert into mywebtoolsdb.cron_archive_log(create_user,update_user,dns,dbname,tablename,partition_name,partition_count,loginfo,status) 21 VALUES ('archive_py','archive_py','%(server_dns)s','%(db_name)s','%(table_name)s','%(delpartition)s','%(del_num)s',"%(loginfo)s","%(status)s")""" 22 dblogconn = DBHelper(dbconfig.connectshort(keyname="config_0_152")) 23 logsql = logsql % {'server_dns': server_dns, 'db_name': db_name, 'table_name': table_name, 24 'delpartition': delpartition, 'del_num': del_num, 'loginfo': loginfo, 'status': status} 25 dblogconn.execute(1, logsql) 26 27 28 # 配置更新 29 def configchange(server_dns, db_name, table_name): 30 dbcfgconn = DBHelper(dbconfig.connectshort(keyname="config_0_152")) 31 sql = "update mywebtoolsdb.cron_archive_config set status =1 where status=0 and dns = '%(server_dns)s' and dbname = '%(db_name)s' and tablename = '%(table_name)s'" \ 32 % {'db_name': db_name, 'table_name': table_name, 'server_dns': server_dns} 33 results = dbcfgconn.execute(1, sql) 34 35 backup_path = "/data1/zyq/archive/" #备份数据路径 36 # backup_path = "E:\\temp\\" 37 38 # 数据库备份账号密码 39 mysql_user = "root" 40 mysql_pwd = "xxxxxx" 41 42 mysql_exec = "mysql -u" + mysql_user + " -p" + mysql_pwd 43 mysql_dump = "mysqldump -u" + mysql_user + " -p" + mysql_pwd 44 45 # 获取归档信息 46 dbcfgconn = DBHelper(dbconfig.connectshort(keyname="config_0_152")) 47 sql = "SELECT dns,dns_dump,tidb_info,dbname,tablename,remain_column,remain_month,remain_type FROM mywebtoolsdb.cron_archive_config WHERE STATUS='0' " 48 results = dbcfgconn.querydic(1, sql) 49 50 for data in results: 51 # 参数初始化 52 server_dns = data['dns'] 53 server = data['dns'].split(',') 54 server_ip = server[0] 55 server_port = server[1] 56 dump_server = data['dns_dump'].split(',') 57 dump_server_ip = dump_server[0] 58 dump_server_port = dump_server[1] 59 tidb_info = data['tidb_info'].split(',') 60 tidb_server_ip = tidb_info[0] 61 tidb_server_port = tidb_info[1] 62 tidb_db_name = tidb_info[2] 63 tidb_table_name = data['tablename'] 64 db_name = data['dbname'] 65 table_name = data['tablename'] 66 remain_month = data['remain_month'] 67 remain_column = data['remain_column'] 68 remain_type = data['remain_type'] 69 70 deldate = str(datetime.datetime.now())[0:10] 71 delinfo = dateformat(deldate, remain_month) 72 delpartition = delinfo[0] 73 deltime = delinfo[1] 74 75 # 备份路径初始化 76 backup_path = backup_path + delpartition.replace('p', '') + "/" 77 if not os.path.exists(backup_path): 78 os.makedirs(backup_path) 79 80 # 生成备份文件名 81 backup_filename = backup_path + db_name + "_" + table_name + "_" + delpartition + ".sql" 82 backup_tarname = backup_path + db_name + "_" + table_name + "_" + delpartition + ".tar.gz" 83 84 # mysqldump命令初始化--只能按照每月 85 mysql_dump = mysql_dump + " -h " + dump_server_ip + " -P" + dump_server_port + " " + db_name + " " + table_name + \ 86 " --where=\"" + remain_column + ">= DATE_ADD('" + deltime + "',INTERVAL -1 MONTH) and " + remain_column + "<'" + deltime + "'\"" + \ 87 " --default-character-set=UTF8 --replace --no-create-db=TRUE --no-create-info=TRUE --add-drop-table=false >" + \ 88 backup_filename 89 90 # 执行sql命令初始化 91 mysql_exec = mysql_exec + " -h " + tidb_server_ip + " -P" + tidb_server_port + " " + tidb_db_name + "<" + backup_filename 92 93 # tar命令初始化 94 filetar = "tar czf "+ backup_tarname + " " + backup_filename + " --remove-files" 95 96 # SQL脚本初始化 97 delsql = "select count(*),max(id) from %(db_name)s.%(table_name)s where %(remain_column)s>=DATE_ADD('%(deltime)s',INTERVAL -1 MONTH) and %(remain_column)s < '%(deltime)s'" \ 98 % {'db_name': db_name, 'table_name': table_name, 'remain_column': remain_column, 'deltime': deltime, } 99 100 tidbsql = "select count(*),max(id) from %(tidb_db_name)s.%(table_name)s where %(remain_column)s>=DATE_ADD('%(deltime)s',INTERVAL -1 MONTH) and %(remain_column)s < '%(deltime)s'" \ 101 % {'tidb_db_name': tidb_db_name, 'table_name': table_name, 'remain_column': remain_column, 102 'deltime': deltime, } 103 104 105 106 # 判断归档类型进行操作 107 if remain_type == '2': 108 del_num = 0 109 try: 110 # 获取总删除行数 111 dbconn = DBHelper(dbconfig.connectprdmysql(dns=dump_server_ip, port=dump_server_port)) 112 results = dbconn.query(1, delsql) 113 del_num = results[0][0] 114 # dump数据 115 os.system(mysql_dump) 116 # dump数据导入归档环境 117 os.system(mysql_exec) 118 119 # 获取归档环境导入总数 120 dbconn = DBHelper(dbconfig.connectprdmysql(dns=tidb_server_ip, port=tidb_server_port)) 121 results = dbconn.query(1, tidbsql) 122 check_num = results[0][0] 123 124 # 判断总行数及归档环境总数，如果一致则删除源环境分区 125 if del_num == check_num: 126 os.system(filetar) 127 dbconn = DBHelper(dbconfig.connectprdmysql_releasa(dns=server_ip, port=server_port)) 128 sql = "alter table %(db_name)s.%(table_name)s drop partition %(delpartition)s;" % {'db_name': db_name, 129 'table_name': table_name, 130 'delpartition': delpartition} 131 dbconn.execute(1, sql) 132 else: 133 raise Exception("dump num don`t match tidb num!") 134 135 # 写入日志 136 archivelog(server_dns= server_dns, db_name= db_name, table_name=table_name, delpartition= delpartition, del_num= del_num, loginfo= 'Finished', status= '1') 137 138 # 配置表更新状态 139 configchange(server_dns=server_dns, db_name=db_name, table_name=table_name) 140 141 # 异常处理 142 except Exception, e: 143 archivelog(server_dns= server_dns, db_name= db_name, table_name=table_name, delpartition= delpartition, del_num= del_num, loginfo= e, status= '2') 144 145 if remain_type == '3': 146 del_num = 0 147 try: 148 # 获取删除数的最大ID 149 dbconn = DBHelper(dbconfig.connectprdmysql(dns=dump_server_ip, port=dump_server_port)) 150 results = dbconn.query(1, delsql) 151 del_num = results[0][0] 152 del_maxid = results[0][1] 153 # dump数据 154 os.system(mysql_dump) 155 # 获取文件中最大ID 156 maxid_check = """tail -n 14 %(backup_filename)s | head -n 1 |awk -F "[(]" '{ print $NF}' | awk -F "," '{ print $1}'""" % { 157 'backup_filename': backup_filename} 158 p = subprocess.Popen(maxid_check, shell=True, stdout=subprocess.PIPE) 159 out = p.stdout.readlines() 160 for line in out: 161 maxid = line.strip() 162 # 判断最大id及归文件最大id，如果一致则删除源环境分区 163 if str(del_maxid) == str(maxid.replace("'", "")): 164 os.system(filetar) 165 dbconn = DBHelper(dbconfig.connectprdmysql_releasa(dns=server_ip, port=server_port)) 166 167 sql = "alter table %(db_name)s.%(table_name)s drop partition %(delpartition)s;" % {'db_name': db_name, 168 'table_name': table_name, 169 'delpartition': delpartition} 170 dbconn.execute(1, sql) 171 else: 172 raise Exception("dump maxid don`t match file maxid!") 173 # 写入日志 174 archivelog(server_dns=server_dns, db_name=db_name, table_name=table_name, delpartition=delpartition, del_num=del_num, loginfo='Finished', status='1') 175 176 # 配置表更新状态 177 configchange(server_dns=server_dns, db_name=db_name, table_name=table_name) 178 179 # 异常处理 180 except Exception, e: 181 archivelog(server_dns= server_dns, db_name= db_name, table_name=table_name, delpartition= delpartition, del_num= del_num, loginfo= e, status= '2') 182 183 if remain_type == '4': 184 del_num = 0 185 try: 186 # 获取总删除行数 187 dbconn = DBHelper(dbconfig.connectprdmysql(dns=server_ip, port=server_port)) 188 results = dbconn.query(1, delsql) 189 del_num = results[0][0] 190 191 # 删除分区 192 dbconn = DBHelper(dbconfig.connectprdmysql_releasa(dns=server_ip, port=server_port)) 193 sql = "alter table %(db_name)s.%(table_name)s drop partition %(delpartition)s;" % {'db_name': db_name, 194 'table_name': table_name, 195 'delpartition': delpartition} 196 dbconn.execute(1, sql) 197 198 199 archivelog(server_dns=server_dns, db_name=db_name, table_name=table_name, delpartition=delpartition, del_num=del_num, loginfo='Finished', status='1') 200 201 # 配置表更新状态 202 configchange(server_dns=server_dns, db_name=db_name, table_name=table_name) 203 204 except Exception, e: 205 archivelog(server_dns= server_dns, db_name= db_name, table_name=table_name, delpartition= delpartition, del_num= del_num, loginfo= e, status= '2')