1. Applicable scenarios
During the project work, we will encounter a lot of test environments, such as: n sets of beta environments, multiple sets of pre-production environments, multiple sets of uat environments and other test environments. In order to ensure the consistency of the structure of all test environment tables, if it is simply checked and updated manually, it would be too laborious and inefficient, and it is easy to miss problems in the implementation process.
Therefore, it is extremely important to use automated scripts to complete this work. The main content shared today is to help you find differentiated content between different test environments through automated scripts, so as to avoid omissions in the synchronization process.
Second, the code implementation
1. Pre-processing and basic configuration
1import pymysql
2from email.mime.multipart import MIMEMultipart
3from email.mime.text import MIMEText
4import smtplib
5import time
6
7#忽略掉的db字典
8ignoreDb={'information_schema': 'information_schema',
9 'mysql': 'mysql',
10 'performance_schema': 'performance_schema',
11 'sys': 'sys'
12 }
13
14#忽略掉的db在查询sql中使用的
15ignoreDbSQL="('information_schema', 'mq_store','performance_schema', 'sys','edsystem')"
16
17#收diff报告的邮箱地址
18emails='[email protected]'
19
20#排除不在diff范围内的表名称列表
21exclude_table = ['tt', 'hurdle_policy_back', 'preferences_0309', 'preferences_0524', 'preferences_0310','t1','t2','mock_data']
2. Used to send diff reports
1def send_mail(receivers, title, content):
2 sender = '[email protected]'
3 mailto = receivers.split(",")
4 try:
5 msg = MIMEMultipart()
6 msg['Subject'] = title
7 to_user = ",".join(mailto)
8
9 print("receivers...", to_user)
10 msg['to'] = to_user
11 msg['From'] = sender
12
13 body = MIMEText(content, _subtype='html', _charset='utf-8')
14 msg.attach(body)
15 smtp = smtplib.SMTP('smtp.office365.com', 587)
16 smtp.starttls()
17 print("sending")
18 smtp.login("[email protected]", "test123456")
19 smtp.sendmail(sender, mailto, msg.as_string())
20 print("send")
21 smtp.quit()
22 except smtplib.SMTPException as e:
23 print(e)
3. Query to get all column name data
1def queryAllColumns(mycursor):
2 sql = " SELECT TABLE_SCHEMA 库名,TABLE_NAME 表名,COLUMN_NAME 列名, COLUMN_TYPE 数据类型, DATA_TYPE 字段类型, " \
3 "CHARACTER_MAXIMUM_LENGTH 长度, IS_NULLABLE 是否为空, COLUMN_DEFAULT 默认值, " \
4 "COLUMN_COMMENT 备注 FROM INFORMATION_SCHEMA.COLUMNS " \
5 "where table_schema not in " + ignoreDbSQL;
6 # print(sql)
7 mycursor.execute(sql)
8 result = mycursor.fetchall()
9
10 d = {};
11 for x in result:
12 r={}
13 r['TABLE_SCHEMA'] = x[0];
14 r['TABLE_NAME'] = x[1];
15 if(x[1] in exclude_table):
16 continue
17 r['COLUMN_NAME'] = x[2];
18 r['COLUMN_TYPE'] = x[3];
19 r['DATA_TYPE'] = x[4];
20 r['CHARACTER_MAXIMUM_LENGTH'] = x[5];
21 r['IS_NULLABLE'] = x[6];
22 r['COLUMN_DEFAULT'] = x[7];
23 r['COLUMN_COMMENT'] = x[8];
24 d[x[0] + "." + x[1] + "." + x[2]] = r;
4. The final data pattern of d is as follows and returns it
1 d=
2 {'route_config_meta.entity_sharding_config.create_date':
3 {'TABLE_SCHEMA': 'route_config_meta', 'TABLE_NAME': 'entity_sharding_config', 'COLUMN_NAME': 'create_date',
4 'COLUMN_TYPE': 'datetime', 'DATA_TYPE': 'datetime', 'CHARACTER_MAXIMUM_LENGTH': None, 'IS_NULLABLE': 'NO',
5 'COLUMN_DEFAULT': None, 'COLUMN_COMMENT': '创建时间'},
6 'route_config_meta.entity_sharding_config.update_time':
7 {'TABLE_SCHEMA': 'route_config_meta', 'TABLE_NAME': 'entity_sharding_config', 'COLUMN_NAME': 'update_time',
8 'COLUMN_TYPE': 'datetime', 'DATA_TYPE': 'datetime', 'CHARACTER_MAXIMUM_LENGTH': None, 'IS_NULLABLE': 'NO',
9 'COLUMN_DEFAULT': 'CURRENT_TIMESTAMP', 'COLUMN_COMMENT': '更新时间'}
10 }
11 return d;
5. Query to get all index data
1def queryAllIndex(mycursor):
2 sql = "Select TABLE_SCHEMA 库名称,TABLE_NAME 表名称,INDEX_NAME 索引的名称,SEQ_IN_INDEX 索引中的列序列号,COLUMN_NAME 列名称 from INFORMATION_SCHEMA.STATISTICS " \
3 "where TABLE_SCHEMA not in " + ignoreDbSQL;
4
5 sql=
6 Select TABLE_SCHEMA,TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX,COLUMN_NAME from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA
7 not in ('information_schema', 'mq_store', 'mysql', 'performance_schema', 'slow_query_log','repeater',
8 'repeater_console','sys','edsystem','crs_adapter','oxi-adapter')
9
10 mycursor.execute(sql)
11
12 result = mycursor.fetchall()
13 # print(result)
14
15 result=
16 (
17 ('auth', 'authorities', 'PRIMARY', 1, 'id'),
18 ('auth', 'authorities', 'idx_code', 1, 'code')
19 )
20
21 d = {};
22 for x in result:
23 r = {}
24 r['TABLE_SCHEMA'] = x[0];
25 r['TABLE_NAME'] = x[1];
26 if (x[1] in exclude_table):
27 continue
28 r['INDEX_NAME'] = x[2];
29 r['SEQ_IN_INDEX'] = x[3];
30 r['COLUMN_NAME'] = x[4];
31 d[x[0]+"."+x[1]+"."+x[2]]=r;
32
33 d=
34 {
35 'auth.authorities.PRIMARY': {'TABLE_SCHEMA': 'auth', 'TABLE_NAME': 'authorities', 'INDEX_NAME': 'PRIMARY', 'SEQ_IN_INDEX': 1, 'COLUMN_NAME': 'id'},
36 'auth.authorities.idx_code': {'TABLE_SCHEMA': 'auth', 'TABLE_NAME': 'authorities', 'INDEX_NAME': 'idx_code', 'SEQ_IN_INDEX': 1, 'COLUMN_NAME': 'code'}
37 }
38 return d;
6, query to obtain the name of the database instance
1def queryDbs(mycursor):
2 sql = 'show databases';
3 mycursor.execute(sql)
4 result = mycursor.fetchall()
5
6 d = {};
7 for x in result:
8 d[x[0]] = x[0];
9 return d;
7. Build a diff report in html format
1def buildHtml(db1,db2,cst,ist,tip):
2 # tip = db2.get('name') + "(" + db2.get('host') + ")"+ " 对比 " + db1.get('name') + "(" + db1.get('host') + ")"
3 str = '<!DOCTYPE html> <html> <meta charset="utf-8"> <head> <style type="text/css"> table.gridtable {font-family: verdana,arial,sans-serif; font-size:11px; color:#333333; border-width: 1px; border-color: #666666; border-collapse: collapse; } table.gridtable th {border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #dedede; } table.gridtable td {border-width: 1px; padding: 8px; border-style: solid; border-color: #666666; background-color: #ffffff; } </style> </head>';
4 str = str+'<body>'
5
6 str = str+' <table class="gridtable">'
7 str = str+ tip+' 缺少字段：'
8 str = str+' <tr> <th>database</th> <th>table</th> <th>column</th> <tr/> '
9
10 for x in cst:
11 xs = x.split('.')
12 str=str + '<tr> <td>'+xs[0]+'</td> <td>'+xs[1]+'</td> <td>'+xs[2]+'</td> <tr/>'
13 str = str + ' </table>'
14
15
16 str = str + ' <table class="gridtable">'
17 str = str + tip + ' 缺少索引：'
18 str = str + ' <tr> <th>database</th> <th>table</th> <th>index</th> <tr/> <tr>'
19
20 for x in ist:
21 xs = x.split('.')
22 str = str + '<tr> <td>' + xs[0] + '</td> <td>' + xs[1] + '</td> <td>' + xs[2] + '</td> <tr/>'
23
24 str = str + ' </table>'
25 str = str+'</body> </html>';
26 return str;
8. Compare database fields with index diff, generate diff report, and send email
1def diff(db1,db2):
2 mydb1 = pymysql.connect(
3 host=db1.get('host'),
4 user=db1.get('user'),
5 passwd=db1.get('password')
6 );
7
8 mydb2 = pymysql.connect(
9 host=db2.get('host'),
10 user=db2.get('user'),
11 passwd=db2.get('password')
12 )
13
14 mycursor1 = mydb1.cursor()
15 mycursor2 = mydb2.cursor()
16
17 #获取两个库里面的所有字段相关值
18 all_columns1 = queryAllColumns(mycursor1)
19 all_columns2 = queryAllColumns(mycursor2)
20
21 #获取两个库里面的所有索引相关值
22 all_index1 = queryAllIndex(mycursor1)
23 all_index2 = queryAllIndex(mycursor2)
24
25 mycursor1.close()
26 mycursor2.close()
27 mydb1.close()
28 mydb2.close()
29
30 #定义了一个提示信息标题头
31 tip = db2.get('name') +" 对比 " + db1.get('name') + "--(" + db2.get('host') + " 对比 "+ db1.get('host') + ")"
32
33 all_columns1的数据格式与如下all_index1雷同（{key:value}），但是数据值上是有差异的。
34
35 cs1 = set(all_columns1)
36 cs2 = set(all_columns2)
37 cst = cs1.difference(cs2)
9. The sample data of the index is as follows
1 all_index1=
2 {
3 'auth.authorities.PRIMARY': {'TABLE_SCHEMA': 'auth', 'TABLE_NAME': 'authorities', 'INDEX_NAME': 'PRIMARY', 'SEQ_IN_INDEX': 1, 'COLUMN_NAME': 'id'},
4 'auth.authorities.idx_code': {'TABLE_SCHEMA': 'auth', 'TABLE_NAME': 'authorities', 'INDEX_NAME': 'idx_code', 'SEQ_IN_INDEX': 1, 'COLUMN_NAME': 'code'}
5 }
10. Use set to do de-duplication, leaving only the only key
1 is1 = set(all_index1)
2 is2 = set(all_index2)
3 #
4 ist = is1.difference(is2)
5
6 ist=
7 {'rate.tt.PRIMARY', 'entity_storage_1.reservation_log_0.transaction_id', 'rate.rate_header.idx_tcode_tax_include'}
8
9 content = buildHtml(db1,db2,sorted(cst),sorted(ist),tip)
10
11 fmt = '%Y-%m-%d %a %H:%M:%S' # 定义时间显示格式
12 nowtime = time.strftime(fmt, time.localtime(time.time())) # 把传入的元组按照格式，输出字符串
13 print ('当前的时间：', nowtime)
14
15 if len(ist) >= 0 or len(cst) >0:
16 send_mail(emails,'DBDIFF:'+tip ,content)
17
18if __name__ == '__main__':
19 beta1 = {'name': 'beta1', 'host': '10.7.36.34', 'user': 'root', 'password': '123456'}
20 beta2 = {'name': 'beta2', 'host': '10.7.36.2', 'user': 'root', 'password': '123456'}
21
22 diff(beta1, beta2)
11. The content of the output mail is as follows
beta2 对比 beta1--(10.7.36.2 对比 10.7.36.34) 缺少字段：
database table column
db1 task_statistic create_date
db1 task_statistic execute_task_count
beta2 对比 beta1--(10.7.36.2 对比 10.7.36.34) 缺少索引：
database table index
db1 award_test idx_membership_related_id
db2 record_flow idx_related_member_id
Three, summary
The content shared today is more practical, and the Python implementation code is dry goods. It is recommended that hands-on operations are more helpful to deepen understanding~
Welcome to pay attention to [The Way of Infinite Testing] public account , reply [Receive Resources]
Python programming learning resources dry goods,
Python+Appium framework APP UI automation,
Python+Selenium framework Web UI automation,
Python+Unittest framework API automation,
Resources and codes are sent for free~
There is a QR code of the official account at the bottom of the article, you can just scan it on WeChat and follow it.
Remarks: My personal public account has been officially opened, dedicated to the sharing of test technology, including: big data testing, functional testing, test development, API interface automation, test operation and maintenance, UI automation testing, etc., WeChat search public account: "Infinite The Way of Testing", or scan the QR code below:
Add attention and let us grow together!