1. 背景
公司大佬需要报表实时更新，从业务报表同步到数据仓库，可根据需要设置同步时间几个crontab 或者while true,用shell脚本获取增量数据，实时更新，业务系统存在物理删除的场景，这就需要去监控业务库删除操作，自己写了粗糙的脚本。本来是可以用otter 工具同步实现，但是好资源，我们也就只有几张表而已。
2. 前提
数据库配置文件：
[hxw@test1 Shell]$ cat dbConfig.sh ADMIN_DB="mysql -s -h127.0.0.1 -P3306 -uXXXXXXX -pXXXXXXXX GKIY"
mysql配置表：
3.代码
---------------------------数据库代码---------------------- CREATE TABLE `t_sys_real_sync_config` ( `id` int(11) NOT NULL AUTO_INCREMENT, `source_db` varchar(50) DEFAULT NULL, `source_table` varchar(250) DEFAULT NULL, `source_delete_cnt` varchar(50) DEFAULT NULL, `target_db` varchar(50) DEFAULT NULL, `target_table` varchar(250) DEFAULT NULL, `source_sql` varchar(5000) DEFAULT NULL, `target_sql` varchar(5000) DEFAULT NULL, `s_id` varchar(50) DEFAULT '' COMMENT '源表主键id', `t_id` varchar(50) DEFAULT '' COMMENT '仓库表主键', `status` tinyint(2) DEFAULT '0', `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, `updatetime` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间', PRIMARY KEY (`id`) ) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8;
-------------------------------shell------------------------------
#!/bin/bash
#date :2018-03-02
#create by :houxiaowei
#comment : used to sysnc the real data from source db to the target db ,just for phyica detele SQL from the source DB
#import the DB configurtion
. /data/stat/lib/bondlib.sh
script_path=`dirname $0`
script_name=`basename $0`
tempFile=${script_path}/${script_name/.sh/_temp}
s_priId_temp=${script_path}/s_priId_temp.temp
t_priId_temp=${script_path}/t_priId_temp.temp
logFile_temp=${script_path}/error_temp.log
logFile=${script_path}/${script_name/.sh/.log}
#the global vairables of the mysql Handler_delete ,it is The number of times that rows have been deleted from tables.
#the global vairables of the mysql com_delete ,The Com_delete statement counter variables indicate the number of times each delete statement has been executed.
# the Handler_delete increase just when the real row is delete . but com_delete is delete when the delete sql is executed,do not care the affacted the rows.
#the delete count in the target DB:
${REPORT_DB} -s -e "select id, source_db,ifnull(source_delete_cnt,0) as source_delete_cnt ,source_table,target_db,target_table,source_sql,target_sql,s_id,t_id from SYS.t_sys_real_sync_config where status=1" 2>${logFile} |sed 's/\t/|/g' 1>${tempFile}
################-----------------------------------------log part---------------------------------------------------------
#usage: log_write "$FUNCNAME" "$LINENO"
log_write()
{
#判断删除成功失败
file_cnt=$(cat ${logFile_temp} |grep -v Warning |wc -l)
if [ $file_cnt -gt 0 ];then
dateT=`date "+%Y-%m-%d %T"`
echo "$dateT [LINE]:$2 Function:$1 executed faild!!" >${logFile}
exit 0;
fi
}
###############----------------------------------------- end -----------------------------------------------------------
#uasge : do_check $source_db $target_db $source_sql $target_sql
do_check()
{
#取变量的地址,eval 解决动态变量问题
eval local sourcedb="$"$1
eval local targetdb="$"$2
local sourcesql=$3
local targetsql=$4
V_Source_Del_cnt=$(${sourcedb} -N -e "show global status like 'Handler_delete' " 2>${logFile_temp}|cut -f2)
log_write "$FUNCNAME" "$LINENO"
#直接重定向解决Mysql因为安全考虑输出的Warning。
v_Source_Total_cnt= $sourcedb -s -e "$sourcesql" 2>${logFile_temp};
log_write "$FUNCNAME" "$LINENO"
V_Target_Total_cnt= ${targetdb} -s -e "$targetsql" 2>${logFile_temp};
log_write "$FUNCNAME" "$LINENO"
#如果全局变量相等则不执行更新，否则比较表记录数据
if [ ${V_Source_Del_cnt} -eq ${source_delete_cnt} ];then
return 0;
elif [ ${V_Source_Del_cnt} -gt ${source_delete_cnt} ];then
#源表记录数跟仓库记录数不一致，执行删除，更新数据库
if [ $v_Source_Total_cnt != $V_Target_Total_cnt ];then
do_delete ${target_db} ${target_table} ${t_id}
do_feedback "${V_Source_Del_cnt}" "${source_db}"
else
return 0
fi
else
do_feedback "${V_Source_Del_cnt}" "${source_db}"
fi
}
#执行delete操作
#参数 do_delete [targetdb] [targettable] [t_id]
do_delete()
{
eval local targetdb="$"$1
eval local targetable=$2
eval local t_id=$3
do_findout_diff ${source_db} ${source_table} ${s_id} ${target_db} ${target_table} ${t_id}
$targetdb -e "delete from ${targetable} where ${t_id} in (${IDS})" 2>${logFile_temp}
log_write "$FUNCNAME" "$LINENO"
do_feedback
}
##执行回写功能
do_feedback()
{
${REPORT_DB} -s -e "update SYS.t_sys_real_sync_config set source_delete_cnt=${V_Source_Del_cnt} where id=${c_id}" >${logFile_temp}
log_write "$FUNCNAME" "$LINENO"
}
#找出不同主键的ID，参数源库表，主键id，连接方式，返回要删除主键列表用逗号分隔。
#usage: do_findout_diff [sourcedb] [sourcetable] [s_id] [targetdb] [targettable] [t_id]
do_findout_diff()
{
##接受参数,local 是变量只能在函数内部使用
eval local sourcedb="$"$1
eval local sourcetable=$2
eval local s_id=$3
eval local targetdb="$"$4
eval local targettable=$5
eval local t_id=$6
$sourcedb -e "select ${s_id} from ${sourcetable}" 2>${logFile_temp} 1>${s_priId_temp}
log_write "$FUNCNAME" "$LINENO"
${targetdb} -e "select ${t_id} from ${targettable}" 2>${logFile_temp} 1>${t_priId_temp}
log_write "$FUNCNAME" "$LINENO"
#id 结果集，函数定义为全局变量
#join 有坑，查出来有问题
#IDS=$(join -v 2 -o 2.1 ${s_priId_temp} ${t_priId_temp} --nocheck-order|xargs echo |sed 's/ /,/g')
#awk执行
IDS=$( awk 'NR==FNR{a[$1]=$0;next}{if(!($1 in a)) print $1}' ${s_priId_temp} ${t_priId_temp} |xargs echo |sed 's/ /,/g')
if [ "$IDS" = "" ];then
exit 0
fi
}
while read line
do
num=1
for name in c_id source_db source_delete_cnt source_table target_db target_table source_sql target_sql s_id t_id
do
#echo -n "$name="
#echo -n "$(echo $line|cut -d "|" -f${num})"
#动态给变量赋值
eval "$name='`echo -n $(echo $line|cut -d "|" -f${num} )`'"
#echo ${source_db} "${target_db}" "${source_sql}" "${target_sql}"
#开始检测业务库和数据仓库数据同步
#do_check "${source_db}" "${target_db}" "${source_sql}" "${target_sql}"
num=`expr $num + 1`
done
do_check "${source_db}" "${target_db}" "${source_sql}" "${target_sql}"
done < ${tempFile}