mysql存储过程跨库建表并抓取数据 TODO 待demo化

一. 简介

           需求： A库有多张按时间分表创建的表a_table_${date}，我想用定时器每天凌晨在B库创建一张和A库昨天的表一样的表b_table_${date}，然后将数据抓取过来,然后再对同步过来的表b_table_${date}做一次统计汇总插入到B库的另一张表b_table_statistic中。 使用存储过程做。可以根据统计时间传参来同步不同时间的数据。

          思路：

               1. 如果B库不存在统计日期的表b_table_${date}则创建一个新表b_table_${date}；

               2. 开启事务，将B库表b_table_${date}的数据清空，方便失败了再次统计；

               3. 将A库a_table_${date}表的数据同步到B库b_table_${date}表；

                   注意： 在插入的时候如果数据量比较大，可以通过分组进行批量插入来避免事务问题和mysql插入数据量限制问题；

               4. 先删除B库b_table_statistic表中统计日期的数据；然后再向B库b_table_statistic表中插入统计日期的统计数据；

               5. 关闭事务；

二. sql建模

三. 脚本

-- statistic_date 统计日期  接收yyyy-MM-dd类型的时间字符串 
CREATE PROCEDURE create_current_day_cdr_table (IN statistic_date VARCHAR(64)) BEGIN
	
	-- 同步t_cdr表记录需要插入的次数
	DECLARE CDR_TABLE_INSERT_COUNT INT;
	
	-- 声明事务
	DECLARE EXIT HANDLER FOR SQLEXCEPTION, SQLWARNING ROLLBACK ;

	-- 每次插入的记录数
	SET @EACH_INSERT_SIZE = 1000;
	
	-- 循环变量 
	SET @i = 1;
	

	-- 根据统计日期拼接话单详情表名称
	SET @tableName = CONCAT("t_cdr_", from_unixtime( unix_timestamp(statistic_date), '%Y%m%d') );  


	-- 如果统计日期的话单详情表不存在则创建 
	SET @createTableStr = CONCAT(" CREATE TABLE IF NOT EXISTS ", @tableName , "( 
					`id` int(11) NOT NULL AUTO_INCREMENT,
					`call_type` char(2) DEFAULT NULL COMMENT '通话类型',
					`cb_sessionid` varchar(64) DEFAULT NULL,
					`busiprefix` varchar(16) DEFAULT NULL,
					`caller_brand` varchar(64) DEFAULT NULL COMMENT '主叫品牌',
					`caller` varchar(64) DEFAULT NULL COMMENT '主叫号码',
					`calleraccess` varchar(64) DEFAULT NULL COMMENT '呼入主叫',
					`callee_brand` varchar(64) DEFAULT NULL COMMENT '被叫品牌',
					`callee` varchar(64) DEFAULT NULL COMMENT '被叫号码',
					`calleeaccess` varchar(64) DEFAULT NULL COMMENT '呼入被叫',
					`calleeoriginal` varchar(64) DEFAULT NULL COMMENT '被叫裸号',
					`callerip` varchar(64) DEFAULT NULL COMMENT '主叫IP',
					`in_gw_media_ip` varchar(64) DEFAULT NULL COMMENT '主叫媒体IP',
					`in_gw_media_isp` int(11) DEFAULT NULL,
					`callergatewayh323id` varchar(512) DEFAULT NULL COMMENT '主叫经由网关',
					`callerproductid` varchar(64) DEFAULT NULL COMMENT '主叫设备',
					`callertogateway` varchar(64) DEFAULT NULL COMMENT '呼出主叫',
					`calleeip` varchar(64) DEFAULT NULL COMMENT '被叫信令IP',
					`calleegatewayh323id` varchar(64) DEFAULT NULL COMMENT '被叫经由网关',
					`calleeproductid` varchar(128) DEFAULT NULL COMMENT '被叫设备',
					`calleetogateway` varchar(64) DEFAULT NULL COMMENT '呼出被叫',
					`rtp_ip` varchar(64) DEFAULT NULL COMMENT '被叫媒体IP',
					`gd_gw_media_isp` int(11) DEFAULT NULL,
					`billingmode` char(1) DEFAULT NULL,
					`calllevel` char(1) DEFAULT NULL,
					`agentfeetime` int(11) DEFAULT NULL,
					`starttime` datetime DEFAULT NULL COMMENT '起始时间',
					`stoptime` datetime DEFAULT NULL COMMENT '结束时间',
					`pdd` int(11) DEFAULT NULL COMMENT '接通延迟(秒)',
					`linepdd` int(11) DEFAULT NULL,
					`waittime` int(11) DEFAULT NULL COMMENT '接续时长(秒)',
					`holdtime` int(11) DEFAULT NULL COMMENT '通话时长(秒)',
					`feeprefix` varchar(64) DEFAULT NULL COMMENT '计费前缀',
					`feetime` int(11) DEFAULT NULL COMMENT '计费时长(秒)',
					`fee` bigint(20) unsigned DEFAULT NULL COMMENT '通话费用(元)',
					`package_id` int(11) DEFAULT NULL,
					`package_fee` bigint(20) unsigned DEFAULT NULL COMMENT '套餐费用(元)',
					`package_feetime` int(11) DEFAULT NULL COMMENT '套餐时长(秒)',
					`account` varchar(32) DEFAULT NULL COMMENT '账户号码',
					`account_name` varchar(255) DEFAULT NULL COMMENT '账户名称',
					`enddirection` char(1) DEFAULT NULL COMMENT '挂断方',
					`endreason` varchar(255) DEFAULT NULL COMMENT '终止原因',
					`agentpackagefee` bigint(20) unsigned DEFAULT NULL,
					`agentpackagefeetime` int(11) DEFAULT NULL,
					`agentpackageid` int(11) DEFAULT NULL,
					`agentaccount` varchar(255) DEFAULT NULL,
					`agentname` varchar(255) DEFAULT NULL,
					`agentfee` bigint(20) unsigned DEFAULT NULL COMMENT '落地费用(元)',
					`agentfeeprefix` varchar(64) DEFAULT '0',
					`gatewayrouting_type` int(11) DEFAULT NULL,
					`caller_area` varchar(32) DEFAULT NULL COMMENT '主叫地区',
					`area_code` varchar(32) DEFAULT NULL COMMENT '被叫地区',
					`province` varchar(64) DEFAULT NULL,
					`city` varchar(64) DEFAULT NULL,
					`switch` varchar(64) DEFAULT NULL COMMENT '主叫经由VLS-switch',
					`operators` int(11) unsigned DEFAULT '0',
					`outgw_area` varchar(64) DEFAULT NULL,
					`ingw_type` varchar(32) DEFAULT NULL,
					`routetrace` varchar(320) DEFAULT NULL COMMENT '路由跟踪',
					`call_encode` varchar(200) DEFAULT NULL COMMENT '语音编码',
					`fee_type` char(1) DEFAULT NULL COMMENT '费率类型',
					`fee_type_ee` char(1) DEFAULT NULL,
					PRIMARY KEY (`id`),
					KEY `index_account` (`account`),
					KEY `index_account_name` (`account_name`),
					KEY `index_agentaccount` (`agentaccount`),
					KEY `index_caller` (`caller`),
					KEY `index_callee` (`callee`),
					KEY `index_callergatewayh323id` (`callergatewayh323id`(64)),
					KEY `index_calleegatewayh323id` (`calleegatewayh323id`),
					KEY `index_starttime` (`starttime`),
					KEY `index_callerproductid` (`callerproductid`),
					KEY `index_calleeproductid` (`calleeproductid`),
					KEY `index_city` (`city`),
					KEY `index_call_type` (`call_type`),
					KEY `index_stoptime` (`stoptime`)
			)ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8");
		
	PREPARE createtablestmt FROM	@createTableStr;
	EXECUTE createtablestmt;


	-- 开启事务  注意： 必须要在建表语句下面  需要事务的地方进行 如果放在上面 则事务不起作用  
  START TRANSACTION ;
	
	
	-- 如果统计日期的话单详情表存在数据则先删除
	SET @delCDRTableStr = CONCAT( " DELETE FROM ", @tableName );
	PREPARE delCDRTableStmt FROM @delCDRTableStr;
	EXECUTE delCDRTableStmt;
	
	
	-- 同步抓取vlsdbcdr库统计日期的话单详情表数据  由于表数据量较大，采取分批插入 将EXECUTE结果赋值到@CDRTableRowCount变量中
	SET @countCDRTableStr = CONCAT( " SELECT COUNT(1) INTO @CDRTableRowCount FROM vlsdbcdr.", @tableName );
  PREPARE countCDRTableStmt FROM @countCDRTableStr;
	EXECUTE countCDRTableStmt;
	
	
  -- 计算需要插入的次数
	SELECT CEIL (@CDRTableRowCount / @EACH_INSERT_SIZE) INTO CDR_TABLE_INSERT_COUNT; 
	
	WHILE @i <= CDR_TABLE_INSERT_COUNT DO 
	
	
		SET @baseColumn = " `id`, `call_type`, `cb_sessionid`, `busiprefix`, `caller_brand`, `caller`, `calleraccess`, `callee_brand`, `callee`, `calleeaccess`, `calleeoriginal`, 
												`callerip`, `in_gw_media_ip`, `in_gw_media_isp`, `callergatewayh323id`, `callerproductid`, `callertogateway`, `calleeip`, `calleegatewayh323id`, `calleeproductid`,
												`calleetogateway`, `rtp_ip`, `gd_gw_media_isp`, `billingmode`, `calllevel`, `agentfeetime`, `starttime`, `stoptime`, `pdd`, `linepdd`, `waittime`, `holdtime`,
												`feeprefix`, `feetime`, `fee`, `package_id`, `package_fee`, `package_feetime`, `account`, `account_name`, `enddirection`, `endreason`, `agentpackagefee`,
												`agentpackagefeetime`, `agentpackageid`, `agentaccount`, `agentname`, `agentfee`, `agentfeeprefix`, `gatewayrouting_type`, `caller_area`, `area_code`,
												`province`, `city`, `switch`, `operators`, `outgw_area`, `ingw_type`, `routetrace`, `call_encode`, `fee_type`, `fee_type_ee` ";
												
		SET @insertCDRStr = CONCAT(" INSERT INTO ", @tableName, "(", @baseColumn, ")", " SELECT ", @baseColumn, " FROM vlsdbcdr.", @tableName, 
																" LIMIT ", (@i - 1) * @EACH_INSERT_SIZE, ",", @EACH_INSERT_SIZE );

		PREPARE insertCDRstmt FROM	@insertCDRStr;
		EXECUTE insertCDRstmt;
	
		SET @i = @i + 1;
		
	END WHILE;
	
	
	
	-- 删除销售客户账单表已存在的统计日期数据  
	DELETE FROM t_sale_customer_day_cost WHERE statistical_time = statistic_date;
	
	
	-- 向销售客户日账单表插入统计日期的数据  
	SET @insertSCDCStr = CONCAT( "
	
			INSERT INTO osp.t_sale_customer_day_cost
				( statistical_time, customer_id, account_id, service_type, fee_type, call_times, caller, caller_area, feetime, fee, create_time)
			SELECT '", statistic_date,"' AS statistical_time, (SELECT out_id FROM t_account AS A WHERE A.account_id = CDR.account AND A.type = 0) AS customer_id, 
					CDR.account, '5', CDR.fee_type, count(1) AS call_count, CDR.caller, CDR.caller_area, SUM(CDR.feetime / 60), SUM(CDR.fee), NOW() 
			FROM ",
				@tableName, " AS CDR
			WHERE
				CDR.call_type = '5'
			GROUP BY
				customer_id, CDR.account, CDR.call_type, CDR.fee_type ");

	PREPARE insertSCDCstmt FROM	@insertSCDCStr;
	EXECUTE insertSCDCstmt;


  -- 提交事务
	COMMIT ;

END;







	

四 定时器代码

package com.yzx.osp.modular.task.controlller;

import com.yzx.osp.common.util.DateUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;

import java.util.Date;

/**
 * @author yuanshushu
 * @date 2018/9/13
 * @description 话单详情定时任务类
 */
@Component
public class CdrDateTask {

    @Autowired
    private JdbcTemplate jdbcTemplate;

    @Scheduled(cron = "0/10 * *  * * ?")
    public void syncTCdrDateData() {

        try {

            // 按日期生成话单详情表
            this.jdbcTemplate.execute("call create_current_day_cdr_table()");

        } catch (Exception e) {
            e.printStackTrace();
        }


    }


}

五. 存储过程DEMO     小

5.1 遍历分批插入

CREATE PROCEDURE test () BEGIN


	DECLARE CDR_TABLE_ROW_COUNT BIGINT;
	DECLARE CDR_TABLE_INSERT_COUNT INT;

  -- 声明事务
	DECLARE EXIT HANDLER FOR SQLEXCEPTION, SQLWARNING ROLLBACK ;
	
	SET @EACH_INSERT_SIZE = 100;
	SET @i = 1;
	
	
	
	SELECT count(1) FROM t_cdr_20180811 INTO CDR_TABLE_ROW_COUNT;
	SELECT CEIL (CDR_TABLE_ROW_COUNT / @EACH_INSERT_SIZE) INTO CDR_TABLE_INSERT_COUNT;

	
	START TRANSACTION ;
	

	WHILE @i <= CDR_TABLE_INSERT_COUNT DO
		

		-- 拼接插入sql   注意： mysql  limit m,n  m为开始行数, n为查询行数  
		SET @insertStr = CONCAT( "
						INSERT INTO t_cdr_20180812 
								(call_type, cb_sessionid)
						SELECT
								call_type, cb_sessionid 
						FROM
								t_cdr_20180811
						limit ", (@i - 1) * @EACH_INSERT_SIZE, ",", @EACH_INSERT_SIZE );
						
		PREPARE insertStrStmt FROM @insertStr;
		EXECUTE insertStrStmt; 
						
		SET @i = @i + 1;
		
	END WHILE;
	
	COMMIT ;

END;

 5.2 获取EXECUTE执行结果

-- statistic_date 统计日期  接收yyyy-MM-dd类型的时间字符串 
CREATE PROCEDURE create_current_day_cdr_table (IN statistic_date VARCHAR(64)) BEGIN
	

	
	SET @tableName = 't_cdr_20180811';
	
    
	SET @countCDRTableStr = CONCAT( " SELECT COUNT(1) INTO @STABLE_CDR_ROW_COUNT FROM vlsdbcdr.", @tableName);
    PREPARE countCDRTableStmt FROM @countCDRTableStr;
	EXECUTE countCDRTableStmt;
	
	SELECT @STABLE_CDR_ROW_COUNT;

END;