需要将json格式的报文存到mysql中，支持后期从mysql恢复成json
支持表自动创建（含表说明），表字段自动创建（类型、长度自动扩展）
用到Jfinal，mysql
需要有一个表clientlog存原始报文，还需要有一个表存表关系sys_tbl_tree（mysql表名有长度限制，所以会用到真实表名与原始表名）
CREATE TABLE `sys_tbl_tree` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`TABLE_NAME` varchar(500) DEFAULT NULL COMMENT '真实表名',
`ORG_NAME` varchar(500) DEFAULT NULL COMMENT 'table原始表名',
`PARENT` varchar(500) DEFAULT NULL,
`TYPE` int(1) DEFAULT '1' COMMENT '1多对1 2 1对1',
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=161 DEFAULT CHARSET=utf8 COMMENT='系统-表结构关系';
CREATE TABLE `clientlog` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`method` varchar(50) DEFAULT NULL COMMENT '接口方法',
`params` longtext,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=3509 DEFAULT CHARSET=utf8 COMMENT='系统-报文日志';
解析报文入库
1.先将原始报文存到clientlog表中（Clientlog model请用jfinal官方提供的工具生成）
2.MysqlTableColumnUtil.init();全局的只需要调用一次，我放在JFinalConfig的onStart方法中了
3.new MysqlTableColumnUtil().apiDataInDo(data, bean);data为json报文，bean为CheckUserBean
public class CheckUserBean extends LanlinUtil {
/**
* 请求方法中文描述
*/
public String portInfoName;
/**
* json报文
*/
public String params;
/**
* json报文存库
*/
public Clientlog clientlog;
}
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import bean.CheckUserBean;
import com.jfinal.kit.HashKit;
import com.jfinal.kit.PropKit;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;
public class MysqlTableColumnUtil extends LanlinUtil {
public static final String VARCHAR = "VARCHAR";
public static final String TEXT = "TEXT";
public static final String LONGTEXT = "LONGTEXT";
/**
* 自动处理表全部用这个做主键
*/
public static final String CDEC_UUID = "CDEC_UUID";
public static synchronized void init() {
String table_schema = PropKit.get("table_schema");
List<Record> tables = Db.use("information_schema").find("select TABLE_NAME from TABLES where TABLE_SCHEMA = '" + table_schema + "'");
List<Record> colomns = Db.use("information_schema").find("select TABLE_NAME, COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE from COLUMNS where TABLE_SCHEMA = '" + table_schema + "'");
for(Record table : tables) {
String TABLE_NAME = table.getStr("TABLE_NAME");
List<Record> childs = new LinkedList<Record>();
for(Record cl : colomns) {
String t = cl.getStr("TABLE_NAME");
if(null != t && t.equals(TABLE_NAME)) {
cl.remove("TABLE_NAME");
childs.add(cl);
}
}
table.set("COLOMNS", childs);
}
MysqlTableColumnUtil.tables = tables;
}
// "TABLE_NAME"，"TABLE_COMMENT", "COLOMNS" : List<Record> {"DATA_TYPE","CHARACTER_MAXIMUM_LENGTH","NUMERIC_PRECISION","NUMERIC_SCALE"}
/**
* TINYINT 1 字节 (-128，127) SMALLINT 2 字节 (-32 768，32 767) MEDIUMINT 3 字节 (-8 388 608，8 388 607) INT或INTEGER 4 字节 (-2 147 483 648，2 147 483 647)
* BIGINT 8 字节 (-9 233 372 036 854 775 808，9 223 372 036 854 775 807) FLOAT 4 字节 (-3.402 823 466 E+38，1.175 494 351 E-38)，0，(1.175 494 351
* E-38，3.402 823 466 351 E+38) DOUBLE 8 字节 (1.797 693 134 862 315 7 E+308，2.225 073 858 507 201 4 E-308)，0，(2.225 073 858 507 201 4 E-308，1.797
* 693 134 862 315 7 E+308) DECIMAL 对DECIMAL(M,D) ，如果M>D，为M+2否则为D+2
*/
/**
* DATE 3字节 TIME 3字节 YEAR 1字节 DATETIME 8字节 TIMESTAMP 8字节
*/
/**
* CHAR 255字节 VARCHAR 65535字节 TINYTEXT 255字节 TEXT 65535字节 LONGTEXT 4294967295字节
*/
/**
* {
* "TABLE_NAME" : "表名",
* "TABLE_COMMENT" : "备注",
* "COLOMNS" ： {
* "COLUMN_NAME" : "字段名",
* "DATA_TYPE" : "字段类型"
* "CHARACTER_MAXIMUM_LENGTH" : "varchar,text等最大长度"
* "NUMERIC_PRECISION" : "数值型的长度", "NUMERIC_SCALE" : "数值型的小数位"
* }
* }
* 所有字段统一处理成VARCHAR，TEXT，LONGTEXT
*/
public static List<Record> tables;
/**
* @param tableName
* @return 根据表名获取相关表
*/
public static Record getTable(String tableName) {
tableName = doTblName(tableName);
Record rc = null;
for(Record rr : tables) {
if(rr.getStr("TABLE_NAME").equals(tableName)) {
rc = rr;
break;
}
}
return rc;
}
/**
* @param tableName
* @return 根据字段名获取相关字段
*/
public static Record colomnsHasColomn(List<Record> COLOMNS, String colomn) {
for(Record col : COLOMNS) {
if(col.getStr("COLUMN_NAME").equals(colomn)) {
return col;
}
}
return null;
}
/**
* @param name
* @param length
* @param COLOMNS
* CHAR 255字节 VARCHAR 65535字节 TINYTEXT 255字节 TEXT 65535字节 LONGTEXT 4294967295字节
*/
public void addColomn(String name, Integer length, Record table) {
String tblName = table.getStr("TABLE_NAME");
List<Record> COLOMNS = table.get("COLOMNS");
Record r = new Record();
r.set("COLUMN_NAME", name);
r.set("DATA_TYPE", "varchar");
r.set("CHARACTER_MAXIMUM_LENGTH", length);
COLOMNS.add(r);
if(!name.equals(CDEC_UUID)) {
addTableFieldType(tblName, name, VARCHAR + "(" + length + ")");
}
}
public Object[] getLength(Object value) {
Integer length = null;
try {
length = value.toString().getBytes("UTF-8").length;
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
if(length < 400) {
return new Object[]{VARCHAR, 200};
} else if(length < 800) {
return new Object[]{VARCHAR, 400};
} else if(length < 2000) {
return new Object[]{VARCHAR, 1000};
} else if(length < 4000) {
return new Object[]{VARCHAR, 2000};
} else if(length < 65535) {
return new Object[]{TEXT, null};
} else if(length < 4294967295l) {
return new Object[]{LONGTEXT, null};
}
return new Object[]{VARCHAR, 200};
}
/**
* @param name
* @param type
* CHAR 255字节 VARCHAR 65535字节 TINYTEXT 255字节 TEXT 65535字节 LONGTEXT 4294967295字节
* @throws Exception
*/
public void addColomn(String key, Object value, Record table) {
String tblName = table.getStr("TABLE_NAME");
List<Record> COLOMNS = table.get("COLOMNS");
Record r = new Record();
Object[] xx = getLength(value);
r.set("COLUMN_NAME", key);
r.set("DATA_TYPE", xx[0]);
r.set("CHARACTER_MAXIMUM_LENGTH", xx[1]);
COLOMNS.add(r);
addTableFieldType(tblName, key, xx[0].toString().equals(VARCHAR) ? VARCHAR + "(" + xx[1] + ")" : xx[0].toString());
}
public void apiDataInDo(Object data, CheckUserBean bean) {
apiDataInDo(data, bean.getMethod(), null, null, false, bean);
}
/**
* @param data 数据
* @param tblName 表名
* @param parTblName 上级表名
* @param parentKey 上级主键
* @param isArray 是否1对多
* @param clientLogId log id
* @param portInfoName 请求接口方法备注，放在表备注里面
*/
public void apiDataInDo(Object data, String tblName, String parTblName, String parentKey,
Boolean isArray, CheckUserBean bean) {
tblName = tblName.toLowerCase();
if(null != parTblName) {
parTblName = parTblName.toLowerCase();
}
//数组，继续递归
if (data instanceof JSONArray) {
JSONArray dataArr = (JSONArray) data;
for (int i = 0; i < dataArr.size(); i++) {
Object job = dataArr.get(i);
apiDataInDo(job, tblName, parentKey, parTblName, true, bean);
}
} else if (data instanceof JSONObject) {
Record table = getTable(tblName);
List<Record> COLOMNS = null;
//处理表 不存在新增
if(null == table) {//表不存在
table = new Record();
table.set("TABLE_NAME", doTblName(tblName));
table.set("TABLE_COMMENT", "");
createTable(tblName, parTblName, isArray, bean.portInfoName);//新建一个表
COLOMNS = new ArrayList<Record>();
table.set("COLOMNS", COLOMNS);
addColomn(CDEC_UUID, 50, table);//主键不会新建字段
addColomn("CDEC_PARENTUUID", 50, table);//上级id
addColomn("CDEC_CLIENTLOGID", 50, table);//clientLogId id
tables.add(table);
} else {
COLOMNS = table.get("COLOMNS");
}
//处理表 end
String uuid = UUID();
JSONObject dataObj = (JSONObject) data;
Iterator<?> iterator = dataObj.keys();
Record saveObj = new Record();
saveObj.set(CDEC_UUID, uuid);
saveObj.set("CDEC_CLIENTLOGID", bean.clientlog.getID());
saveObj.set("CDEC_PARENTUUID", parentKey);
while (iterator.hasNext()) {
String key = (String) iterator.next();
Object value = dataObj.get(key);
if(null == value) {
} else if (value instanceof JSONArray) {
JSONArray dataArr = (JSONArray) value;
for (int i = 0; i < dataArr.size(); i++) {
Object job = dataArr.get(i);
apiDataInDo(job, tblName + "_" + key, tblName, uuid, true, bean);
}
} else if (value instanceof JSONObject) {
if(((JSONObject)value).isNullObject()) {//null
return;
}
apiDataInDo(value, tblName + "_" + key, tblName, uuid, false, bean);
} else {//字段
saveObj.set(key, value);
//处理字段 存在、类型、长度start
Record colomn = colomnsHasColomn(COLOMNS, key);
if(null == colomn) {//没有该字段
addColomn(key, value, table);
} else {
Object[] xx = getLength(value);
String type = colomn.getStr("DATA_TYPE");
switch (type.toUpperCase()) {
case "LONGTEXT":
break;
case "TEXT":
if(xx[0].toString().equals("LONGTEXT")) {//更改字段类型
modifyTableFieldType(tblName, key, "LONGTEXT");
}
break;
case "VARCHAR":
if(!xx[0].toString().equals("VARCHAR")) {//更改字段类型
modifyTableFieldType(tblName, key, xx[0].toString());
} else {//判断字段长度
Integer CHARACTER_MAXIMUM_LENGTH = colomn.getInt("CHARACTER_MAXIMUM_LENGTH");
Integer x1 = (Integer)xx[1];
if(CHARACTER_MAXIMUM_LENGTH.compareTo(x1) < 0) {//更改字段长度
modifyTableFieldType(tblName, key, "VARCHAR(" + x1 + ")");
}
}
break;
default:
break;
}
}
//处理字段 存在、类型、长度end
}
}
//需要保存
if(null != saveObj) {
Db.save(doTblName(tblName), CDEC_UUID, saveObj);
}
}
}
/**
* 更改字段类型,长度等
*/
static String modifyTableFieldType = "ALTER TABLE `%s` MODIFY COLUMN `%s` %s CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL;";
public void modifyTableFieldType(String table, String field, String type) {
Db.update(String.format(modifyTableFieldType, doTblName(table), field, type));
}
/**
* 新增字段
*/
static String addTableFieldType = "ALTER TABLE `%s` ADD COLUMN `%s` %s NULL DEFAULT NULL;;";
public void addTableFieldType(String table, String field, String type) {
Db.update(String.format(addTableFieldType, doTblName(table), field, type));
}
/**
* 新增空表，主键CDEC_UUID
*/
static String addTable = "CREATE TABLE `%s` (`CDEC_UUID` varchar(50) NOT NULL ,PRIMARY KEY (`CDEC_UUID`)) COMMENT='%s';";
public void createTable(String tblName,String parTblName, Boolean isArray, String portInfoMemo) {
String realTblName = doTblName(tblName);
Db.update(String.format(addTable, realTblName, portInfoMemo + "_" + tblName));
//处理表tree结构
Record tblTree = new Record();
tblTree.set("TABLE_NAME", realTblName);
tblTree.set("PARENT", parTblName);
tblTree.set("ORG_NAME", tblName);
tblTree.set("TYPE", isArray ? 1 : 2);
Db.save("sys_tbl_tree", "ID", tblTree);
}
/**
* @param mysql 表最多64字节，超长特殊处理
* @return
*/
public static String doTblName(String tblName) {
if(tblName.getBytes().length > 64) {
tblName = tblName.replace("_", "");
}
if(tblName.getBytes().length > 64) {
tblName = new String(HashKit.sha1(tblName));
}
return tblName;
}
}效果如下
