Need to save the json formatted message in mysql, support later recovery from mysql to json
Support automatic table creation (including table description), automatic creation of table fields (automatic expansion of type and length)
Use Jfinal, mysql
There needs to be a table clientlog to store the original message, and a table to store the table relationship sys_tbl_tree (mysql table name has a length limit, so the real table name and the original table name will be used)
CREATE TABLE `sys_tbl_tree` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`TABLE_NAME` varchar(500) DEFAULT NULL COMMENT '真实表名',
`ORG_NAME` varchar(500) DEFAULT NULL COMMENT 'table原始表名',
`PARENT` varchar(500) DEFAULT NULL,
`TYPE` int(1) DEFAULT '1' COMMENT '1多对1 2 1对1',
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=161 DEFAULT CHARSET=utf8 COMMENT='系统-表结构关系';
CREATE TABLE `clientlog` (
`ID` int(11) NOT NULL AUTO_INCREMENT,
`method` varchar(50) DEFAULT NULL COMMENT '接口方法',
`params` longtext,
PRIMARY KEY (`ID`)
) ENGINE=InnoDB AUTO_INCREMENT=3509 DEFAULT CHARSET=utf8 COMMENT='系统-报文日志';
Parse the message into the database
1. First save the original message in the clientlog table (Clientlog model, please use the official tool provided by jfinal to generate)
2. MysqlTableColumnUtil.init(); only needs to be called once globally, I put it in the onStart method of JFinalConfig
3.new MysqlTableColumnUtil().apiDataInDo(data, bean); data is json message, bean is CheckUserBean
public class CheckUserBean extends LanlinUtil {
/**
* 请求方法中文描述
*/
public String portInfoName;
/**
* json报文
*/
public String params;
/**
* json报文存库
*/
public Clientlog clientlog;
}
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import bean.CheckUserBean;
import com.jfinal.kit.HashKit;
import com.jfinal.kit.PropKit;
import com.jfinal.plugin.activerecord.Db;
import com.jfinal.plugin.activerecord.Record;
public class MysqlTableColumnUtil extends LanlinUtil {
public static final String VARCHAR = "VARCHAR";
public static final String TEXT = "TEXT";
public static final String LONGTEXT = "LONGTEXT";
/**
* 自动处理表全部用这个做主键
*/
public static final String CDEC_UUID = "CDEC_UUID";
public static synchronized void init() {
String table_schema = PropKit.get("table_schema");
List<Record> tables = Db.use("information_schema").find("select TABLE_NAME from TABLES where TABLE_SCHEMA = '" + table_schema + "'");
List<Record> colomns = Db.use("information_schema").find("select TABLE_NAME, COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE from COLUMNS where TABLE_SCHEMA = '" + table_schema + "'");
for(Record table : tables) {
String TABLE_NAME = table.getStr("TABLE_NAME");
List<Record> childs = new LinkedList<Record>();
for(Record cl : colomns) {
String t = cl.getStr("TABLE_NAME");
if(null != t && t.equals(TABLE_NAME)) {
cl.remove("TABLE_NAME");
childs.add(cl);
}
}
table.set("COLOMNS", childs);
}
MysqlTableColumnUtil.tables = tables;
}
// "TABLE_NAME"，"TABLE_COMMENT", "COLOMNS" : List<Record> {"DATA_TYPE","CHARACTER_MAXIMUM_LENGTH","NUMERIC_PRECISION","NUMERIC_SCALE"}
/**
* TINYINT 1 字节 (-128，127) SMALLINT 2 字节 (-32 768，32 767) MEDIUMINT 3 字节 (-8 388 608，8 388 607) INT或INTEGER 4 字节 (-2 147 483 648，2 147 483 647)
* BIGINT 8 字节 (-9 233 372 036 854 775 808，9 223 372 036 854 775 807) FLOAT 4 字节 (-3.402 823 466 E+38，1.175 494 351 E-38)，0，(1.175 494 351
* E-38，3.402 823 466 351 E+38) DOUBLE 8 字节 (1.797 693 134 862 315 7 E+308，2.225 073 858 507 201 4 E-308)，0，(2.225 073 858 507 201 4 E-308，1.797
* 693 134 862 315 7 E+308) DECIMAL 对DECIMAL(M,D) ，如果M>D，为M+2否则为D+2
*/
/**
* DATE 3字节 TIME 3字节 YEAR 1字节 DATETIME 8字节 TIMESTAMP 8字节
*/
/**
* CHAR 255字节 VARCHAR 65535字节 TINYTEXT 255字节 TEXT 65535字节 LONGTEXT 4294967295字节
*/
/**
* {
* "TABLE_NAME" : "表名",
* "TABLE_COMMENT" : "备注",
* "COLOMNS" ： {
* "COLUMN_NAME" : "字段名",
* "DATA_TYPE" : "字段类型"
* "CHARACTER_MAXIMUM_LENGTH" : "varchar,text等最大长度"
* "NUMERIC_PRECISION" : "数值型的长度", "NUMERIC_SCALE" : "数值型的小数位"
* }
* }
* 所有字段统一处理成VARCHAR，TEXT，LONGTEXT
*/
public static List<Record> tables;
/**
* @param tableName
* @return 根据表名获取相关表
*/
public static Record getTable(String tableName) {
tableName = doTblName(tableName);
Record rc = null;
for(Record rr : tables) {
if(rr.getStr("TABLE_NAME").equals(tableName)) {
rc = rr;
break;
}
}
return rc;
}
/**
* @param tableName
* @return 根据字段名获取相关字段
*/
public static Record colomnsHasColomn(List<Record> COLOMNS, String colomn) {
for(Record col : COLOMNS) {
if(col.getStr("COLUMN_NAME").equals(colomn)) {
return col;
}
}
return null;
}
/**
* @param name
* @param length
* @param COLOMNS
* CHAR 255字节 VARCHAR 65535字节 TINYTEXT 255字节 TEXT 65535字节 LONGTEXT 4294967295字节
*/
public void addColomn(String name, Integer length, Record table) {
String tblName = table.getStr("TABLE_NAME");
List<Record> COLOMNS = table.get("COLOMNS");
Record r = new Record();
r.set("COLUMN_NAME", name);
r.set("DATA_TYPE", "varchar");
r.set("CHARACTER_MAXIMUM_LENGTH", length);
COLOMNS.add(r);
if(!name.equals(CDEC_UUID)) {
addTableFieldType(tblName, name, VARCHAR + "(" + length + ")");
}
}
public Object[] getLength(Object value) {
Integer length = null;
try {
length = value.toString().getBytes("UTF-8").length;
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
if(length < 400) {
return new Object[]{VARCHAR, 200};
} else if(length < 800) {
return new Object[]{VARCHAR, 400};
} else if(length < 2000) {
return new Object[]{VARCHAR, 1000};
} else if(length < 4000) {
return new Object[]{VARCHAR, 2000};
} else if(length < 65535) {
return new Object[]{TEXT, null};
} else if(length < 4294967295l) {
return new Object[]{LONGTEXT, null};
}
return new Object[]{VARCHAR, 200};
}
/**
* @param name
* @param type
* CHAR 255字节 VARCHAR 65535字节 TINYTEXT 255字节 TEXT 65535字节 LONGTEXT 4294967295字节
* @throws Exception
*/
public void addColomn(String key, Object value, Record table) {
String tblName = table.getStr("TABLE_NAME");
List<Record> COLOMNS = table.get("COLOMNS");
Record r = new Record();
Object[] xx = getLength(value);
r.set("COLUMN_NAME", key);
r.set("DATA_TYPE", xx[0]);
r.set("CHARACTER_MAXIMUM_LENGTH", xx[1]);
COLOMNS.add(r);
addTableFieldType(tblName, key, xx[0].toString().equals(VARCHAR) ? VARCHAR + "(" + xx[1] + ")" : xx[0].toString());
}
public void apiDataInDo(Object data, CheckUserBean bean) {
apiDataInDo(data, bean.getMethod(), null, null, false, bean);
}
/**
* @param data 数据
* @param tblName 表名
* @param parTblName 上级表名
* @param parentKey 上级主键
* @param isArray 是否1对多
* @param clientLogId log id
* @param portInfoName 请求接口方法备注，放在表备注里面
*/
public void apiDataInDo(Object data, String tblName, String parTblName, String parentKey,
Boolean isArray, CheckUserBean bean) {
tblName = tblName.toLowerCase();
if(null != parTblName) {
parTblName = parTblName.toLowerCase();
}
//数组，继续递归
if (data instanceof JSONArray) {
JSONArray dataArr = (JSONArray) data;
for (int i = 0; i < dataArr.size(); i++) {
Object job = dataArr.get(i);
apiDataInDo(job, tblName, parentKey, parTblName, true, bean);
}
} else if (data instanceof JSONObject) {
Record table = getTable(tblName);
List<Record> COLOMNS = null;
//处理表 不存在新增
if(null == table) {//表不存在
table = new Record();
table.set("TABLE_NAME", doTblName(tblName));
table.set("TABLE_COMMENT", "");
createTable(tblName, parTblName, isArray, bean.portInfoName);//新建一个表
COLOMNS = new ArrayList<Record>();
table.set("COLOMNS", COLOMNS);
addColomn(CDEC_UUID, 50, table);//主键不会新建字段
addColomn("CDEC_PARENTUUID", 50, table);//上级id
addColomn("CDEC_CLIENTLOGID", 50, table);//clientLogId id
tables.add(table);
} else {
COLOMNS = table.get("COLOMNS");
}
//处理表 end
String uuid = UUID();
JSONObject dataObj = (JSONObject) data;
Iterator<?> iterator = dataObj.keys();
Record saveObj = new Record();
saveObj.set(CDEC_UUID, uuid);
saveObj.set("CDEC_CLIENTLOGID", bean.clientlog.getID());
saveObj.set("CDEC_PARENTUUID", parentKey);
while (iterator.hasNext()) {
String key = (String) iterator.next();
Object value = dataObj.get(key);
if(null == value) {
} else if (value instanceof JSONArray) {
JSONArray dataArr = (JSONArray) value;
for (int i = 0; i < dataArr.size(); i++) {
Object job = dataArr.get(i);
apiDataInDo(job, tblName + "_" + key, tblName, uuid, true, bean);
}
} else if (value instanceof JSONObject) {
if(((JSONObject)value).isNullObject()) {//null
return;
}
apiDataInDo(value, tblName + "_" + key, tblName, uuid, false, bean);
} else {//字段
saveObj.set(key, value);
//处理字段 存在、类型、长度start
Record colomn = colomnsHasColomn(COLOMNS, key);
if(null == colomn) {//没有该字段
addColomn(key, value, table);
} else {
Object[] xx = getLength(value);
String type = colomn.getStr("DATA_TYPE");
switch (type.toUpperCase()) {
case "LONGTEXT":
break;
case "TEXT":
if(xx[0].toString().equals("LONGTEXT")) {//更改字段类型
modifyTableFieldType(tblName, key, "LONGTEXT");
}
break;
case "VARCHAR":
if(!xx[0].toString().equals("VARCHAR")) {//更改字段类型
modifyTableFieldType(tblName, key, xx[0].toString());
} else {//判断字段长度
Integer CHARACTER_MAXIMUM_LENGTH = colomn.getInt("CHARACTER_MAXIMUM_LENGTH");
Integer x1 = (Integer)xx[1];
if(CHARACTER_MAXIMUM_LENGTH.compareTo(x1) < 0) {//更改字段长度
modifyTableFieldType(tblName, key, "VARCHAR(" + x1 + ")");
}
}
break;
default:
break;
}
}
//处理字段 存在、类型、长度end
}
}
//需要保存
if(null != saveObj) {
Db.save(doTblName(tblName), CDEC_UUID, saveObj);
}
}
}
/**
* 更改字段类型,长度等
*/
static String modifyTableFieldType = "ALTER TABLE `%s` MODIFY COLUMN `%s` %s CHARACTER SET utf8 COLLATE utf8_general_ci NULL DEFAULT NULL;";
public void modifyTableFieldType(String table, String field, String type) {
Db.update(String.format(modifyTableFieldType, doTblName(table), field, type));
}
/**
* 新增字段
*/
static String addTableFieldType = "ALTER TABLE `%s` ADD COLUMN `%s` %s NULL DEFAULT NULL;;";
public void addTableFieldType(String table, String field, String type) {
Db.update(String.format(addTableFieldType, doTblName(table), field, type));
}
/**
* 新增空表，主键CDEC_UUID
*/
static String addTable = "CREATE TABLE `%s` (`CDEC_UUID` varchar(50) NOT NULL ,PRIMARY KEY (`CDEC_UUID`)) COMMENT='%s';";
public void createTable(String tblName,String parTblName, Boolean isArray, String portInfoMemo) {
String realTblName = doTblName(tblName);
Db.update(String.format(addTable, realTblName, portInfoMemo + "_" + tblName));
//处理表tree结构
Record tblTree = new Record();
tblTree.set("TABLE_NAME", realTblName);
tblTree.set("PARENT", parTblName);
tblTree.set("ORG_NAME", tblName);
tblTree.set("TYPE", isArray ? 1 : 2);
Db.save("sys_tbl_tree", "ID", tblTree);
}
/**
* @param mysql 表最多64字节，超长特殊处理
* @return
*/
public static String doTblName(String tblName) {
if(tblName.getBytes().length > 64) {
tblName = tblName.replace("_", "");
}
if(tblName.getBytes().length > 64) {
tblName = new String(HashKit.sha1(tblName));
}
return tblName;
}
}The effect is as follows
