Grist is a tool that supports excel import, data processing, data analysis and visualization. The main body is implemented in node ts language, and its functions are similar to excel. Security of Data Processing. The data is stored in sqlite3, and the api interface of sqlite3 is implemented by C++, which is a customized api interface and supports data DDL, CURD, transaction and Backup functions.
The goal this time is to modify its database storage to mysql8. This modification is not as simple as changing the database connection, but requires many detailed modifications and adjustments.
It is necessary to fully understand the differences between sqlite3 and mysql8, such as: there are differences in the expression of SQL statements, mainly in:
1. The difference in data type requires modifying the table creation statement, that is, the DDL statement, which is mainly reflected in the fact that some Text needs to be modified to Varchar(256), which mainly supports
2. The CURD statement needs to be modified;
3. It is necessary to implement the mysql interface according to the sqlite3 interface, including:
a. You need to choose a mysql interface that supports promise. By comparing mysql, mysql2 and mysqlx, you must finally use mysql2/promise
b. It is necessary to marshall the queried data, because the sqlite3 data is marshalled by default in the C++ interface, so that the data can be displayed normally in grist, that is, the data is marshalled into Buffer type.
c. Need to support transactions, transactions do not support direct implementation through sql commands, but use the transaction method of connection;
d. The queried data needs to be converted into a data format similar to sqlite3, so as to directly obtain the value of the data field, such as: ResultRow type;
e. In terms of data insertion, in order to improve performance, it is necessary to use arrays for batch insertion, and the performance can reach 20,000+ records/second;
f. In terms of data update, in order to improve performance, it is necessary to use batch update method to update, and the performance can reach 100,000+ records/second;
g. SQL statement conversion, you need to pay attention to the conversion of keywords such as table name and field name ", group, etc., Json class data "{}", you need to change the double quotes to single quotes '{}' for storage ;
h. Data backup cannot use the sqldump tool, but can only be realized by table through sql (lower efficiency than sqldump).
Attach some core codes for reference (including transaction, batch insert, batch update, single query, batch query, backup, batch marsall):
public async getConnection(doc_name:string):Promise<mysql.Connection>
{
const host = process.env.MYSQL_HOST?process.env.MYSQL_HOST:'localhost'
const port = process.env.MYSQL_PORT?process.env.MYSQL_PORT:'3310'
const user = process.env.MYSQL_USER?process.env.MYSQL_USER:'root'
const password = process.env.MYSQL_PASSWORD?process.env.MYSQL_PASSWORD:'123456789'
const database = process.env.MYSQL_DATABASE?process.env.MYSQL_DATABASE:'mysql'
var conn = mysql.createConnection(`mysql://${user}:${password}@${host}:${port}/${database}`);
(await conn).execute("create database if not exists `"+doc_name+"`");
(await conn).end();
return mysql.createConnection(`mysql://${user}:${password}@${host}:${port}/${doc_name}?multipleStatements=true`);
}public async backup(src:string, dest:string):Promise<void>{
const dest_name = replaceAll(src+'-',"",dest);
console.log('backup src:',src);
console.log('backup dest:',dest);
console.log('backup:dest_name:',dest_name);
const dest_db = MySQLDB.openDBRaw(dest);
//获取所有表
const tblRows = await this.all(`show tables from ${this.doc_name}`)
for (const tblRow of tblRows) {
for (const key in tblRow){
const createTblRows = await this.all(`show create table ${tblRow[key]}`)
for (const createTbl of createTblRows){
for (const key2 in createTbl){
if (createTbl[key2]!=tblRow[key])
{
const create_sql = createTbl[key2];
await (await dest_db).exec(create_sql);
const sql = `insert into \`${dest_name}\`.${tblRow[key]} select * from ${tblRow[key]}`;
await this.exec(sql);
}
}
}
}
}
return
}public async run(sql: string, ...params: any[]): Promise<mysql.OkPacket[]> {
const db = await this.getConnection(this.doc_name)
sql = replaceAll("\"group\"","`group`",sql);
sql = replaceAll(" group ","`group`",sql);
sql = replaceAll("group=","`group`=",sql); //mysql 中group为关键字
sql = replaceAll("1e999","0",sql);
console.log('run sql:',sql, 'params:',params);
var rows:mysql.OkPacket[]
if (params.length>0)
if (typeof params[0]==='object')
[rows,,] =await db.query(sql,params[0]);
else
[rows,,] =await db.query(sql,params);
else
[rows,,] =await db.query(sql);
//console.log('rows:',rows)
//console.log('fields:',fields)
db.destroy()
return rows;
}public async get(sql:string, ...params: any[]):Promise<ResultRow> {
const db = await this.getConnection(this.doc_name)
console.log('get sql:',sql, 'params:',params);
var rows:mysql.RowDataPacket[]
if (params.length>0)
{
[rows,,] = (await db.query(sql,params));
}
else{
[rows,,] = (await db.query(sql));
}
//console.log('rows:',rows)
//console.log('fields:',fields)
var row:ResultRow = {};
for(var line of rows)
{
row = line;
}
// console.log('get row:',row);
db.destroy()
return row;
}
public async all(sql: string, ...params: any[]): Promise<ResultRow[]> {
const db = await this.getConnection(this.doc_name)
//console.log('all sql:',sql, 'params:',params);
var rows:mysql.RowDataPacket[]
if (params.length>0)
{
[rows, ,] = (await db.query(sql,params));
}
else{
[rows, ,] = (await db.query(sql));
}
//console.log('fields:',fields)
var table:ResultRow[]=[];
for (var line of rows)
{
var row:ResultRow ={}
row = line;
table.push(row);
}
//console.log('all table:',table)
db.destroy()
return table;
}public async allMarshal(sql: string, ...params: any[]): Promise<Buffer> {
const db = await this.getConnection(this.doc_name)
console.log('allMarshal sql:',sql, 'params:',params);
var rows:mysql.RowDataPacket[], fields:mysql.FieldPacket[]
if (params.length>0&¶ms[0].length>0)
{
[rows, fields] = (await db.query(sql, params));
}
else{
[rows, fields] = (await db.query(sql));
}
//console.log('allMarshal fields:',fields)
const marshaller = new marshal.Marshaller({version: 2});
var table:{[key:string]:any[]} = {}
if (rows.length>0)
{
for(var line of rows)
{
for(var col of fields)
{
var col_value = line[col.name];
var values = table[col.name];
if (values === undefined)
values=[]
if (Buffer.isBuffer(col_value))//解决BLOB的解码问题
if (isNumber(col_value.toString()))
values.push(Number(col_value.toString()))
else
values.push(col_value.toString())
else
values.push(col_value);
table[col.name]= values;
}
}
}
else
{
for(var col of fields)
{
table[col.name]= [];
}
}
marshaller.marshal(table);
const buf = marshaller.dump();
db.destroy()
return Buffer.from(buf);
}public async prepare(sql: string, ...params: any[]): Promise<any> {
const db = await this.getConnection(this.doc_name)
sql = replaceAll("\"group\"","`group`",sql);
sql = replaceAll(" group ","`group`",sql);
sql = replaceAll("group=","`group`=",sql); //mysql 中group为关键字
sql = replaceAll("1e999","0",sql);
console.log('prepare sql:',sql, 'params[0].length:',params[0].length);
var start = process.uptime();
var end = process.uptime();
var begin = process.uptime();
var diff = 1;
if (params.length>0)
//批量插入"INSERT INTO TABLE() VALUES ?",其中?是个包含数组的数组[params[0]]
if (/INSERT INTO/.test(sql))
{
await db.query(sql,[params[0]]);
}
else//批量更新
{
if (typeof params[0]==='object')
{
//sql likes "update table field=? where id in ?"
//toparams likes [`case id when 1 then "A" end`, [[1,2,3]]]
const UPDATE_NUM = 200;
var field_cases:{[key:string]:string}={}
var id_ins = []
var i = 0
var left = 0
var toparams =[]
for(var param of params[0])
{
var fields:{[key:string]:string|number|boolean} = param[0];
var ids:{[key:string]:number} = param[1];
id_ins.push(ids['id']);
for (var key in fields)
{
if (field_cases[key] === undefined)
{
if (typeof fields[key] ==='string')
field_cases[key] = `case id when ${ids['id']} then '${fields[key]}' `
else
field_cases[key] = `case id when ${ids['id']} then ${fields[key]} `
}
else
{
if (typeof fields[key] ==='string')
field_cases[key] = field_cases[key].concat(`when ${ids['id']} then '${fields[key]}' `)
else
field_cases[key] = field_cases[key].concat(`when ${ids['id']} then ${fields[key]} `)
}
}
i = i + 1;
left = i%UPDATE_NUM;
if (left===0)
{
for (var key in field_cases)
{
toparams.push(field_cases[key].concat(' end'));
}
toparams.push([id_ins]);
sql = mysql.format(sql, toparams);
sql = replaceAll("\'case","case",sql);
sql = replaceAll("end\'","end",sql);
sql = replaceAll("\\","",sql);
//console.log('sql:',sql);
await db.query(sql);
end = process.uptime()
diff = end-start
//console.log('提交 i:',i, '执行速度:', Math.round(UPDATE_NUM/diff),'/秒');
start = process.uptime();
field_cases = {}
toparams = []
id_ins = []
}
}
//剩下的
if (left>0)
{
for (var key in field_cases)
{
toparams.push(field_cases[key].concat(' end'));
}
toparams.push([id_ins]);
sql = mysql.format(sql, toparams);
sql = replaceAll("\'case","case",sql);
sql = replaceAll("end\'","end",sql);
sql = replaceAll("\\","",sql);
//console.log('sql:',sql);
await db.query(sql);
end = process.uptime()
diff = end-start
//console.log('提交 i:',i, '执行速度:', Math.round(left/diff),'/秒');
}
}
else
{
console.log('params:',params)
await db.execute(sql,params);
}
}
else
await db.query(sql);
end = process.uptime()
diff = end-begin
console.log('prepare sql:', params[0].length, '执行速度:', Math.round(params[0].length/diff),'/秒');
db.destroy()
return
}private async _execTransactionImpl<T>(callback: () => Promise<T>): Promise<T> {
// We need to swallow errors, so that one failed transaction doesn't cause the next one to fail.
await this._prevTransaction.catch(noop);
const db = await this.getConnection(this.doc_name)
await db.beginTransaction()
try {
const value = await callback();
await db.commit()
return value;
} catch (err) {
try {
await db.rollback()
} catch (rollbackErr) {
log.error("MySQLDB[%s]: Rollback failed: %s", this._dbPath, rollbackErr);
}
db.destroy()
throw err; // Throw the original error from the transaction.
}
}