grist是一款支持excel导入，支持数据处理、数据分析及可视化工具，主体使用node ts语言实现，功能与excel类似，差别在于数据存储在服务端，数据处理通过沙箱中的python实现，保证了数据处理的安全。数据存储在sqlite3中，其中sqlite3的api接口是通过C++实现的，是个定制化的api接口，支持数据的DDL、CURD、事务及Backup功能。
本次的目标是希望能将其数据库存储修改为mysql8，这个修改不仅仅是换个数据库连接那么简单，而是要做很多细节的修改和调整。
需要充分了解sqlite3和mysql8的差异，如：SQL语句表达上存在差异，主要表现在：
1.数据类型的差异，需要修改表创建语句，即DDL语句，主要体现在部分Text需要修改为Varchar(256)，主要支持
2.需要修改CURD语句；
3.需要根据sqlite3的接口，实现mysql的接口，包括：
a.需要选择支持promise的mysql接口，通过对比mysql,mysql2和mysqlx，最后确定使用mysql2/promise
b.需要对查询出来的数据进行marshall，因为sqlite3的数据在C++接口中默认是marshall过的，这样数据在grist中才能正常显示，即将数据marshall成Buffer类型。
c.需要支持事务，事务不支持通过sql命令直接实现，而是使用的connection的事务方法；
d.查询出来的数据需要转化为sqlite3类似的数据格式，以便直接获取数据字段的值，如：ResultRow类型；
e.数据插入方面，为了提升性能，需要使用数组的方式进行批量插入，性能可以达到2万+条/秒；
f.数据更新方面，为了提升性能，需要使用批量更新的方式进行更新，性能可以达到10万+条/秒；
g.SQL语句转换上，需要注意去掉表名、字段名的双引号"、group等关键字的转换，Json类数据"{}"，需要将双引号修改为单引号'{}'，进行存储；
h.数据备份不能使用sqldump工具，只能通过sql按表进行实现（效率比sqldump低）。
附上部分核心代码供参考（包括事务、批量插入、批量更新、单条查询、批量查询、备份、批量marsall）：
public async getConnection(doc_name:string):Promise<mysql.Connection>
{
const host = process.env.MYSQL_HOST?process.env.MYSQL_HOST:'localhost'
const port = process.env.MYSQL_PORT?process.env.MYSQL_PORT:'3310'
const user = process.env.MYSQL_USER?process.env.MYSQL_USER:'root'
const password = process.env.MYSQL_PASSWORD?process.env.MYSQL_PASSWORD:'123456789'
const database = process.env.MYSQL_DATABASE?process.env.MYSQL_DATABASE:'mysql'
var conn = mysql.createConnection(`mysql://${user}:${password}@${host}:${port}/${database}`);
(await conn).execute("create database if not exists `"+doc_name+"`");
(await conn).end();
return mysql.createConnection(`mysql://${user}:${password}@${host}:${port}/${doc_name}?multipleStatements=true`);
}public async backup(src:string, dest:string):Promise<void>{
const dest_name = replaceAll(src+'-',"",dest);
console.log('backup src:',src);
console.log('backup dest:',dest);
console.log('backup:dest_name:',dest_name);
const dest_db = MySQLDB.openDBRaw(dest);
//获取所有表
const tblRows = await this.all(`show tables from ${this.doc_name}`)
for (const tblRow of tblRows) {
for (const key in tblRow){
const createTblRows = await this.all(`show create table ${tblRow[key]}`)
for (const createTbl of createTblRows){
for (const key2 in createTbl){
if (createTbl[key2]!=tblRow[key])
{
const create_sql = createTbl[key2];
await (await dest_db).exec(create_sql);
const sql = `insert into \`${dest_name}\`.${tblRow[key]} select * from ${tblRow[key]}`;
await this.exec(sql);
}
}
}
}
}
return
}public async run(sql: string, ...params: any[]): Promise<mysql.OkPacket[]> {
const db = await this.getConnection(this.doc_name)
sql = replaceAll("\"group\"","`group`",sql);
sql = replaceAll(" group ","`group`",sql);
sql = replaceAll("group=","`group`=",sql); //mysql 中group为关键字
sql = replaceAll("1e999","0",sql);
console.log('run sql:',sql, 'params:',params);
var rows:mysql.OkPacket[]
if (params.length>0)
if (typeof params[0]==='object')
[rows,,] =await db.query(sql,params[0]);
else
[rows,,] =await db.query(sql,params);
else
[rows,,] =await db.query(sql);
//console.log('rows:',rows)
//console.log('fields:',fields)
db.destroy()
return rows;
}public async get(sql:string, ...params: any[]):Promise<ResultRow> {
const db = await this.getConnection(this.doc_name)
console.log('get sql:',sql, 'params:',params);
var rows:mysql.RowDataPacket[]
if (params.length>0)
{
[rows,,] = (await db.query(sql,params));
}
else{
[rows,,] = (await db.query(sql));
}
//console.log('rows:',rows)
//console.log('fields:',fields)
var row:ResultRow = {};
for(var line of rows)
{
row = line;
}
// console.log('get row:',row);
db.destroy()
return row;
}
public async all(sql: string, ...params: any[]): Promise<ResultRow[]> {
const db = await this.getConnection(this.doc_name)
//console.log('all sql:',sql, 'params:',params);
var rows:mysql.RowDataPacket[]
if (params.length>0)
{
[rows, ,] = (await db.query(sql,params));
}
else{
[rows, ,] = (await db.query(sql));
}
//console.log('fields:',fields)
var table:ResultRow[]=[];
for (var line of rows)
{
var row:ResultRow ={}
row = line;
table.push(row);
}
//console.log('all table:',table)
db.destroy()
return table;
}public async allMarshal(sql: string, ...params: any[]): Promise<Buffer> {
const db = await this.getConnection(this.doc_name)
console.log('allMarshal sql:',sql, 'params:',params);
var rows:mysql.RowDataPacket[], fields:mysql.FieldPacket[]
if (params.length>0&¶ms[0].length>0)
{
[rows, fields] = (await db.query(sql, params));
}
else{
[rows, fields] = (await db.query(sql));
}
//console.log('allMarshal fields:',fields)
const marshaller = new marshal.Marshaller({version: 2});
var table:{[key:string]:any[]} = {}
if (rows.length>0)
{
for(var line of rows)
{
for(var col of fields)
{
var col_value = line[col.name];
var values = table[col.name];
if (values === undefined)
values=[]
if (Buffer.isBuffer(col_value))//解决BLOB的解码问题
if (isNumber(col_value.toString()))
values.push(Number(col_value.toString()))
else
values.push(col_value.toString())
else
values.push(col_value);
table[col.name]= values;
}
}
}
else
{
for(var col of fields)
{
table[col.name]= [];
}
}
marshaller.marshal(table);
const buf = marshaller.dump();
db.destroy()
return Buffer.from(buf);
}public async prepare(sql: string, ...params: any[]): Promise<any> {
const db = await this.getConnection(this.doc_name)
sql = replaceAll("\"group\"","`group`",sql);
sql = replaceAll(" group ","`group`",sql);
sql = replaceAll("group=","`group`=",sql); //mysql 中group为关键字
sql = replaceAll("1e999","0",sql);
console.log('prepare sql:',sql, 'params[0].length:',params[0].length);
var start = process.uptime();
var end = process.uptime();
var begin = process.uptime();
var diff = 1;
if (params.length>0)
//批量插入"INSERT INTO TABLE() VALUES ?",其中?是个包含数组的数组[params[0]]
if (/INSERT INTO/.test(sql))
{
await db.query(sql,[params[0]]);
}
else//批量更新
{
if (typeof params[0]==='object')
{
//sql likes "update table field=? where id in ?"
//toparams likes [`case id when 1 then "A" end`, [[1,2,3]]]
const UPDATE_NUM = 200;
var field_cases:{[key:string]:string}={}
var id_ins = []
var i = 0
var left = 0
var toparams =[]
for(var param of params[0])
{
var fields:{[key:string]:string|number|boolean} = param[0];
var ids:{[key:string]:number} = param[1];
id_ins.push(ids['id']);
for (var key in fields)
{
if (field_cases[key] === undefined)
{
if (typeof fields[key] ==='string')
field_cases[key] = `case id when ${ids['id']} then '${fields[key]}' `
else
field_cases[key] = `case id when ${ids['id']} then ${fields[key]} `
}
else
{
if (typeof fields[key] ==='string')
field_cases[key] = field_cases[key].concat(`when ${ids['id']} then '${fields[key]}' `)
else
field_cases[key] = field_cases[key].concat(`when ${ids['id']} then ${fields[key]} `)
}
}
i = i + 1;
left = i%UPDATE_NUM;
if (left===0)
{
for (var key in field_cases)
{
toparams.push(field_cases[key].concat(' end'));
}
toparams.push([id_ins]);
sql = mysql.format(sql, toparams);
sql = replaceAll("\'case","case",sql);
sql = replaceAll("end\'","end",sql);
sql = replaceAll("\\","",sql);
//console.log('sql:',sql);
await db.query(sql);
end = process.uptime()
diff = end-start
//console.log('提交 i:',i, '执行速度:', Math.round(UPDATE_NUM/diff),'/秒');
start = process.uptime();
field_cases = {}
toparams = []
id_ins = []
}
}
//剩下的
if (left>0)
{
for (var key in field_cases)
{
toparams.push(field_cases[key].concat(' end'));
}
toparams.push([id_ins]);
sql = mysql.format(sql, toparams);
sql = replaceAll("\'case","case",sql);
sql = replaceAll("end\'","end",sql);
sql = replaceAll("\\","",sql);
//console.log('sql:',sql);
await db.query(sql);
end = process.uptime()
diff = end-start
//console.log('提交 i:',i, '执行速度:', Math.round(left/diff),'/秒');
}
}
else
{
console.log('params:',params)
await db.execute(sql,params);
}
}
else
await db.query(sql);
end = process.uptime()
diff = end-begin
console.log('prepare sql:', params[0].length, '执行速度:', Math.round(params[0].length/diff),'/秒');
db.destroy()
return
}private async _execTransactionImpl<T>(callback: () => Promise<T>): Promise<T> {
// We need to swallow errors, so that one failed transaction doesn't cause the next one to fail.
await this._prevTransaction.catch(noop);
const db = await this.getConnection(this.doc_name)
await db.beginTransaction()
try {
const value = await callback();
await db.commit()
return value;
} catch (err) {
try {
await db.rollback()
} catch (rollbackErr) {
log.error("MySQLDB[%s]: Rollback failed: %s", this._dbPath, rollbackErr);
}
db.destroy()
throw err; // Throw the original error from the transaction.
}
}