[SQLite]大量insert
纪录一下过程
问题描述
把通汇银行档(约五千多笔数据)导入系统，发现写入速度太慢，等到天荒地老…
解决方式
将要写入数据库的数据先暂存起来，再一次写到数据库表格，步骤如下：
1. 读取sqlite数据库表格的定义，使用的SQL如下：
读取字段名称及型态
读取字段长度
没有办法直接下SQL取得，采用正规表示式将字段取得。
表格定义
[Filter1] [CHAR](1),
[UnitNo] [CHAR](3),
[GroupNo] [CHAR](4),
[Filter2] [CHAR](1),
[GroupName] [CHAR](40),
[GroupNickName] [CHAR](10),
[Address] [CHAR](40),
[Filter3] [CHAR](1),
[PhoneSecNum] [CHAR](3),
[PhoneNum] [CHAR](8),
[Filter4] [CHAR](1),
[ValueDate] [CHAR](7),
[ChangeType] [CHAR](1),
[DefaultCode] [CHAR](1),
[Remark] [CHAR](2),
[Blank] [CHAR](8)
);正规表示式
".*\[(?
.*)\]\s+\[(?
.*)\]\((?
\d{1,})\).*",
RegexOptions.IgnoreCase
| RegexOptions.CultureInvariant
| RegexOptions.IgnorePatternWhitespace
| RegexOptions.Compiled
);
可以使用工具Expresso进行测试。
2. 使用事务(Transaction)将要写入数据库的数据先暂存起来，再一次写到数据库表格。
程序
using System.Collections.Generic;
using System.Data;
using System.Data.SQLite;
using System.Diagnostics;
using System.Text.RegularExpressions;
using Utility;
namespace DAL
{
///
/// 数据存取层模块
/// 用途：批次数据处理
/// 作者：林大猫
///
public class BatchWork
{
DataAccessLayerSqlite dal = new DataAccessLayerSqlite();
///
/// 插入表格命令
///
/// 表格名
/// where条件
/// 要输入的数据数组
public void InsertTable(string TableName, string[] InData)
{
string sql = string.Empty;
try
{
if (InData.Length == 0)
{
return;
}
string columnName = GetColumnName(TableName);
string columnNameParm = GetColumnNameParm(TableName);
string[] columnNameArray = GetColumnNameArrary(TableName);
sql = " insert into " + TableName + "(" + columnName + " ) values ( " + columnNameParm + " )";
List
sqlParm = new List
();
for (int i = 0; i < InData.Length; i++)
{
sqlParm.Add(new SQLiteParameter(columnNameArray[i], DbType.String));
sqlParm[i].Value = string.Copy(InData[i]);
}
dal.NonQueryResultNoTranc(sql, sqlParm.ToArray());
}
catch (Exception e)
{
throw e;
}
}
///
/// 从文字档写入表格.
///
///
Name of the table.
///
Name of the file.
public void InsertTableFromTextFile(string tableName, string fileName)
{
string sql = string.Empty;
if (fileName.Length == 0)
{
return;
}
int[] format = GetColumnLengths(tableName);
int remainder = 1000;
List
textContent = fileName.ImportText();
string columnName = GetColumnName(tableName);
string columnNameParm = GetColumnNameParm(tableName);
string[] columnNameArray = GetColumnNameArrary(tableName);
sql = " insert into " + tableName + "(" + columnName + " ) values ( " + columnNameParm + " )";
using (SQLiteConnection Sql_Conn = new SQLiteConnection(dal.connectionString))
{
Sql_Conn.Open();
SQLiteTransaction Sql_Transaction;
Sql_Transaction = Sql_Conn.BeginTransaction(System.Data.IsolationLevel.Serializable);
for (int i = 0; i < textContent.Count; i++)
{
List
InData = textContent[i].SplitString(format);
List
sqlParm = new List
(); for (int j = 0; j < format.Length; j++) { sqlParm.Add(new SQLiteParameter(columnNameArray[j], DbType.String)); sqlParm[j].Value = string.Copy(InData[j]); } SQLiteCommand Sql_Command = new SQLiteCommand(sql, Sql_Conn); Sql_Command.Transaction = Sql_Transaction; dal.PrepareCommand(ref Sql_Command, Sql_Conn, sql, sqlParm.ToArray()); Sql_Command.ExecuteNonQuery(); try { if (i == textContent.Count - 1) { Sql_Transaction.Commit(); } else if (i > remainder && i % remainder == 0) { Sql_Transaction.Commit(); Sql_Transaction = Sql_Conn.BeginTransaction(System.Data.IsolationLevel.Serializable); } else { ; } } catch (Exception ex) { Debug.Write(ex.ToString()); Sql_Transaction.Rollback(); } } } } ///
/// 取得字段名称 /// ///
表格名 ///
表格名称字符串用逗号分隔
public static string GetColumnName(string TableName) { DAL.DataAccessLayerSqlite dal = new DAL.DataAccessLayerSqlite(); DataTable dtSource = dal.GetQueryResultToDataTable("PRAGMA table_info( '" + TableName + "' );", null); List
columnNames = new List
(); foreach (DataRow tableInfo in dtSource.Rows) { columnNames.Add(tableInfo["name"].ToString()); } return string.Join(" , ", columnNames.ToArray()); } ///
/// 取得字段名称 /// ///
表格名 ///
表格名称字符串用逗号分隔
public string[] GetColumnNameArrary(string TableName) { DAL.DataAccessLayerSqlite dal = new DAL.DataAccessLayerSqlite(); DataTable dtSource = dal.GetQueryResultToDataTable("PRAGMA table_info( '" + TableName + "' );", null); List
columnNames = new List
(); foreach (DataRow tableInfo in dtSource.Rows) { columnNames.Add(tableInfo["name"].ToString()); } return columnNames.ToArray(); } ///
/// 取得字段名称参数 /// ///
表格名 ///
表格名称字符串用逗号分隔
public string GetColumnNameParm(string TableName) { DAL.DataAccessLayerSqlite dal = new DAL.DataAccessLayerSqlite(); DataTable dtSource = dal.GetQueryResultToDataTable("PRAGMA table_info( '" + TableName + "' );", null); List
columnNames = new List
(); foreach (DataRow tableInfo in dtSource.Rows) { columnNames.Add("@" + tableInfo["name"].ToString()); } return string.Join(" , ", columnNames.ToArray()); } ///
/// 根据表格的Create sql取得字段长度. /// /// ///
Name of the table. ///
.*)\]\s+\[(?
.*)\]\((?
\d{1,})\).*", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.IgnorePatternWhitespace | RegexOptions.Compiled ); SplitItems items = new SplitItems(MyRegex); DAL.DataAccessLayerSqlite dao = new DAL.DataAccessLayerSqlite(); DataTable dt = dao.GetQueryResultToDataTable(DAL.DbInfoDao.GetAllDbInfromation(), null); //return; DataRow[] selectedrows = dt.Select("Trim(TABLE_NAME)='" + TableName + "'"); if (selectedrows.Length > 1 || selectedrows.Length == 0) { throw new Exception("找不到表格名称!"); } List
columnLengths = new List
(); foreach (DataRow dr in selectedrows) { DataTable dbinformation = items.StartSplit(dr["sql"].ToString()); foreach (DataRow dvitem in dbinformation.Rows) { columnLengths.Add(int.Parse(dvitem["Length"].ToString())); } } return columnLengths.ToArray(); } } }
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data.SQLite;
using System.Configuration;
using System.Data;
using System.Diagnostics;
namespace DAL
{
///
/// 读取数据库连线设定(使用sqlite)
///
public class DataAccessLayerSqlite
{
///
/// 连线字符串
///
public string connectionString = string.Empty;
public DataAccessLayerSqlite()
{
connectionString = ConfigurationManager.ConnectionStrings["PCD_RTC_WINFORM.Properties.Settings.PCDDBConnectionString"].ConnectionString;
}
///
/// Prepares the command.
///
/// The CMD.
/// The conn.
/// The cmdtext.
/// The cmdparams.
public void PrepareCommand(ref SQLiteCommand cmd, SQLiteConnection conn, string cmdtext, SQLiteParameter[] cmdparams)
{
if ((conn.State != ConnectionState.Open))
{
conn.Open();
}
cmd.Connection = conn;
cmd.CommandText = cmdtext;
cmd.CommandType = CommandType.Text;
if ((cmdparams != null))
{
foreach (SQLiteParameter paraitem in cmdparams)
{
cmd.Parameters.Add(paraitem);
}
}
}
//取得查询的结果(输出Datatable)
public DataTable GetQueryResultToDataTable(string sqlcmd, SQLiteParameter[] sqlparams)
{
DataTable dtResult = new DataTable();
using (SQLiteConnection Sql_Conn = new SQLiteConnection(connectionString))
{
try
{
SQLiteCommand Sql_Command = new SQLiteCommand(sqlcmd, Sql_Conn);
PrepareCommand(ref Sql_Command, Sql_Conn, sqlcmd, sqlparams);
SQLiteDataAdapter Sql_dataaddapter = new SQLiteDataAdapter(Sql_Command);
Sql_dataaddapter.Fill(dtResult);
Sql_dataaddapter.Dispose();
Sql_Command.Dispose();
return dtResult;
}
catch (Exception ex)
{
Debug.Write(ex.ToString());
return dtResult;
}
}
}
//取得查询的结果(输出DataSet)
public DataSet GetQueryResultToDataset(string sqlcmd, SQLiteParameter[] sqlparams)
{
DataSet dtResult = new DataSet();
using (SQLiteConnection Sql_Conn = new SQLiteConnection(connectionString))
{
try
{
SQLiteCommand Sql_Command = new SQLiteCommand(sqlcmd, Sql_Conn);
PrepareCommand(ref Sql_Command, Sql_Conn, sqlcmd, sqlparams);
SQLiteDataAdapter Sql_dataaddapter = new SQLiteDataAdapter(Sql_Command);
Sql_dataaddapter.Fill(dtResult);
Sql_dataaddapter.Dispose();
Sql_Command.Dispose();
return dtResult;
}
catch (Exception ex)
{
Debug.Write(ex.ToString());
return dtResult;
}
}
}
//取得查询的结果(输出单个数据)
public string GetQueryResultSingleValue(string sqlcmd, SQLiteParameter[] sqlparams)
{
using (SQLiteConnection Sql_Conn = new SQLiteConnection(connectionString))
{
try
{
SQLiteCommand Sql_Command = new SQLiteCommand(sqlcmd, Sql_Conn);
PrepareCommand(ref Sql_Command, Sql_Conn, sqlcmd, sqlparams);
return (Sql_Command.ExecuteScalar() == null) ? string.Empty : Sql_Command.ExecuteScalar().ToString();
}
catch (Exception ex)
{
Debug.Write(ex.ToString());
return string.Empty;
}
}
//return string.Empty;
}
//执行sql命令(没包Transaction)
public string NonQueryResultNoTranc(string sqlcmd, SQLiteParameter[] sqlparams)
{
using (SQLiteConnection Sql_Conn = new SQLiteConnection(connectionString))
{
try
{
SQLiteCommand Sql_Command = new SQLiteCommand(sqlcmd, Sql_Conn);
PrepareCommand(ref Sql_Command, Sql_Conn, sqlcmd, sqlparams);
Sql_Command.ExecuteNonQuery();
Sql_Command.Dispose();
return "OK";
}
catch (Exception ex)
{
throw ex;
}
}
}
//执行sql命令(包Transaction)
public string NonQueryResultTranc(string sqlcmd, SQLiteParameter[] sqlparams)
{
using (SQLiteConnection Sql_Conn = new SQLiteConnection(connectionString))
{
SQLiteCommand Sql_Command = new SQLiteCommand(sqlcmd, Sql_Conn);
PrepareCommand(ref Sql_Command, Sql_Conn, sqlcmd, sqlparams);
SQLiteTransaction Sql_Transaction;
Sql_Transaction = Sql_Conn.BeginTransaction();
Sql_Command.Transaction = Sql_Transaction;
try
{
Sql_Command.ExecuteNonQuery();
Sql_Transaction.Commit();
return "OK";
}
catch (Exception ex)
{
Sql_Transaction.Rollback();
return ex.ToString();
}
}
}
}
}
取得数据库字段长度对象
using System.Collections.Generic;
using System.Data;
using System.Text.RegularExpressions;
namespace Utility
{
///
/// 工具模块
/// 用途：保存正规表示式切割后的字符串群组
/// 作者：林大猫
///
public class SplitItems
{
//格式
private Dictionary
format = new Dictionary
();
//输出结果
private Dictionary
output = new Dictionary
();
private Regex MyRegex;
private int matchcount;
public SplitItems() { }
public SplitItems(Regex inMyRegex)
{
SetDictionary(inMyRegex);
MyRegex = inMyRegex;
}
private void SetValue(int keyindex, string value)
{
int remainder = keyindex % matchcount;
foreach (KeyValuePair
tempvalue in format)
{
if (string.Compare(tempvalue.Value, remainder.ToString()) == 0)
{
output.Add(tempvalue.Key, value);
}
}
}
private DataTable CreateEmptyTable()
{
List
tablecolname = new List
(); foreach (KeyValuePair
tempresult in format) { if (string.Compare(tempresult.Value, "0") != 0) { tablecolname.Add(tempresult.Key); } } return CreateDataTable(tablecolname.ToArray(), null); } private List
GetColumnValues() { List
tablecolvalue = new List
(); foreach (KeyValuePair
tempresult in output) { if (string.Compare(tempresult.Key, "0") != 0) { tablecolvalue.Add(tempresult.Value); } } return tablecolvalue; } ///
/// 建立datatable /// ///
字段名 ///
data数组 , rowmajor ///
DataTable
private DataTable CreateDataTable(string[] ColumnName, string[,] value) { /* 输入范例 string[] cname = new string[]{"name", "sex "}; string[,] aaz = new string[4, 2]; for (int q = 0; q < 4; q++) for (int r = 0; r < 2; r++) aaz[q, r] = "1"; dataGridView1.DataSource = DataUtility.CreateDataTable(cname, aaz); */ int i, j; DataTable kk = new DataTable(); for (i = 0; i < ColumnName.Length; i++) { DataColumn c1 = new DataColumn(ColumnName[i].ToString().Trim(), typeof(string)); kk.Columns.Add(c1); } if (value != null) { for (i = 0; i < value.GetLength(0); i++) { DataRow newrow = kk.NewRow(); for (j = 0; j < ColumnName.Length; j++) { newrow[j] = string.Copy(value[i, j].ToString()); } kk.Rows.Add(newrow); } } return kk; } private void SetDictionary(Regex MyRegex) { // Get the names of all the named and numbered capture groups string[] GroupNames = MyRegex.GetGroupNames(); // Get the numbers of all the named and numbered capture groups int[] GroupNumbers = MyRegex.GetGroupNumbers(); matchcount = GroupNames.Length; for (int i = 0; i < GroupNames.Length; i++) { format.Add(GroupNames[i].ToString(), i.ToString()); } } public DataTable StartSplit(string inputtext) { MatchCollection ms = MyRegex.Matches(inputtext); int count = 0; DataTable dt = CreateEmptyTable(); // 将切割后的结果丢到输出对象当中 foreach (Match temp in ms) { string searchresult = string.Empty; SplitItems tempitem = new SplitItems(MyRegex); foreach (Group grp in temp.Groups) { foreach (Capture cap in grp.Captures) { tempitem.SetValue(count, cap.Value); count++; } } DataRow dr = dt.NewRow(); for (int i = 0; i < dt.Columns.Count; i++) { dr[i] = tempitem.GetColumnValues()[i]; } dt.Rows.Add(dr); } return dt; } } }
文件导入
using System.IO;
using System;
using System.Text;
namespace Utility
{
///
/// 工具模块
/// 用途：文字档处理
/// 作者：林大猫
///
public static class TextFileUtil
{
///
/// Imports the text.
///
/// Name of the input File Name.
///
ImportText(this string InFName)
{
try
{
FileStream fs = new FileStream(InFName, FileMode.Open, FileAccess.Read);
StreamReader sr = new StreamReader(fs, UnicodeEncoding.GetEncoding("Big5"));//转成中文码
List
result = new List
();
while (sr.Peek() >= 0)
{
string temp = sr.ReadLine();
result.Add(temp);
}
sr.Close();
return result;
}
catch (Exception e)
{
Console.Write(e.ToString());
return null;
}
}
}
}
输出结果
真的变快了！从两分多钟减少到不到两秒钟。
修改前
修改后
参考数据
深度解析SQLite批量插入，修改数据库慢的问题
SQLite大量插入的效率问题
[工具]使用Expresso设计正规表示式
How to get a list of column names on sqlite3 / iPhone?