Test preparation
Database: MySQL Server5.5.40
System: Windows 7, Core(TM) i3-4160 @3.6HZ, Arch x64
Database driver: mysql-connector-java.5.1.9
JDK environment JDK1.7.0_75 x64
Experimental requirements
①Insert 10,000 pieces of data each time, and then delete them, a total of 10 times
②Use MyISAM and InnoDB engines to test separately
③ Set the maximum size of mysql packets to prevent cache leakage max_allowed_packet = 8M
Experimental reference
Test the sql statement precompile effect of mysql
MySQL batch insert performance optimization (1)
MySQL batch insert performance optimization (2)
MySQL batch insert performance optimization (3)
Statement and PreparedStatement batch update
test code
table building code
CREATE TABLE `user` (
`id` int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT '用户主键',
`name` varchar(32) NOT NULL COMMENT '用户名',
`password` varchar(128) NOT NULL COMMENT '密码',
`gender` varchar(8) NOT NULL COMMENT '性别',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=10001 DEFAULT CHARSET=utf8;unit test code
package com.thin.mysql;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Enumeration;
import java.util.Random;
public class StatementTestCase {
public static StatementTestCase instance;
public static String url = "jdbc:mysql://localhost:3306/test?characterEncoding=utf8&useSSL=true&useServerPrepStmts=true";
public Connection conn;
public static String DriverClass = "com.mysql.jdbc.Driver";
int MODE = 1000;
public static StatementTestCase shareInstace() {
if (instance == null) {
synchronized (StatementTestCase.class) {
if (instance == null) {
instance = new StatementTestCase();
}
}
}
instance.openConnection();
return instance;
}
private Driver findDriver(String driverName) {
Enumeration<Driver> drivers = DriverManager.getDrivers();
if (drivers != null) {
do {
if (!drivers.hasMoreElements()) {
break;
}
Driver driver = drivers.nextElement();
if (driver.getClass().getName().equals(driverName)) {
return driver;
}
} while (true);
}
return null;
}
private synchronized void openConnection() {
try {
Driver driver = findDriver(DriverClass);
if (driver == null) {
Class.forName(DriverClass);
}
if (conn == null || conn.isClosed()) {
conn = DriverManager.getConnection(url, "root", "root");
}
} catch (SQLException e) {
e.printStackTrace();
} catch (ClassNotFoundException e) {
e.printStackTrace();
}
}
public void executeStatement01() throws SQLException {
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
Random rnd = new Random(System.currentTimeMillis());
String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
for (int i = 1; i <= 10000; i++) {
int intRand = rnd.nextInt();
name = "'" + "zhangsan" + i + "'";
password = "'" + "pass" + intRand + "'";
intRand = Math.abs((intRand) % 2);
gender = "'" + genders[intRand] + "'";
String nextVal = "(" + i + "," + name + "," + password + ","
+ gender + ")";
sql = statSql + nextVal + ";";
stmt.execute(sql);
}
stmt.execute("delete from user where 1=1;");
conn.commit();
stmt.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executeStatement02() throws SQLException {
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
Random rnd = new Random(System.currentTimeMillis());
String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
for (int i = 1; i <= 10000; i++) {
int intRand = rnd.nextInt();
name = "'" + "zhangsan" + i + "'";
password = "'" + "pass" + intRand + "'";
intRand = Math.abs((intRand) % 2);
gender = "'" + genders[intRand] + "'";
String nextVal = "(" + i + "," + name + "," + password + ","
+ gender + ")";
sql = statSql + nextVal + ";";
stmt.addBatch(sql);
}
stmt.addBatch("delete from user where 1=1;");
stmt.executeBatch();
conn.commit();
stmt.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executeStatement03() throws SQLException {
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
Random rnd = new Random(System.currentTimeMillis());
String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
sql = statSql;
for (int i = 1; i <= 10000; i++) {
int intRand = rnd.nextInt();
name = "'" + "zhangsan" + i + "'";
password = "'" + "pass" + intRand + "'";
intRand = Math.abs((intRand) % 2);
gender = "'" + genders[intRand] + "'";
String nextVal = "(" + i + "," + name + "," + password + ","
+ gender + ")";
sql = sql + nextVal + ",";
}
if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
stmt.addBatch(sql);
}
stmt.addBatch("delete from user where 1=1;");
stmt.executeBatch();
conn.commit();
stmt.close();
System.out.println(sql);
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executeStatement04() throws SQLException {
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
Random rnd = new Random(System.currentTimeMillis());
String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
sql = statSql;
for (int i = 1; i <= 10000; i++) {
int intRand = rnd.nextInt();
name = "'" + "zhangsan" + i + "'";
password = "'" + "pass" + intRand + "'";
intRand = Math.abs((intRand) % 2);
gender = "'" + genders[intRand] + "'";
String nextVal = "(" + i + "," + name + "," + password + ","
+ gender + ")";
sql = sql + nextVal + ",";
if (i % 500 == 0) {
if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
stmt.execute(sql);
}
sql = statSql;
}
}
if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
stmt.execute(sql);
}
stmt.execute("delete from user where 1=1;");
conn.commit();
stmt.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executePrepareStatement01() throws SQLException {
long startTime = System.currentTimeMillis();
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
sql = "REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
PreparedStatement pstat = conn.prepareStatement(sql);
conn.setAutoCommit(false);
Random rnd = new Random(System.currentTimeMillis());
for (int i = 1; i <= 10000; i++) {
int intRand = Math.abs(rnd.nextInt() % 2);
name = "zhangsan" + i;
password = "pass" + intRand;
intRand = Math.abs((intRand) % 2);
gender = genders[intRand];
pstat.clearParameters();
pstat.setInt(1, i);
pstat.setString(2, name);
pstat.setString(3, password);
pstat.setString(4, gender);
pstat.execute();
}
pstat.execute("delete from user where 1=1;");
conn.commit();
pstat.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executePrepareStatement02() throws SQLException {
long startTime = System.currentTimeMillis();
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
sql = "REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
PreparedStatement pstat = conn.prepareStatement(sql);
conn.setAutoCommit(false);
Random rnd = new Random(System.currentTimeMillis());
for (int i = 1; i <= 10000; i++) {
int intRand = Math.abs(rnd.nextInt() % 2);
name = "zhangsan" + i;
password = "pass" + intRand;
intRand = Math.abs((intRand) % 2);
gender = genders[intRand];
pstat.setInt(1, i);
pstat.setString(2, name);
pstat.setString(3, password);
pstat.setString(4, gender);
pstat.addBatch();
}
pstat.addBatch("delete from user where 1=1;");
pstat.executeBatch();
conn.commit();
pstat.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executeStatement04_2() throws SQLException {
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
Random rnd = new Random(System.currentTimeMillis());
String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
sql = statSql;
for (int i = 1; i <= 10000; i++) {
int intRand = rnd.nextInt();
name = "'" + "zhangsan" + i + "'";
password = "'" + "pass" + intRand + "'";
intRand = Math.abs((intRand) % 2);
gender = "'" + genders[intRand] + "'";
String nextVal = "(" + i + "," + name + "," + password + ","
+ gender + ")";
sql = sql + nextVal + ",";
if (i % MODE == 0) {
if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
stmt.execute(sql);
}
sql = statSql;
}
}
if (!sql.equalsIgnoreCase(statSql) && sql.endsWith(",")) {
sql = sql.substring(0, sql.length() - 1);
stmt.execute(sql);
}
stmt.execute("delete from user where 1=1;");
conn.commit();
stmt.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executeStatement02_2() throws SQLException {
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
Random rnd = new Random(System.currentTimeMillis());
String statSql = "REPLACE INTO user(id,name,password,gender) VALUES ";
for (int i = 1; i <= 10000; i++) {
int intRand = rnd.nextInt();
name = "'" + "zhangsan" + i + "'";
password = "'" + "pass" + intRand + "'";
intRand = Math.abs((intRand) % 2);
gender = "'" + genders[intRand] + "'";
String nextVal = "(" + i + "," + name + "," + password + ","
+ gender + ")";
sql = statSql + nextVal + ";";
stmt.addBatch(sql);
if (i % MODE == 0) {
stmt.executeBatch();
}
}
stmt.addBatch("delete from user where 1=1;");
stmt.executeBatch();
conn.commit();
stmt.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
public void executePrepareStatement02_02() throws SQLException {
long startTime = System.currentTimeMillis();
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[] { "男", "女" };
sql = "REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
PreparedStatement pstat = conn.prepareStatement(sql);
conn.setAutoCommit(false);
Random rnd = new Random(System.currentTimeMillis());
for (int i = 1; i <= 10000; i++) {
int intRand = Math.abs(rnd.nextInt() % 2);
name = "zhangsan" + i;
password = "pass" + intRand;
intRand = Math.abs((intRand) % 2);
gender = genders[intRand];
pstat.setInt(1, i);
pstat.setString(2, name);
pstat.setString(3, password);
pstat.setString(4, gender);
pstat.addBatch();
if (i % MODE == 0) {
pstat.executeBatch();
}
}
pstat.addBatch("delete from user where 1=1;");
pstat.executeBatch();
conn.commit();
pstat.close();
System.out.println("final : "
+ (System.currentTimeMillis() - startTime));
}
}
Test calling code example
public static void main( String[] args )
{
try {
System.out.println("executeStatement04_2");
for (int i = 0; i < 10; i++) {
StatementTestCase.shareInstace().executeStatement04_2();
}
} catch (SQLException e) {
e.printStackTrace();
}
}Test Results
executeStatement01
final : 1616
final : 1124
final : 995
final : 1213
final : 1055
final : 1245
final : 1339
final : 1055
final : 1506
final : 1305
executeStatement02
final : 1504
final : 1282
final : 1416
final : 1136
final : 1186
final : 1171
final : 1130
final : 1776
final : 1369
final : 1105
executeStatement03
final : 4700
final : 5141
final : 4223
final : 4308
final : 4200
final : 4257
final : 4459
final : 4935
final : 4198
final : 4274
executeStatement04
final : 4271
final : 4310
final : 4167
final : 4581
final : 4159
final : 4259
final : 4657
final : 4384
final : 4166
final : 4217
executePrepareStatement01
final : 1161
final : 1522
final : 1067
final : 998
final : 982
final : 946
final : 1005
final : 886
final : 1306
final : 896
executePrepareStatement02
final : 1149
final : 1024
final : 1701
final : 1137
final : 892
final : 873
final : 964
final : 1006
final : 1188
final : 948
Summary of results
① Above, we used the InnoDB engine that supports transactions. If we do not use the transaction executeStatement03 and executeStatement04, the speed is the fastest, but it is also around 4200 milliseconds and 4200 milliseconds. It is a waste of time without transaction testing, so we save the time without transaction. result.
② According to the [Test Reference], we can conclude that executeStatement03 and executeStatement04 should be the fastest in theory, but after using the jdbc driver, the effect is not ideal. We have reason to doubt the way jdbc is implemented .
In the database, we execute the SQL statement spliced by executeStatement04 , the efficiency is 155 milliseconds, and other statements are slower.
③The use of transactions can improve efficiency
④ Using the preprocessing prepareStatement method can significantly improve the efficiency
⑤jdbc's addBatch&executeBatch execution efficiency is not stable, but it is generally higher than unused (not obvious)
⑥ The first execution of the prepared statement is very slow.
improvement test
Since the conclusion drawn from the experimental conclusion ② does not meet our expectations, and in addition, according to other conclusions, the batch processing efficiency can be appropriately improved, we transform according to three methods with better batch processing performance under different conditions.
executeStatement02
executeStatement04
executePrepareStatement02
After reviewing the data, we found that
① Properly adjusting the granularity of the batch can also improve performance
② Database connection plus parameter rewriteBatchedStatements=true can improve batch performance
<1> To achieve performance improvement by adjusting the granularity
1. We first set max_allowed_packet = 8M
set global max_allowed_packet = 8*1024*1024
2. Improved code, using MODE to control batch granularity
public void executeStatement04_2() throws SQLException
{
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[]{"男","女"};
Random rnd = new Random(System.currentTimeMillis());
String statSql="REPLACE INTO user(id,name,password,gender) VALUES ";
sql = statSql;
for (int i=1;i<=10000;i++)
{
int intRand = rnd.nextInt();
name = "'"+"zhangsan"+i+"'";
password = "'"+"pass"+intRand+"'";
intRand = Math.abs((intRand)%2);
gender = "'"+genders[intRand]+"'";
String nextVal = "("+i+","+name+","+password+","+gender+")";
sql = sql + nextVal+",";
if(i%MODE==0)
{
if(!sql.equalsIgnoreCase(statSql) && sql.endsWith(","))
{
sql = sql.substring(0,sql.length()-1);
stmt.execute(sql);
}
sql = statSql;
}
}
if(!sql.equalsIgnoreCase(statSql) && sql.endsWith(","))
{
sql = sql.substring(0,sql.length()-1);
stmt.execute(sql);
}
stmt.execute("delete from user where 1=1;");
conn.commit();
stmt.close();
System.out.println("final : "+(System.currentTimeMillis()-startTime));
}
public void executeStatement02_2() throws SQLException
{
long startTime = System.currentTimeMillis();
Statement stmt = conn.createStatement();
conn.setAutoCommit(false);
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[]{"男","女"};
Random rnd = new Random(System.currentTimeMillis());
String statSql="REPLACE INTO user(id,name,password,gender) VALUES ";
for (int i=1;i<=10000;i++)
{
int intRand = rnd.nextInt();
name = "'"+"zhangsan"+i+"'";
password = "'"+"pass"+intRand+"'";
intRand = Math.abs((intRand)%2);
gender = "'"+genders[intRand]+"'";
String nextVal = "("+i+","+name+","+password+","+gender+")";
sql = statSql + nextVal +";";
stmt.addBatch(sql);
if(i%MODE==0)
{
stmt.executeBatch();
}
}
stmt.addBatch("delete from user where 1=1;");
stmt.executeBatch();
conn.commit();
stmt.close();
System.out.println("final : "+(System.currentTimeMillis()-startTime));
}
public void executePrepareStatement02_02() throws SQLException
{
long startTime = System.currentTimeMillis();
String sql = null;
String name = null;
String password = null;
String gender = null;
String[] genders = new String[]{"男","女"};
sql="REPLACE INTO user (id,name,password,gender) VALUE (?,?,?,?)";
PreparedStatement pstat = conn.prepareStatement(sql);
conn.setAutoCommit(false);
Random rnd = new Random(System.currentTimeMillis());
for (int i=1;i<=10000;i++)
{
int intRand = Math.abs(rnd.nextInt()%2);
name = "zhangsan"+i;
password = "pass"+intRand;
intRand = Math.abs((intRand)%2);
gender = genders[intRand];
pstat.setInt(1, i);
pstat.setString(2, name);
pstat.setString(3, password);
pstat.setString(4, gender);
pstat.addBatch();
if(i%MODE==0)
{
pstat.executeBatch();
}
}
pstat.addBatch("delete from user where 1=1;");
pstat.executeBatch();
conn.commit();
pstat.close();
System.out.println("final : "+(System.currentTimeMillis()-startTime));
}Test Results
MODE=500
executeStatement02_2
final : 1558
final : 1228
final : 1070
final : 1477
final : 1002
final : 1024
final : 1003
final : 1129
final : 1415
final : 1306
executeStatement04_2
final : 464
final : 511
final : 538
final : 523
final : 568
final : 396
final : 555
final : 429
final : 672
final : 449
executePrepareStatement02_02
final : 1434
final : 968
final : 991
final : 975
final : 1015
final : 1099
final : 1180
final : 1001
final : 889
final : 1291
MODE=1000
executeStatement02_2
final : 1860
final : 1420
final : 1502
final : 1195
final : 1395
final : 1135
final : 1139
final : 1652
final : 1200
final : 1433
executeStatement04_2
final : 642
final : 888
final : 562
final : 603
final : 853
final : 556
final : 595
final : 820
final : 637
final : 596
executePrepareStatement02_02
final : 1663
final : 1281
final : 1131
final : 1381
final : 1341
final : 1230
final : 1189
final : 1227
final : 1585
final : 1231
The above can be concluded that the MODE is 500, the performance is more outstanding
<2> Modify connection parameters to improve performance
public static String url =”jdbc:mysql://localhost:3306/test?characterEncoding=utf8&useSSL=true&useServerPrepStmts=true&rewriteBatchedStatements=true“We test with mode=500 and open the transaction, and the results are as follows
executeStatement02_2
java.sql.BatchUpdateException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near ';REPLACE INTO user(id,name,password,gender) VALUES (2,'zhangsan2','pass-18490631' at line 1
at com.mysql.jdbc.StatementImpl.executeBatchUsingMultiQueries(StatementImpl.java:1231)
at com.mysql.jdbc.StatementImpl.executeBatch(StatementImpl.java:1012)
at com.yyuap.tianwt.StatementTestCase.executeStatement02_2(StatementTestCase.java:350)
at com.yyuap.tianwt.App.main(App.java:16)
executeStatement04_2
final : 448
final : 487
final : 718
final : 388
final : 505
final : 546
final : 553
final : 796
final : 522
final : 602
executePrepareStatement02_02
final : 997
final : 897
final : 899
final : 932
final : 880
final : 816
final : 882
final : 846
final : 745
final : 758
Conclusion after transformation
①Control the granularity through the above conditional MODE method, batch submission, and the efficiency of addBatch&executeBatch has not been significantly improved
②MODE is the most prominent at 500
③. In the case of .MODE+ modifying the connection parameters, the batch processing of precompiled statements is significantly improved
④Non-precompiled addBatch&executeBatch with rewriteBatchedStatements=true is not supported
final conclusion
Through the above experiments and modified experiments, the final conclusion:
①Multiple Values (such as the executeStatement04_2 case) splicing in a non-transactional state is more efficient than ordinary SQL, which can increase the efficiency by dozens of times
②The use of transactions can increase the efficiency by dozens of times
③ Transactions + multiple Values have the highest batch processing efficiency in the granularity of 500-1000
④addBatch&executeBatch under the condition of precompile prepareStatement, setting rewriteBatchedStatements=true can significantly improve batch performance
⑤ The overall efficiency of precompiled SQL is high, but the first execution efficiency is very low