数据库链接长时间无数据交互，发生线程阻塞情况

背景：

 在执行双机房部署的时候，因为应用长时间未访问数据库，导致后面访问的数据库的线程都被挂起。

现象分析：

Java代码 

1. "Thread-74" daemon prio=10 tid=0x00007f1840044000 nid=0x387b runnable [0x00007f18bdb27000]  
2.    java.lang.Thread.State: RUNNABLE  
3.         at java.net.SocketInputStream.socketRead0(Native Method)  
4.         at java.net.SocketInputStream.read(SocketInputStream.java:129)  
5.         at oracle.net.ns.Packet.receive(Unknown Source)  
6.         at oracle.net.ns.DataPacket.receive(Unknown Source)  
7.         at oracle.net.ns.NetInputStream.getNextPacket(Unknown Source)  
8.         at oracle.net.ns.NetInputStream.read(Unknown Source)  
9.         at oracle.net.ns.NetInputStream.read(Unknown Source)  
10.         at oracle.net.ns.NetInputStream.read(Unknown Source)  
11.         at oracle.jdbc.driver.T4CMAREngine.unmarshalUB1(T4CMAREngine.java:1109)  
12.         at oracle.jdbc.driver.T4CMAREngine.unmarshalSB1(T4CMAREngine.java:1080)  
13.         at oracle.jdbc.driver.T4C8Oall.receive(T4C8Oall.java:485)  
14.         at oracle.jdbc.driver.T4CStatement.doOall8(T4CStatement.java:210)  
15.         at oracle.jdbc.driver.T4CStatement.executeForDescribe(T4CStatement.java:804)  
16.         at oracle.jdbc.driver.OracleStatement.executeMaybeDescribe(OracleStatement.java:1051)  
17.         at oracle.jdbc.driver.T4CStatement.executeMaybeDescribe(T4CStatement.java:845)  
18.         at oracle.jdbc.driver.OracleStatement.doExecuteWithTimeout(OracleStatement.java:1156)  
19.         at oracle.jdbc.driver.OracleStatement.executeQuery(OracleStatement.java:1315)  
20.         - locked <0x00000000be528248> (a oracle.jdbc.driver.T4CStatement)  
21.         - locked <0x00000000c4d1b000> (a oracle.jdbc.driver.T4CConnection)  
22.         at oracle.jdbc.driver.PhysicalConnection.doPingDatabase(PhysicalConnection.java:4614)  
23.         at oracle.jdbc.driver.PhysicalConnection$1.run(PhysicalConnection.java:4590)  
24.         at java.lang.Thread.run(Thread.java:662)  
25.   
26.    Locked ownable synchronizers:  
27.   
28. DubboServerHandler-172.21.55.25:20883-thread-13" daemon prio=10 tid=0x00007f1828012800 nid=0x3867 in Object.wait() [0x00007f18be305000]  
29.    java.lang.Thread.State: TIMED_WAITING (on object monitor)  
30.         at java.lang.Object.wait(Native Method)  
31.         - waiting on <0x00000000be51e0b0> (a java.lang.Thread)  
32.         at java.lang.Thread.join(Thread.java:1194)  
33.         - locked <0x00000000be51e0b0> (a java.lang.Thread)  
34.         at oracle.jdbc.driver.PhysicalConnection.pingDatabase(PhysicalConnection.java:4596)  
35.         at sun.reflect.GeneratedMethodAccessor3.invoke(Unknown Source)  
36.         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)  
37.         at java.lang.reflect.Method.invoke(Method.java:597)  
38.         at com.alibaba.druid.pool.vendor.OracleValidConnectionChecker.isValidConnection(OracleValidConnectionChecker.java:94)  
39.         at com.alibaba.druid.pool.DruidAbstractDataSource.testConnectionInternal(DruidAbstractDataSource.java:1252)  
40.         at com.alibaba.druid.pool.DruidDataSource.getConnectionDirect(DruidDataSource.java:954)  
41.         at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:921)  
42.         at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:911)  
43.         at com.alibaba.druid.pool.DruidDataSource.getConnection(DruidDataSource.java:98)  
44.         at org.springframework.jdbc.datasource.DataSourceUtils.doGetConnection(DataSourceUtils.java:113)  
45.         at org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy$TransactionAwareInvocationHandler.invoke(TransactionAwareDataSourceProxy.java:210)  
46.         at $Proxy25.toString(Unknown Source)  
47.         at java.lang.String.valueOf(String.java:2826)  
48.         at java.lang.StringBuffer.append(StringBuffer.java:219)  
49.         - locked <0x00000000be4cd210> (a java.lang.StringBuffer)  
50.         at org.springframework.orm.ibatis.SqlMapClientTemplate.execute(SqlMapClientTemplate.java:194)  
51.         at org.springframework.orm.ibatis.SqlMapClientTemplate.executeWithListResult(SqlMapClientTemplate.java:249)  
52.         at org.springframework.orm.ibatis.SqlMapClientTemplate.queryForList(SqlMapClientTemplate.java:296)  
53.         at com.lianpay.bank.mng.dao.impl.BaseIbatisDAOImpl.queryAll(BaseIbatisDAOImpl.java:123)  
54.         at com.lianpay.bank.mng.service.impl.QueryBankListServiceImpl.queryPayTypeList(QueryBankListServiceImpl.java:319)  
55.         at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)  
56.         at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)  
57.         at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)  

    我们可以发现线程thread-13一直被阻塞在pingDatabase方法调用中，而线程Thread-74一直卡在socketRead0，初步分析是没有设置socket timeout时间。

解决：

参考文档：

http://agapple.iteye.com/blog/772507 

http://www.importnew.com/2466.html

http://agapple.iteye.com/blog/1024508

查看上面两个参考文档之后，发现需要配置validationQueryTimeout，oracle.net.CONNECT_TIMEOUT=3000&oracle.net.READ_TIMEOUT=60000这几个参数。

源码分析：

问题1：我们设置了maxWait参数，为什么获取链接的时候未超时，报异常

问题2：我们设置了validationQuery，为什么没有定期发送ping包过去保活

   一： maxWait参数主要作用在getConnectionInternal，而我们现在是卡在testConnectionInternal中，表示是已经获取到connection，但是卡在验证connection的有效性中。

Java代码 

1. public DruidPooledConnection getConnectionDirect(long maxWaitMillis) throws SQLException {  
2.       int notFullTimeoutRetryCnt = 0;  
3.       for (;;) {  
4.           // handle notFullTimeoutRetry  
5.           DruidPooledConnection poolableConnection;  
6.           try {  
7.               poolableConnection = getConnectionInternal(maxWaitMillis);  
8.           } catch (GetConnectionTimeoutException ex) {  
9.               if (notFullTimeoutRetryCnt <= this.notFullTimeoutRetryCount && !isFull()) {  
10.                   notFullTimeoutRetryCnt++;  
11.                   if (LOG.isWarnEnabled()) {  
12.                       LOG.warn("not full timeout retry : " + notFullTimeoutRetryCnt);  
13.                   }  
14.                   continue;  
15.               }  
16.               throw ex;  
17.           }  
18.   
19.           if (isTestOnBorrow()) {  
20.               boolean validate = testConnectionInternal(poolableConnection.getConnection());  
21.               if (!validate) {  
22.                   if (LOG.isDebugEnabled()) {  
23.                       LOG.debug("skip not validate connection.");  
24.                   }  
25.   
26.                   Connection realConnection = poolableConnection.getConnection();  
27.                   discardConnection(realConnection);  
28.                   continue;  
29.               }  

   

    2. validationQuery不是用来保证socket长链接不断开的。validationQuery的调用有两处，一是创建链接的时候，二是获取到链接的时候，会验证有效性。

Java代码 

1. protected boolean testConnectionInternal(Connection conn) {  
2.        String sqlFile = JdbcSqlStat.getContextSqlFile();  
3.        String sqlName = JdbcSqlStat.getContextSqlName();  
4.   
5.        if (sqlFile != null) {  
6.            JdbcSqlStat.setContextSqlFile(null);  
7.        }  
8.        if (sqlName != null) {  
9.            JdbcSqlStat.setContextSqlName(null);  
10.        }  
11.        try {  
12.            if (validConnectionChecker != null) {  
13.                return validConnectionChecker.isValidConnection(conn, validationQuery, validationQueryTimeout);  
14.            }  
15.   
16.            if (conn.isClosed()) {  
17.                return false;  
18.            }  
19.   
20.            if (null == validationQuery) {  
21.                return true;  
22.            }  
23.   
24.            Statement stmt = null;  
25.            ResultSet rset = null;  
26.            try {  
27.                stmt = conn.createStatement();  
28.                if (getValidationQueryTimeout() > 0) {  
29.                    stmt.setQueryTimeout(validationQueryTimeout);  
30.                }  
31.                rset = stmt.executeQuery(validationQuery);  
32.                if (!rset.next()) {  
33.                    return false;  
34.                }  
35.            } finally {  
36.                JdbcUtils.close(rset);  
37.                JdbcUtils.close(stmt);  
38.            }  
39.   
40.            return true;  
41.        } catch (Exception ex) {  
42.            // skip  
43.            return false;  
44.        } finally {  
45.            if (sqlFile != null) {  
46.                JdbcSqlStat.setContextSqlFile(sqlFile);  
47.            }  
48.            if (sqlName != null) {  
49.                JdbcSqlStat.setContextSqlName(sqlName);  
50.            }  
51.        }  
52.    }  

    如果socket链接断开不可用，也不会导致应用异常，druid框架会不断的重试，抛弃不可用链接，直到拿到可用的链接

Java代码 

1. public DruidPooledConnection getConnectionDirect(long maxWaitMillis) throws SQLException {  
2.        int notFullTimeoutRetryCnt = 0;  
3.        for (;;) {  
4.            // handle notFullTimeoutRetry  
5.            DruidPooledConnection poolableConnection;  
6.            try {  
7.                poolableConnection = getConnectionInternal(maxWaitMillis);  
8.            } catch (GetConnectionTimeoutException ex) {  
9.                if (notFullTimeoutRetryCnt <= this.notFullTimeoutRetryCount && !isFull()) {  
10.                    notFullTimeoutRetryCnt++;  
11.                    if (LOG.isWarnEnabled()) {  
12.                        LOG.warn("not full timeout retry : " + notFullTimeoutRetryCnt);  
13.                    }  
14.                    continue;  
15.                }  
16.                throw ex;  
17.            }  
18.   
19.            if (isTestOnBorrow()) {  
20.                boolean validate = testConnectionInternal(poolableConnection.getConnection());  
21.                if (!validate) {  
22.                    if (LOG.isDebugEnabled()) {  
23.                        LOG.debug("skip not validate connection.");  
24.                    }  
25.   
26.                    Connection realConnection = poolableConnection.getConnection();  
27.                    discardConnection(realConnection);  
28.                    continue;  
29.                }  
30.            } else {  
31.                Connection realConnection = poolableConnection.getConnection();  
32.                if (realConnection.isClosed()) {  
33.                    discardConnection(null); // 传入null，避免重复关闭  
34.                    continue;  
35.                }  
36.   
37.                if (isTestWhileIdle()) {  
38.                    final long currentTimeMillis = System.currentTimeMillis();  
39.                    final long lastActiveTimeMillis = poolableConnection.getConnectionHolder().getLastActiveTimeMillis();  
40.                    final long idleMillis = currentTimeMillis - lastActiveTimeMillis;  
41.                    long timeBetweenEvictionRunsMillis = this.getTimeBetweenEvictionRunsMillis();  
42.                    if (timeBetweenEvictionRunsMillis <= 0) {  
43.                        timeBetweenEvictionRunsMillis = DEFAULT_TIME_BETWEEN_EVICTION_RUNS_MILLIS;  
44.                    }  
45.   
46.                    if (idleMillis >= timeBetweenEvictionRunsMillis) {  
47.                        boolean validate = testConnectionInternal(poolableConnection.getConnection());  
48.                        if (!validate) {  
49.                            if (LOG.isDebugEnabled()) {  
50.                                LOG.debug("skip not validate connection.");  
51.                            }  
52.   
53.                            discardConnection(realConnection);  
54.                            continue;  
55.                        }  
56.                    }  
57.                }  
58.            }  
59.   
60.            if (isRemoveAbandoned()) {  
61.                StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();  
62.                poolableConnection.setConnectStackTrace(stackTrace);  
63.                poolableConnection.setConnectedTimeNano();  
64.                poolableConnection.setTraceEnable(true);  
65.   
66.                synchronized (activeConnections) {  
67.                    activeConnections.put(poolableConnection, PRESENT);  
68.                }  
69.            }  
70.   
71.            if (!this.isDefaultAutoCommit()) {  
72.                poolableConnection.setAutoCommit(false);  
73.            }  
74.   
75.            return poolableConnection;  
76.        }  
77.    }  

ps：相关参数值可以根据实际场景进行设置

配置定义： 

#initialSize
initialSize=10
#minPoolSize
minIdle=10
#maxPoolSize
maxActive=100
poolPreparedStatements=true
maxPoolPreparedStatementPerConnectionSize=100
#是否自动提交事务，如果应用中通过Spring或其他框架显示控制事务提交则可以设置为false，否则设置为true
defaultAutoCommit=false
#配置获取连接等待超时的时间
maxWait=10000
#配置间隔多久才进行一次检测,检测需要关闭的空闲连接,单位毫秒
timeBetweenEvictionRunsMillis=60000
#配置一个连接在池中最小生存的时间,单位毫秒
minEvictableIdleTimeMillis=300000
#连接检测属性配置
testWhileIdle=true
validationQuery=SELECT 'x' FROM DUAL
validationQueryTimeout=5000
connectionProperties=oracle.net.CONNECT_TIMEOUT=3000;oracle.net.READ_TIMEOUT=5000
removeAbandoned=true
removeAbandonedTimeoutMillis=300000
testOnBorrow=false

Spring配置文件：

<bean id="dataSource" class="com.alibaba.druid.pool.DruidDataSource" init-method="init" destroy-method="close">
  <property name="driverClassName" value="${jdbc.driverClassName}" />
  <property name="url" value="${jdbc.url}" />
  <property name="username" value="${jdbc.username}" />
  <property name="password" value="${jdbc.password}" />
  <property name="initialSize" value="${initialSize}" />
  <property name="minIdle" value="${minIdle}" />
  <property name="maxActive" value="${maxActive}" />
  <property name="defaultAutoCommit" value="${defaultAutoCommit}" />
  <property name="validationQuery" value="${validationQuery}" />
  <property name="testWhileIdle" value="${testWhileIdle}" />
  <property name="testOnBorrow" value="${testOnBorrow}" />
  <property name="poolPreparedStatements" value="${poolPreparedStatements}" />
  <property name="maxPoolPreparedStatementPerConnectionSize" value="${maxPoolPreparedStatementPerConnectionSize}" />
  <property name="maxWait" value="${maxWait}" />
  <property name="timeBetweenEvictionRunsMillis" value="${timeBetweenEvictionRunsMillis}" />
  <property name="minEvictableIdleTimeMillis" value="${minEvictableIdleTimeMillis}" />
  <property name="validationQueryTimeout" value="${validationQueryTimeout}" />
  <property name="connectionProperties" value="${connectionProperties}" />
  <property name="removeAbandoned" value="${removeAbandoned}" />
  <property name="removeAbandonedTimeoutMillis" value="${removeAbandonedTimeoutMillis}" />
</bean>