有个后台计算的服务6台机器，全假死了，服务停止运算，一开好奇怪。自己重启了5台恢复线上，留一台查看原因。
通过jstack查看，发现接受mp的线程WAITING
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x00000006addee0c8> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:186)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2043)
at org.apache.commons.pool2.impl.LinkedBlockingDeque.takeFirst(LinkedBlockingDeque.java:524)
at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:433)
at org.apache.commons.pool2.impl.GenericObjectPool.borrowObject(GenericObjectPool.java:360)
at redis.clients.util.Pool.getResource(Pool.java:40)
at redis.clients.jedis.JedisPool.getResource(JedisPool.java:84)
at com.sohu.snscommon.cluster.service.impl.OrMetaRedisClusterServiceImpl.getJd(OrMetaRedisClusterServiceImpl.java:111)
at com.sohu.snscommon.cluster.service.impl.OrMetaRedisClusterServiceImpl.getItem(OrMetaRedisClusterServiceImpl.java:241)
at com.sohu.sns_protobuf.service.impl.SnsProtoCacheServerImpl.loadCache(SnsProtoCacheServerImpl.java:44)
at com.sohu.sns_protobuf.service.impl.SnsPersServerImpl.loadItemJson(SnsPersServerImpl.java:167)
at com.sohu.sns_protobuf.utils.PersistenceUtil.getItem(PersistenceUtil.java:109)
at com.sohu.sns.service.pull.service.FeedPullService.getUserRelation(FeedPullService.java:215)
at com.sohu.sns.service.pull.service.FeedPullService.addPullMarkInternal(FeedPullService.java:172)
at com.sohu.sns.service.pull.service.FeedPullService.addPullMark(FeedPullService.java:153)
at com.sohu.sns.service.pull.mq.PullServerMQConsumer$1.run(PullServerMQConsumer.java:92)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask.run(FutureTask.java:262)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:178)
at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:292)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:744)通过可以看到jedisPool拿不到getResource,一查看代码原来忘记忘记maxWaitMillis,也是遇见鬼了
大部分故障都是细节引起的