HttpClient 4.3高并发，多线程配置

直接上码

最近用到多线程来对请求HTTP ， 但是http的每次请求都是一次连接后重新连接，导致大量的数据积压，所以根据资料，重新写了一个HTTP的工具类

 

[java]  view plain  copy

 

 print ?

1. package cn.zto.util;  
2.   
3. import java.io.UnsupportedEncodingException;  
4. import java.util.ArrayList;  
5. import java.util.List;  
6. import java.util.Map;  
7.   
8. import org.apache.commons.lang.StringUtils;  
9. import org.apache.http.HttpEntity;  
10. import org.apache.http.NameValuePair;  
11. import org.apache.http.client.config.RequestConfig;  
12. import org.apache.http.client.entity.UrlEncodedFormEntity;  
13. import org.apache.http.client.methods.CloseableHttpResponse;  
14. import org.apache.http.client.methods.HttpGet;  
15. import org.apache.http.client.methods.HttpPost;  
16. import org.apache.http.entity.StringEntity;  
17. import org.apache.http.impl.client.CloseableHttpClient;  
18. import org.apache.http.impl.client.HttpClients;  
19. import org.apache.http.impl.client.LaxRedirectStrategy;  
20. import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;  
21. import org.apache.http.message.BasicNameValuePair;  
22. import org.apache.http.util.EntityUtils;  
23.   
24. import cn.zto.entity.HttpResEntity;  
25.   
26. public class HttpUtils {  
27.   
28.     private static final CloseableHttpClient httpClient;  
29.     public static final String CHARSET_GBK = "GBK";  
30.     public static final String CHARSET_UTF8 = "UTF-8";  
31.       // 将最大连接数增加到  
32.     public static final int MAX_TOTAL = 600;  
33.      // 将每个路由基础的连接增加到  
34.     public static final int MAX_ROUTE_TOTAL = 300;  
35.     public static final int REQUEST_TIMEOUT = 5*1000;  
36.     public static final int REQUEST_SOCKET_TIME = 5*1000;  
37.       
38.       
39.     /*static { 
40.         //设置连接超时 的时间 
41.         RequestConfig config = RequestConfig.custom().setConnectTimeout(60000) 
42.                 .setSocketTimeout(15000).build(); 
43.          
44.         httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config) 
45.                 .build(); 
46.     }*/  
47.       
48.     static {  
49.         // 设置连接超时 的时间  
50.   
51.         PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager();  
52.         httpClientConnectionManager.setMaxTotal(MAX_TOTAL);  
53.         httpClientConnectionManager.setDefaultMaxPerRoute(MAX_ROUTE_TOTAL);  
54.           
55.         RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(REQUEST_TIMEOUT)  
56.                 .setSocketTimeout(REQUEST_SOCKET_TIME).build();  
57.         //设置重定向策略    
58.         LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();    
59.           
60.         httpClient = HttpClients.custom().setConnectionManager(httpClientConnectionManager)    
61.                 .setDefaultRequestConfig(requestConfig).setRedirectStrategy(redirectStrategy).build();    
62.     }  
63.   
64.     public static HttpResEntity doGet(String url, Map<String, Object> params,  
65.             String charset) {  
66.         HttpResEntity entity = new HttpResEntity();  
67.         if (StringUtils.isBlank(url)) {  
68.             entity.setMsg("请求地址异常");  
69.             entity.setStatus(404);  
70.             return entity;  
71.         }  
72.         try {  
73.             if (params != null && !params.isEmpty()) {  
74.                 List<NameValuePair> pairs = new ArrayList<NameValuePair>(  
75.                         params.size());  
76.                 for (Map.Entry<String, Object> entry : params.entrySet()) {  
77.                     Object value = entry.getValue();  
78.                     if (value != null) {  
79.                         pairs.add(new BasicNameValuePair(entry.getKey(),  
80.                                 (String) value));  
81.                     }  
82.                 }  
83.                 url += "?"  
84.                         + EntityUtils.toString(new UrlEncodedFormEntity(pairs,  
85.                                 charset));  
86.             }  
87.             HttpGet httpGet = new HttpGet(url);  
88.             CloseableHttpResponse response = httpClient.execute(httpGet);  
89.             int statusCode = response.getStatusLine().getStatusCode();  
90.             if (statusCode != 200) {  
91.                 httpGet.abort();  
92.                 entity.setData("请求异常");  
93.                 entity.setStatus(statusCode);  
94.                 return entity;  
95.             }  
96.             HttpEntity httpEntity = response.getEntity();  
97.             String result = null;  
98.             if (entity != null) {  
99.                 result = EntityUtils.toString(httpEntity, charset);  
100.             }  
101.             EntityUtils.consume(httpEntity);  
102.             response.close();  
103.             entity.setStatus(statusCode);  
104.             entity.setData(result);  
105.             return entity;  
106.         } catch (Exception e) {  
107.             e.printStackTrace();  
108.         }  
109.         return null;  
110.     }  
111.   
112.     public static HttpResEntity doPost(String url, Map<String, Object> params,  
113.             String charset) {  
114.         List<NameValuePair> pairs = null;  
115.         if (params != null && !params.isEmpty()) {  
116.             pairs = new ArrayList<NameValuePair>(params.size());  
117.             for (Map.Entry<String, Object> entry : params.entrySet()) {  
118.                 Object value = entry.getValue();  
119.                 if (value != null) {  
120.                     pairs.add(new BasicNameValuePair(entry.getKey(),  
121.                             (String) value));  
122.                 }  
123.             }  
124.         }  
125.         StringEntity httpEntity = null;  
126.         try {  
127.             httpEntity = new UrlEncodedFormEntity(pairs, charset);  
128.         } catch (UnsupportedEncodingException e) {  
129.             e.printStackTrace();  
130.         }  
131.         return doPost(url, httpEntity, charset);  
132.     }  
133.   
134.     public static HttpResEntity doPost(String url, String params, String charset) {  
135.         StringEntity entity = null;  
136.         try {  
137.             entity = new StringEntity(params, charset);  
138.         } catch (UnsupportedEncodingException e) {  
139.             e.printStackTrace();  
140.         }  
141.         return doPost(url, entity, charset);  
142.     }  
143.   
144.     /** 
145.      * HTTP Post 获取内容 
146.      *  
147.      * @param url 
148.      *            请求的url地址 ?之前的地址 
149.      * @param params 
150.      *            请求的参数 
151.      * @param charset 
152.      *            编码格式 
153.      * @return 页面内容 
154.      */  
155.     public static HttpResEntity doPost(String url, StringEntity params,  
156.             String charset) {  
157.         HttpResEntity entity = new HttpResEntity();  
158.         if (StringUtils.isBlank(url)) {  
159.             entity.setMsg("请求地址异常");  
160.             entity.setStatus(404);  
161.             return entity;  
162.         }  
163.         try {  
164.             HttpPost httpPost = new HttpPost(url);  
165.             if (params != null) {  
166.                 httpPost.setEntity(params);  
167.             }  
168.             CloseableHttpResponse response = httpClient.execute(httpPost);  
169.             int statusCode = response.getStatusLine().getStatusCode();  
170.             if (statusCode != 200) {  
171.                 httpPost.abort();  
172.                 entity.setData("请求异常");  
173.                 entity.setStatus(statusCode);  
174.                 return entity;  
175.             }  
176.             HttpEntity httpEntity = response.getEntity();  
177.             String result = null;  
178.             if (httpEntity != null) {  
179.                 result = EntityUtils.toString(httpEntity, charset);  
180.             }  
181.             EntityUtils.consume(httpEntity);  
182.             response.close();  
183.             entity.setStatus(statusCode);  
184.             entity.setData(result);  
185.             return entity;  
186.         } catch (Exception e) {  
187.             e.printStackTrace();  
188.         }  
189.         return null;  
190.     }  
191.   
192. }  

 

 

 

 

[java]  view plain  copy

 

 print ?

1. static {  
2.         //设置连接超时 的时间  
3.         RequestConfig config = RequestConfig.custom().setConnectTimeout(60000)  
4.                 .setSocketTimeout(15000).build();  
5.           
6.         httpClient = HttpClientBuilder.create().setDefaultRequestConfig(config)  
7.                 .build();  

 

这是原先的设置的超时时间，其中并没有设置连接池，所以在500个线程并发，并且每个线程有100次循环时，耗时极长。

 

贴上来一段多线程的测试 

 

 

[java]  view plain  copy

 

 print ?

1. public static void main(String[] args) throws InterruptedException,  
2.             ExecutionException {  
3.         long startTime = System.currentTimeMillis();  
4.         System.out.println("程序开始运行");  
5.         int taskSize = 500;  
6.         Date date = new Date();  
7.         // 创建一个线程池  
8.         ExecutorService pool = Executors.newFixedThreadPool(taskSize);  
9.         // 创建多个有返回值的任务  
10.         List<Future> list = new ArrayList<Future>();  
11.         for (int i = 0; i < taskSize; i++) {  
12.             Callable c = new MyCallable2(i + "");  
13.             Future f = pool.submit(c);  
14.             list.add(f);  
15.         }  
16.   
17.         pool.shutdown();  
18.         // 获取所有并发任务的运行结果  
19.         for (Future f : list) {  
20.             // 从Future对象上获取任务的返回值，并输出到控制台  
21.             System.out.println(">>>" + f.get().toString());  
22.             Date date2 = new Date();  
23.             System.out.println("----程序结束运行----，程序运行时间【"  
24.                     + (date2.getTime() - date.getTime()) + "毫秒】");  
25.         }  
26.           
27.         System.out.println("程序共耗时："+(System.currentTimeMillis()-startTime)+"毫秒");  
28.     }  

现在把配置改为

 

 

[java]  view plain  copy

 

 print ?

1. static {  
2.         // 设置连接超时 的时间  
3.   
4.         PoolingHttpClientConnectionManager httpClientConnectionManager = new PoolingHttpClientConnectionManager();  
5.         httpClientConnectionManager.setMaxTotal(MAX_TOTAL);  
6.         httpClientConnectionManager.setDefaultMaxPerRoute(MAX_ROUTE_TOTAL);  
7.           
8.         RequestConfig requestConfig = RequestConfig.custom().setConnectTimeout(REQUEST_TIMEOUT)  
9.                 .setSocketTimeout(REQUEST_SOCKET_TIME).build();  
10.         //设置重定向策略    
11.         LaxRedirectStrategy redirectStrategy = new LaxRedirectStrategy();    
12.           
13.         httpClient = HttpClients.custom().setConnectionManager(httpClientConnectionManager)    
14.                 .setDefaultRequestConfig(requestConfig).setRedirectStrategy(redirectStrategy).build();    
15.     }  

运行时间从原来的10分钟能够左右，降低到了1分钟左右。

 

推荐文章

 

使用httpclient必须知道的参数设置及代码写法、存在的风险 ：http://jinnianshilongnian.iteye.com/blog/2089792

 

HttpClient4.3教程 第二章 连接管理：http://niuzhenxin.iteye.com/blog/2100100

 

原文章地址：http://blog.csdn.net/wangqingqi20005/article/details/47904153