这里简单地记录了 org.apache.commons.httpclient 与 org.apache.commons.httpclient.MultiThreadedHttpConnectionManager 相结合,构建 http 连接池,提高这种因多次连接而产生不必要的时间消耗效率. 这里还有个很重要的类,org.apache.commons.httpclient.methods.PostMethod,它是整个http请求的业务处理类;
org.apache.commons.httpclient.MultiThreadedHttpConnectionManager: 负责连接的管理; 基于服务器ip为key; 存在两个池,一个用于存放 服务器集合,另一个用于存放各服务器所对应的 连接集合
org.apache.commons.httpclient.methods.PostMethod: 负责http业务调度; １:进行请求头的拼装 ２:调用connection连接,进行数据的请求 ３:响应的处理,解析相应的请求头信息
org.apache.commons.httpclient : 封装类,把上述的 MultiThreadedHttpConnectionManager 与 PostMethod 整合,供外部使用.
http仅仅只是一个超文本协议,非传输技术,作用于应用层; 它依赖于传输层的 tcp/ip 协议,进行两端的信息交互; 而tcp/ip 也只是一种交互协议,而socket是目前较为流行的开源java方法类. 因此客户端与服务端间进行http协议交互,实际上是通过下层的socket工具类,进行 tcp/ip 交互.
这里也顺便提一下平常用开的jdbc,它是直接走 传输层,利用 tcp/ip 协议与数据库进行交互,实际上还是通过socket技术与数据库进行二进制的协议数据交互.(有空可以百度了解 二进制协议数据结构)
首先开始我们以 HttpClientUtil 测试类进行开篇,简单地使用 HttpClient 与server进行数据请求.
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.SocketTimeoutException;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import org.apache.commons.httpclient.DefaultHttpMethodRetryHandler;
import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpConnectionManager;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;
import org.apache.commons.httpclient.params.HttpConnectionManagerParams;
import org.apache.commons.httpclient.params.HttpMethodParams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.log4j.chainsaw.Main;
import com.g3net.tool.HttpUtils;
public class HttpClientUtil {
/**
* 日志处理类
*/
private static final Log log = LogFactory.getLog(HttpClientUtil.class);
// 读取超时
private final static int SOCKET_TIMEOUT = 10000;
// 连接超时
private final static int CONNECTION_TIMEOUT = 10000;
// 每个HOST的最大连接数量
private final static int MAX_CONN_PRE_HOST = 20;
// 连接池的最大连接数
private final static int MAX_CONN = 60;
// 连接池
private final static HttpConnectionManager httpConnectionManager;
static {
httpConnectionManager = new MultiThreadedHttpConnectionManager();
HttpConnectionManagerParams params = httpConnectionManager.getParams();
params.setConnectionTimeout(CONNECTION_TIMEOUT);
params.setSoTimeout(SOCKET_TIMEOUT);
params.setDefaultMaxConnectionsPerHost(MAX_CONN_PRE_HOST);
params.setMaxTotalConnections(MAX_CONN);
}
/**
* 发送主要方法,异常捕获
* @param post
* @param code
* @return
*/
public static String doHttpRequest(PostMethod post, String code) {
HttpClient httpClient = new HttpClient(httpConnectionManager);
BufferedReader in = null;
String resultString = "";
try {
httpClient.executeMethod(post);
in = new BufferedReader(new InputStreamReader(post
.getResponseBodyAsStream(), code));
StringBuffer buffer = new StringBuffer();
String line = "";
while ((line = in.readLine()) != null) {
buffer.append(line);
}
resultString = buffer.toString();
} catch (SocketTimeoutException e) {
log.error("连接超时" + e.toString());
resultString = returnError("连接超时");
} catch (HttpException e) {
log.error("读取外部服务器数据失败" + e.toString());
resultString = returnError("读取外部服务器数据失败");
} catch (UnknownHostException e) {
log.error("请求的主机地址无效" + e.toString());
resultString = returnError("请求的主机地址无效");
} catch (IOException e) {
log.error("向外部接口发送数据失败" + e.toString());
resultString = returnError("向外部接口发送数据失败");
} finally {
if (in != null) {
try {
in.close();
} catch (IOException e) {
e.printStackTrace();
}
}
post.releaseConnection();
}
return resultString;
}
/**
* 设置一下返回错误的通用提示,可以自定义格式.
* @param reason
* @return
*/
public static String returnError(String reason) {
StringBuffer buffer = new StringBuffer();
buffer.append("<?xml version=\"1.0\" encoding=\"GBK\"?>");
buffer.append("<Response>");
buffer.append("<Success>false</Success>");
buffer.append("<reason>");
buffer.append(reason);
buffer.append("</reason>");
buffer.append("</Response>");
return buffer.toString();
}
public final static String REQUEST_HEADER = "x-forwarded-for";
/**
* 将客户IP写入请求头
* 这个设置可以伪装IP请求,注意使用
* @param client
* @param ip
* @return
*/
public static void resetRequestHeader(HttpClient client, String ip) {
List<Header> headers = new ArrayList<Header>();
headers.add(new Header(REQUEST_HEADER, ip));
client.getHostConfiguration().getParams().setParameter(
"http.default-headers", headers);
}
public static void main(String[] args) throws Exception{
String url = "http://192.168.162.38/applib/webcontent/interface/searchnew.jsp?key=theme&ty=0&n=100";
PostMethod postMethod = new PostMethod(url);// 放地址
//请求头设置
postMethod.setRequestHeader("Host", "www.baidu.com");
postMethod.setRequestHeader("Connection", "keep-alive");
postMethod.setRequestHeader("User-Agent", "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1229.92 Safari/537.4");
postMethod.setRequestHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
postMethod.setRequestHeader("Accept-Encoding", "gzip,deflate,sdch");
postMethod.setRequestHeader("Accept-Language", "zh-CN,zh;q=0.8");
postMethod.setRequestHeader("Accept-Charset", "GBK,utf-8;q=0.7,*;q=0.3");
String result = HttpClientUtil.doHttpRequest(postMethod, "utf-8");
System.out.println(result);
}
}
httpClient.executeMethod(post);
public int executeMethod(HttpMethod method)
throws IOException, HttpException
{
LOG.trace("enter HttpClient.executeMethod(HttpMethod)");
return executeMethod(null, method, null);
}
在HttpClient 中,我们发现,它把 连接池以及服务的请求处理,封装到了 org.apache.commons.httpclient.HttpMethodDirector.
public int executeMethod(HostConfiguration hostconfig, HttpMethod method, HttpState state)
throws IOException, HttpException
{
LOG.trace("enter HttpClient.executeMethod(HostConfiguration,HttpMethod,HttpState)");
if(method == null)
throw new IllegalArgumentException("HttpMethod parameter may not be null");
//这里是获取服务端信息,ip 端口号 协议 等.
HostConfiguration defaulthostconfig = getHostConfiguration();
if(hostconfig == null)
hostconfig = defaulthostconfig;
URI uri = method.getURI();
if(hostconfig == defaulthostconfig || uri.isAbsoluteURI())
{
hostconfig = (HostConfiguration)hostconfig.clone();
if(uri.isAbsoluteURI())
hostconfig.setHost(uri);
}
//这里把连接的获取,以及服务的请求,给封装到一齐
HttpMethodDirector methodDirector = new HttpMethodDirector(getHttpConnectionManager(), hostconfig, params, state != null ? state : getState());
methodDirector.executeMethod(method);
return method.getStatusCode();
}
在 org.apache.commons.httpclient.HttpMethodDirector ,通过 executeMethod 进行业务请求处理
public void executeMethod(HttpMethod method)
throws IOException, HttpException
{
if(method == null)
throw new IllegalArgumentException("Method may not be null");
hostConfiguration.getParams().setDefaults(params);
method.getParams().setDefaults(hostConfiguration.getParams());
Collection defaults = (Collection)hostConfiguration.getParams().getParameter("http.default-headers");
if(defaults != null)
{
for(Iterator i = defaults.iterator(); i.hasNext(); method.addRequestHeader((Header)i.next()));
}
int maxRedirects = params.getIntParameter("http.protocol.max-redirects", 100);
int redirectCount = 0;
do
{
if(conn != null && !hostConfiguration.hostEquals(conn))
{
conn.setLocked(false);
conn.releaseConnection();
conn = null;
}
if(conn == null)
{
//通过连接池connectionManager,从池中提取有效连接
conn = connectionManager.getConnectionWithTimeout(hostConfiguration, params.getConnectionManagerTimeout());
conn.setLocked(true);
if(params.isAuthenticationPreemptive() || state.isAuthenticationPreemptive())
{
LOG.debug("Preemptively sending default basic credentials");
method.getHostAuthState().setPreemptive();
method.getHostAuthState().setAuthAttempted(true);
if(conn.isProxied() && !conn.isSecure())
{
method.getProxyAuthState().setPreemptive();
method.getProxyAuthState().setAuthAttempted(true);
}
}
}
authenticate(method);
executeWithRetry(method);
if(connectMethod != null)
{
fakeResponse(method);
break;
}
boolean retry = false;
if(isRedirectNeeded(method) && processRedirectResponse(method))
{
retry = true;
if(++redirectCount >= maxRedirects)
{
LOG.error("Narrowly avoided an infinite loop in execute");
throw new RedirectException("Maximum redirects (" + maxRedirects + ") exceeded");
}
if(LOG.isDebugEnabled())
LOG.debug("Execute redirect " + redirectCount + " of " + maxRedirects);
}
if(isAuthenticationNeeded(method) && processAuthenticationResponse(method))
{
LOG.debug("Retry authentication");
retry = true;
}
if(!retry)
break;
if(method.getResponseBodyAsStream() != null)
method.getResponseBodyAsStream().close();
} while(true);
if(conn != null)
conn.setLocked(false);
if((releaseConnection || method.getResponseBodyAsStream() == null) && conn != null)
conn.releaseConnection();
break MISSING_BLOCK_LABEL_583;
Exception exception;
exception;
if(conn != null)
conn.setLocked(false);
if((releaseConnection || method.getResponseBodyAsStream() == null) && conn != null)
conn.releaseConnection();
throw exception;
}
private void executeWithRetry(HttpMethod method)
throws IOException, HttpException
{
int execCount = 0;
_L2:
execCount++;
if(LOG.isTraceEnabled())
LOG.trace("Attempt number " + execCount + " to process request");
if(conn.getParams().isStaleCheckingEnabled())
conn.closeIfStale();
if(!conn.isOpen())
{
conn.open();
if(conn.isProxied() && conn.isSecure() && !(method instanceof ConnectMethod) && !executeConnect())
return;
}
try
{
try
{
applyConnectionParams(method);
//method 为业务处理类,包含了请求头/响应头等信息,但需要 conn 作为桥梁与服务器进行通讯
method.execute(state, conn);
break MISSING_BLOCK_LABEL_452;
}
catch(HttpException e)
{
throw e;
}
catch(IOException e)
{
LOG.debug("Closing the connection.");
conn.close();
HttpMethodRetryHandler handler;
if(method instanceof HttpMethodBase)
{
handler = ((HttpMethodBase)method).getMethodRetryHandler();
if(handler != null && !handler.retryMethod(method, conn, new HttpRecoverableException(e.getMessage()), execCount, method.isRequestSent()))
{
LOG.debug("Method retry handler returned false. Automatic recovery will not be attempted");
throw e;
}
}
handler = (HttpMethodRetryHandler)method.getParams().getParameter("http.method.retry-handler");
if(handler == null)
handler = new DefaultHttpMethodRetryHandler();
if(!handler.retryMethod(method, e, execCount))
{
LOG.debug("Method retry handler returned false. Automatic recovery will not be attempted");
throw e;
}
if(LOG.isInfoEnabled())
LOG.info("I/O exception (" + e.getClass().getName() + ") caught when processing request: " + e.getMessage());
if(LOG.isDebugEnabled())
LOG.debug(e.getMessage(), e);
LOG.info("Retrying request");
}
}
catch(IOException e)
{
if(conn.isOpen())
{
LOG.debug("Closing the connection.");
conn.close();
}
releaseConnection = true;
throw e;
}
catch(RuntimeException e)
{
if(conn.isOpen())
{
LOG.debug("Closing the connection.");
conn.close();
}
releaseConnection = true;
throw e;
}
if(true) goto _L2; else goto _L1
_L1:
}
上述干了许多活儿,到最后还是调用了最开始的 org.apache.commons.httpclient.methods.PostMethod 中的 excute 方法,进行http请求
public int execute(HttpState state, HttpConnection conn)
throws HttpException, IOException
{
LOG.trace("enter HttpMethodBase.execute(HttpState, HttpConnection)");
responseConnection = conn;
checkExecuteConditions(state, conn);
statusLine = null;
connectionCloseForced = false;
conn.setLastResponseInputStream(null);
if(effectiveVersion == null)
effectiveVersion = params.getVersion();
//请求信息发送,包括些请求头/请求体等
writeRequest(state, conn);
requestSent = true;
readResponse(state, conn);
used = true;
return statusLine.getStatusCode();
}
protected void writeRequest(HttpState state, HttpConnection conn)
throws IOException, HttpException
{
int readTimeout;
LOG.trace("enter HttpMethodBase.writeRequest(HttpState, HttpConnection)");
//以下则根据 http 超文本协议,进行协议内容发送,包括请求头/请求体
writeRequestLine(state, conn);
writeRequestHeaders(state, conn);
conn.writeLine();
if(Wire.HEADER_WIRE.enabled())
Wire.HEADER_WIRE.output("\r\n");
HttpVersion ver = getParams().getVersion();
Header expectheader = getRequestHeader("Expect");
String expectvalue = null;
if(expectheader != null)
expectvalue = expectheader.getValue();
if(expectvalue == null || expectvalue.compareToIgnoreCase("100-continue") != 0)
break MISSING_BLOCK_LABEL_265;
if(!ver.greaterEquals(HttpVersion.HTTP_1_1))
break MISSING_BLOCK_LABEL_247;
conn.flushRequestOutputStream();
readTimeout = conn.getParams().getSoTimeout();
conn.setSocketTimeout(3000);
readStatusLine(state, conn);
processStatusLine(state, conn);
readResponseHeaders(state, conn);
processResponseHeaders(state, conn);
if(statusLine.getStatusCode() == 100)
{
statusLine = null;
LOG.debug("OK to continue received");
break MISSING_BLOCK_LABEL_184;
}
conn.setSocketTimeout(readTimeout);
return;
conn.setSocketTimeout(readTimeout);
break MISSING_BLOCK_LABEL_265;
InterruptedIOException e;
e;
if(!ExceptionUtil.isSocketTimeoutException(e))
throw e;
removeRequestHeader("Expect");
LOG.info("100 (continue) read timeout. Resume sending the request");
conn.setSocketTimeout(readTimeout);
break MISSING_BLOCK_LABEL_265;
Exception exception;
exception;
conn.setSocketTimeout(readTimeout);
throw exception;
removeRequestHeader("Expect");
LOG.info("'Expect: 100-continue' handshake is only supported by HTTP/1.1 or higher");
writeRequestBody(state, conn);
conn.flushRequestOutputStream();
return;
}
protected void readResponse(HttpState state, HttpConnection conn)
throws IOException, HttpException
{
LOG.trace("enter HttpMethodBase.readResponse(HttpState, HttpConnection)");
do
{
if(statusLine != null)
break;
//读取服务器返回内容,主要为 http 响应信息
readStatusLine(state, conn);
processStatusLine(state, conn);
readResponseHeaders(state, conn);
processResponseHeaders(state, conn);
int status = statusLine.getStatusCode();
if(status >= 100 && status < 200)
{
if(LOG.isInfoEnabled())
LOG.info("Discarding unexpected response: " + statusLine.toString());
statusLine = null;
}
} while(true);
readResponseBody(state, conn);
processResponseBody(state, conn);
}
来到这里,我们再回过头来看看,在 org.apache.commons.httpclient.HttpMethodDirector 处理过程中,是如何获取到与服务器间的连接,见org.apache.commons.httpclient.MultiThreadedHttpConnectionManager
public HttpConnection getConnectionWithTimeout(HostConfiguration hostConfiguration, long timeout)
throws ConnectionPoolTimeoutException
{
LOG.trace("enter HttpConnectionManager.getConnectionWithTimeout(HostConfiguration, long)");
if(hostConfiguration == null)
throw new IllegalArgumentException("hostConfiguration is null");
if(LOG.isDebugEnabled())
LOG.debug("HttpConnectionManager.getConnection: config = " + hostConfiguration + ", timeout = " + timeout);
HttpConnection conn = doGetConnection(hostConfiguration, timeout);
return new HttpConnectionAdapter(conn);
}
private HttpConnection doGetConnection(HostConfiguration hostConfiguration, long timeout)
throws ConnectionPoolTimeoutException
{
HttpConnection connection;
int maxHostConnections;
connection = null;
maxHostConnections = params.getMaxConnectionsPerHost(hostConfiguration);
int maxTotalConnections = params.getMaxTotalConnections();
ConnectionPool connectionpool = connectionPool;
JVM INSTR monitorenter ;
HostConnectionPool hostPool;
WaitingThread waitingThread;
boolean useTimeout;
long timeToWait;
long startWait;
hostConfiguration = new HostConfiguration(hostConfiguration);
//这里是根据服务端ip,从数据池 connectionPool 里面提取对应的 连接池.
//因此这里涉及到两个池:一个是 connectionPool,可以存储多服务器; 另一个是 hostPool,存储了一台服务器所有的连接.
hostPool = connectionPool.getHostPool(hostConfiguration, true);
waitingThread = null;
useTimeout = timeout > 0L;
timeToWait = timeout;
startWait = 0L;
long endWait = 0L;
_L2:
if(connection != null)
break; /* Loop/switch isn't completed */
if(shutdown)
throw new IllegalStateException("Connection factory has been shutdown.");
//下面是根据 hostPool 里面的连接数情况,进行连接的获取
if(hostPool.freeConnections.size() > 0)
{
connection = connectionPool.getFreeConnection(hostConfiguration);
continue; /* Loop/switch isn't completed */
}
if(hostPool.numConnections < maxHostConnections && connectionPool.numConnections < maxTotalConnections)
{
connection = connectionPool.createConnection(hostConfiguration);
continue; /* Loop/switch isn't completed */
}
if(hostPool.numConnections < maxHostConnections && connectionPool.freeConnections.size() > 0)
{
connectionPool.deleteLeastUsedConnection();
connection = connectionPool.createConnection(hostConfiguration);
continue; /* Loop/switch isn't completed */
}
if(useTimeout && timeToWait <= 0L)
throw new ConnectionPoolTimeoutException("Timeout waiting for connection");
if(LOG.isDebugEnabled())
LOG.debug("Unable to get a connection, waiting..., hostConfig=" + hostConfiguration);
if(waitingThread == null)
{
waitingThread = new WaitingThread();
waitingThread.hostConnectionPool = hostPool;
waitingThread.thread = Thread.currentThread();
} else
{
waitingThread.interruptedByConnectionPool = false;
}
if(useTimeout)
startWait = System.currentTimeMillis();
hostPool.waitingThreads.addLast(waitingThread);
connectionPool.waitingThreads.addLast(waitingThread);
connectionPool.wait(timeToWait);
if(!waitingThread.interruptedByConnectionPool)
{
hostPool.waitingThreads.remove(waitingThread);
connectionPool.waitingThreads.remove(waitingThread);
}
if(useTimeout)
{
long endWait = System.currentTimeMillis();
timeToWait -= endWait - startWait;
}
continue; /* Loop/switch isn't completed */
InterruptedException e;
e;
if(!waitingThread.interruptedByConnectionPool)
{
LOG.debug("Interrupted while waiting for connection", e);
throw new IllegalThreadStateException("Interrupted while waiting in MultiThreadedHttpConnectionManager");
}
if(!waitingThread.interruptedByConnectionPool)
{
hostPool.waitingThreads.remove(waitingThread);
connectionPool.waitingThreads.remove(waitingThread);
}
if(useTimeout)
{
long endWait = System.currentTimeMillis();
timeToWait -= endWait - startWait;
}
if(true) goto _L2; else goto _L1
_L1:
Exception exception;
break MISSING_BLOCK_LABEL_555;
exception;
if(!waitingThread.interruptedByConnectionPool)
{
hostPool.waitingThreads.remove(waitingThread);
connectionPool.waitingThreads.remove(waitingThread);
}
if(useTimeout)
{
long endWait = System.currentTimeMillis();
timeToWait -= endWait - startWait;
}
throw exception;
Exception exception1;
exception1;
throw exception1;
return connection;
}
public synchronized HttpConnection createConnection(HostConfiguration hostConfiguration)
{
HostConnectionPool hostPool = getHostPool(hostConfiguration, true);
if(MultiThreadedHttpConnectionManager.LOG.isDebugEnabled())
MultiThreadedHttpConnectionManager.LOG.debug("Allocating new connection, hostConfig=" + hostConfiguration);
//创建连接,这里的继承了 httpconnection ,根据 ip port 信息,与服务器建立 socket 连接
HttpConnectionWithReference connection = new HttpConnectionWithReference(hostConfiguration);
connection.getParams().setDefaults(params);
connection.setHttpConnectionManager(MultiThreadedHttpConnectionManager.this);
numConnections++;
hostPool.numConnections++;
MultiThreadedHttpConnectionManager.storeReferenceToConnection(connection, hostConfiguration, this);
return connection;
}
public HttpConnection(String proxyHost, int proxyPort, String host, int port, Protocol protocol)
{
hostName = null;
portNumber = -1;
proxyHostName = null;
proxyPortNumber = -1;
socket = null;
inputStream = null;
outputStream = null;
lastResponseInputStream = null;
isOpen = false;
params = new HttpConnectionParams();
locked = false;
usingSecureSocket = false;
tunnelEstablished = false;
if(host == null)
throw new IllegalArgumentException("host parameter is null");
if(protocol == null)
{
throw new IllegalArgumentException("protocol is null");
} else
{
proxyHostName = proxyHost;
proxyPortNumber = proxyPort;
hostName = host;
portNumber = protocol.resolvePort(port);
protocolInUse = protocol;
return;
}
}
public void open()
throws IOException
{
LOG.trace("enter HttpConnection.open()");
String host = proxyHostName != null ? proxyHostName : hostName;
int port = proxyHostName != null ? proxyPortNumber : portNumber;
assertNotOpen();
if(LOG.isDebugEnabled())
LOG.debug("Open connection to " + host + ":" + port);
try
{
if(socket == null)
{
usingSecureSocket = isSecure() && !isProxied();
ProtocolSocketFactory socketFactory = null;
if(isSecure() && isProxied())
{
Protocol defaultprotocol = Protocol.getProtocol("http");
socketFactory = defaultprotocol.getSocketFactory();
} else
{
socketFactory = protocolInUse.getSocketFactory();
}
//与服务器间建立起socket通道,进行后续的数据交互
socket = socketFactory.createSocket(host, port, localAddress, 0, params);
}
socket.setTcpNoDelay(params.getTcpNoDelay());
socket.setSoTimeout(params.getSoTimeout());
int linger = params.getLinger();
if(linger >= 0)
socket.setSoLinger(linger > 0, linger);
int sndBufSize = params.getSendBufferSize();
if(sndBufSize >= 0)
socket.setSendBufferSize(sndBufSize);
int rcvBufSize = params.getReceiveBufferSize();
if(rcvBufSize >= 0)
socket.setReceiveBufferSize(rcvBufSize);
int outbuffersize = socket.getSendBufferSize();
if(outbuffersize > 2048 || outbuffersize <= 0)
outbuffersize = 2048;
int inbuffersize = socket.getReceiveBufferSize();
if(inbuffersize > 2048 || inbuffersize <= 0)
inbuffersize = 2048;
inputStream = new BufferedInputStream(socket.getInputStream(), inbuffersize);
outputStream = new BufferedOutputStream(socket.getOutputStream(), outbuffersize);
isOpen = true;
}
catch(IOException e)
{
closeSocketAndStreams();
throw e;
}
}