I wanted to encapsulate HttpClient5. The basic get request has been implemented, but the Post with parameter submission has been unable to obtain the parameters. Record the code and find a solution later.
pom.xml configuration, httpclient version is
<httpclient.version>5.1.3</httpclient.version>
<!-- httpclient5 提供http请求服务的优秀框架 -->
<dependency>
<groupId>org.apache.httpcomponents.client5</groupId>
<artifactId>httpclient5</artifactId>
<version>${httpclient.version}</version>
</dependency>Encapsulate the configuration class HttpConnectorConfig, the code is as follows:
package com.vtarj.pythagoras.explorer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.cookie.BasicCookieStore;
import org.apache.hc.client5.http.cookie.CookieStore;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.hc.client5.http.socket.ConnectionSocketFactory;
import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory;
import org.apache.hc.client5.http.ssl.NoopHostnameVerifier;
import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.config.Registry;
import org.apache.hc.core5.http.config.RegistryBuilder;
import org.apache.hc.core5.http.io.SocketConfig;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicHeader;
import org.apache.hc.core5.util.TimeValue;
import org.apache.tomcat.util.json.JSONParser;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
/**
* @Author Vtarj
* @Description TODO
* @Time 2022/3/30 11:03
**/
public class HttpConfig {
//定义HttpClient构造器
private HttpClientBuilder builder = HttpClientBuilder.create();
//定义头信息
private Map<String,String> headerMap;
//定义CookieStore对象
private CookieStore cookieStore;
//定义Basic Auth管理对象
private BasicCredentialsProvider basicCredentialsProvider;
//定义请求参数
private List<NameValuePair> pairs;
//定义默认请求类型
private static String contentType = "application/json";
//定义请求和响应字符编码
private Charset reqCode = StandardCharsets.UTF_8;
private Charset resCode = StandardCharsets.UTF_8;
public HttpConfig(){
//设置连接基本配置
//注册访问协议相关的Socket工厂
Registry<ConnectionSocketFactory> registry = RegistryBuilder.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.INSTANCE)
.register("https",trustHttpsCertificates())
.build();
//设置连接池管理器
PoolingHttpClientConnectionManager manager = new PoolingHttpClientConnectionManager(registry);
manager.setMaxTotal(100);
manager.setDefaultMaxPerRoute(manager.getMaxTotal() / 2);
manager.setValidateAfterInactivity(TimeValue.ofMinutes(5));
manager.setDefaultSocketConfig(SocketConfig.custom().setSoTimeout(30,TimeUnit.SECONDS).setTcpNoDelay(true).build());
//为HttpClientBuilder设置连接池
builder.setConnectionManager(manager);
//设置定期清理连接池中过期的连接
builder.evictExpiredConnections();
builder.evictIdleConnections(TimeValue.ofMinutes(3));
//设置请求基础配置
//创建默认CookieStore
cookieStore = new BasicCookieStore();
//创建默认Basic Auth对象
basicCredentialsProvider = new BasicCredentialsProvider();
//设置Http请求基本参数
RequestConfig requestConfig = RequestConfig.custom()
// 设置启用重定向
.setRedirectsEnabled(true)
// 设置最大重定向次数
.setMaxRedirects(30)
// 设置连接超时时间
.setConnectTimeout(2,TimeUnit.MINUTES)
// 设置请求超时时间
.setConnectionRequestTimeout(2,TimeUnit.MINUTES)
// 设置响应超时时间
.setResponseTimeout(2,TimeUnit.MINUTES)
.build();
//为HttpClientBuilder设置连接配置
builder.setDefaultRequestConfig(requestConfig);
//为HttpClientBuilder设置头信息
builder.setDefaultHeaders(buildHeader());
builder.setDefaultCookieStore(cookieStore);
builder.setDefaultCredentialsProvider(basicCredentialsProvider);
}
/**
* 设置默认请求头
*/
private void setDefaultHeader(){
if (this.getHeaderMap() == null){
headerMap = new HashMap<>();
}
if(!headerMap.containsKey("User-Agent")){
headerMap.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.74 Safari/537.36 Edg/99.0.1150.55");
}
if (!headerMap.containsKey("Content-Type")){
headerMap.put("Content-Type", contentType);
}
}
/**
* 将Map类型的Header构建为标准Header列表
* @return
*/
private List<Header> buildHeader(){
setDefaultHeader();
return this.getHeaderMap()
.entrySet().stream()
.map(v -> new BasicHeader(v.getKey(),v.getValue()))
.collect(Collectors.toList());
}
/**
* 将参数列表转换为字符串拼接，以&符分隔
* @param list 参数列表
* @return 转换后的字符串
*/
public static String pairsToString(List<NameValuePair> list){
return list.stream().map(v -> v.getName() + "=" + v.getValue()).collect(Collectors.joining("&"));
}
/**
* 构建Get请求
* @param uri 请求地址
*/
public HttpGet buildGet(String uri){
uri = formatURI(uri);
if (this.getPairs() != null && this.getPairs().size() > 0){
uri = uri.contains("?") ? uri + "&" : uri + "?";
uri = uri + pairsToString(this.getPairs());
}
HttpGet httpGet = new HttpGet(uri);
/**
List<Header> headerList = this.buildHeader();
Header[] headers = headerList.toArray(new Header[headerList.size()]);
httpGet.setHeaders(headers);
**/
return httpGet;
}
/**
* 构建Post请求，模拟Form表单提交
* @param uri 请求地址
* @return Post请求
*/
public HttpPost buildPost(String uri){
uri = formatURI(uri);
HttpPost httpPost = new HttpPost(uri);
httpPost.addHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
//设置请求参数
if (this.getPairs() != null && this.getPairs().size() > 0){
httpPost.setEntity(new UrlEncodedFormEntity(pairs,this.getReqCode()));
}
return httpPost;
}
/**
* 格式化uri连接地址，补全协议
* @param uri 待格式化的连接地址
* @return 格式化后的地址
*/
private static String formatURI(String uri) {
if (!uri.toLowerCase().startsWith("http://") && !uri.toLowerCase().startsWith("https://")){
return "http://" + uri;
}
return uri;
}
/**
* Https证书管理
* @return 可识别证书集合
*/
private static ConnectionSocketFactory trustHttpsCertificates() {
TrustManager[] trustAllCertificates = new TrustManager[]{
new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}
};
SSLContext sslContext = null;
try {
sslContext = SSLContext.getInstance("TLS");
sslContext.init(null,trustAllCertificates,new SecureRandom());
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
} catch (KeyManagementException e) {
e.printStackTrace();
}
return new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE);
}
public HttpClientBuilder getBuilder() {
return builder;
}
public Map<String, String> getHeaderMap() {
return headerMap;
}
/**
* 设置请求头信息
* @param headerMap 请求头信息
* @return 返回HttpConfig，用于链式调用
*/
public HttpConfig setHeaderMap(Map<String, String> headerMap) {
this.headerMap = headerMap;
return this;
}
/**
* 自定义添加请求头信息
* @param key 请求头键
* @param value 请求头值
* @return 返回HttpConfig，用于链式调用
*/
public HttpConfig addHeader(String key,String value){
this.headerMap.put(key,value);
return this;
}
public CookieStore getCookieStore() {
return cookieStore;
}
/**
* 设置CookieStore
* @param cookieStore cookiestore
* @return 返回HttpConfig，用于链式调用
*/
public HttpConfig setCookieStore(CookieStore cookieStore) {
this.cookieStore = cookieStore;
return this;
}
public BasicCredentialsProvider getBasicCredentialsProvider() {
return basicCredentialsProvider;
}
/**
*
* @param basicCredentialsProvider
* @return 返回HttpConfig，用于链式调用
*/
public HttpConfig setBasicCredentialsProvider(BasicCredentialsProvider basicCredentialsProvider) {
this.basicCredentialsProvider = basicCredentialsProvider;
return this;
}
public List<NameValuePair> getPairs() {
return pairs;
}
/**
* 设置请求参数
* @param pairs 请求参数集合
* @return 返回HttpConfig，用于链式调用
*/
public HttpConfig setPairs(List<NameValuePair> pairs) {
this.pairs = pairs;
return this;
}
public Charset getReqCode() {
return reqCode;
}
public HttpConfig setReqCode(Charset reqCode) {
this.reqCode = reqCode;
return this;
}
public Charset getResCode() {
return resCode;
}
public HttpConfig setResCode(Charset resCode) {
this.resCode = resCode;
return this;
}
public HttpExplorer build(){
return new HttpExplorer(this);
}
}
Encapsulate the detector class HttpExplorer, the code is as follows:
package com.vtarj.pythagoras.explorer;
import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
/**
* @Author Vtarj
* @Description TODO
* @Time 2022/3/30 11:00
**/
public class HttpExplorer {
private CloseableHttpClient httpClient = null;
private HttpConfig config;
public HttpExplorer (HttpConfig config) {
if (httpClient == null){
synchronized (HttpExplorer.class) {
this.config = config;
httpClient = config.getBuilder().build();
}
}
}
/**
* 向指定地址发送Get请求
* @param uri 目标地址
* @return 响应信息
*/
public HttpResponse<String> get(String uri){
try {
CloseableHttpResponse response = this.httpClient.execute(config.buildGet(uri));
return HttpResponseHandler.ofString(config.getResCode()).handler(response,this);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* 向指定地址发送Get请求
* @param uri 目标地址
* @return 响应信息
*/
public HttpResponse<File> getFile(String uri, String path){
try {
CloseableHttpResponse response = this.httpClient.execute(config.buildGet(uri));
return HttpResponseHandler
.ofFile(Path.of(path), StandardCopyOption.REPLACE_EXISTING)
.handler(response,this);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* 向指定地址发送Post请求
* @param uri 目标地址
* @return 响应信息
*/
public HttpResponse<String> post(String uri){
try {
HttpPost httpPost = config.buildPost(uri);
CloseableHttpResponse response = this.httpClient.execute(httpPost,new HttpClientContext());
return HttpResponseHandler.ofString(config.getResCode()).handler(response,this);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* 创建按构造器，用于构造HttpClient配置
* @return
*/
public static HttpConfig builder(){
return new HttpConfig();
}
}
Encapsulate the response class HttpResponse, the code is as follows:
package com.vtarj.pythagoras.explorer;
import org.apache.hc.core5.http.ClassicHttpResponse;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.ProtocolVersion;
import java.util.Locale;
/**
* @Author Vtarj
* @Description 响应信息封装
* @Time 2022/3/30 15:30
**/
public class HttpResponse<T> {
private final int code;
private final HttpEntity entity;
private final T data;
private final ProtocolVersion version;
private final Locale locale;
private final String reasonPhrase;
public HttpResponse(int code, HttpEntity entity, T data, ProtocolVersion version, Locale locale, String reasonPhrase) {
this.code = code;
this.entity = entity;
this.data = data;
this.version = version;
this.locale = locale;
this.reasonPhrase = reasonPhrase;
}
public static <T> HttpResponse<T> build(ClassicHttpResponse response, T data) {
return new HttpResponse<T>(response.getCode(), response.getEntity(), data, response.getVersion(), response.getLocale(), response.getReasonPhrase());
}
public int getCode() {
return code;
}
public HttpEntity getEntity() {
return entity;
}
public T getData() {
return data;
}
public ProtocolVersion getVersion() {
return version;
}
public Locale getLocale() {
return locale;
}
public String getReasonPhrase() {
return reasonPhrase;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("Response{");
sb.append("code=").append(code);
sb.append(", entity=").append(entity);
sb.append(", data=").append(data);
sb.append(", version=").append(version);
sb.append(", locale=").append(locale);
sb.append(", reasonPhrase='").append(reasonPhrase).append('\'');
sb.append('}');
return sb.toString();
}
}
Response proxy interface HttpResponseHandler, the code is as follows:
package com.vtarj.pythagoras.explorer;
import org.apache.hc.core5.http.ClassicHttpResponse;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import java.io.File;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.nio.file.CopyOption;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* @Author Vtarj
* @Description 响应信息处理
* @Time 2022/3/30 15:31
**/
public interface HttpResponseHandler<T> {
HttpResponse<T> handler(ClassicHttpResponse response, HttpExplorer explorer);
/**
* 返回字符串
*
* @param defaultCharset 默认字符集
*/
static HttpResponseHandler<String> ofString(Charset... defaultCharset) {
return (response, client) -> {
try {
Charset charset = defaultCharset != null && defaultCharset.length > 0 ? defaultCharset[0] : Charset.defaultCharset();
return HttpResponse.build(response, EntityUtils.toString(response.getEntity(), charset));
} catch (Exception e) {
throw new RuntimeException(e);
}
};
}
/**
* 保存为文件
*
* @param path 保存文件路径
* @param options StandardCopyOption: REPLACE_EXISTING(替换更新), COPY_ATTRIBUTES(复制属性), ATOMIC_MOVE(原子移动)
*/
static HttpResponseHandler<File> ofFile(Path path, CopyOption... options) {
return (response, client) -> {
File file = path.toFile();
if (!file.exists()) {
file.getParentFile().mkdirs();
}
try (InputStream in = response.getEntity().getContent()) {
Files.copy(in, path, options);
} catch (Exception e) {
throw new RuntimeException(e);
}
return HttpResponse.build(response, file);
};
}
}
program test
@Test
void testHttpExplorer4Get(){
//网页资源访问
HttpResponse<String> stringHttpResponse = HttpExplorer.builder()
.build()
.get("http://www.stats.gov.cn/tjsj/tjbz/tjyqhdmhcxhfdm/2021/index.html");
System.out.println(stringHttpResponse.toString());
}
@Test
void testHttpExplorer4GetFile(){
//文件下载
HttpResponse<File> fileHttpResponse = HttpExplorer.builder()
.build()
.getFile("http://**********/YZSoft/Attachment/default.ashx?202203280003","D://a.pdf");
System.out.println(fileHttpResponse.getData().getName());
}
/**
* HttpClient5 的post带参提交无法获得参数，测试失败
*/
@Test
void testPost(){
try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
HttpPost httpPost = new HttpPost("https://register-api.did.id/v1/account/search");
List<NameValuePair> nvps = new ArrayList<>();
nvps.add(new BasicNameValuePair("account","mama.bit"));
nvps.add(new BasicNameValuePair("address",""));
httpPost.setEntity(new UrlEncodedFormEntity(nvps));
CloseableHttpResponse response = httpclient.execute(httpPost);
HttpResponse<String> result = HttpResponseHandler.ofString().handler(response,null);
System.out.println(result);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* HttpClient5 的post带参提交无法获得参数，测试失败
*/
@Test
void testHttpExplorer4PostWithDAS(){
List<NameValuePair> nvps = new ArrayList<>();
nvps.add(new BasicNameValuePair("account","mama.bit"));
nvps.add(new BasicNameValuePair("address",""));
HttpResponse<String> response = HttpExplorer
.builder()
.setPairs(nvps)
.addHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8")
.build()
.post("https://register-api.did.id/v1/account/search");
System.out.println(response.toString());
}
/**
* HttpClient5 的post带参提交无法获得参数，测试失败
*/
@Test
void testHttpExplorer4Post(){
List<NameValuePair> nvps = new ArrayList<>();
nvps.add(new BasicNameValuePair("username","sysadmin"));
nvps.add(new BasicNameValuePair("password","Test000000@"));
nvps.add(new BasicNameValuePair("execution","e1s1"));
nvps.add(new BasicNameValuePair("_eventId","submit"));
nvps.add(new BasicNameValuePair("submit", "登录"));
HttpResponse<String> response = HttpExplorer.builder()
.addHeader("Content-Type","text/html;charset=UTF-8")
.addHeader("Referer","http://192.168.1.191:8008/cas/login")
.setPairs(nvps)
.build()
.post("http://192.168.1.191:8008/cas/login");
System.out.println(response.toString());
}
The basic encapsulation has been completed, but there are two questions:
Question 1: Why does HttpClientBuilder provide the setDefaultHeaders method?
HttpClientBuilder is used to construct the HttpClient channel, but what is the purpose of setting the Header here? After the subsequent construction of HttpRequest, the Header still needs to be reset, so I don’t understand why the Header and other information should be set here?
Question 2: HttpPost.setEntity searched on the Internet and learned that it simulates Form submission in the form of application/x-www-form-urlencoded, so there is no need to specify the header of the request, but why the server still cannot obtain the form after the form is submitted parameter?
So far, the encapsulation has come to an end, I hope it will be of some help to you