Apache HttpClient 5 detailed tutorial

HTTP Components Logo

The Hypertext Transfer Protocol (HTTP) may be one of the most important protocols on the Internet today. Web services, microservices, and services on various devices that support the network are almost all HTTP protocols. The HTTP protocol has moved from Web browsers to more Wide range of usage scenarios.

Although  java.net the package already provides the basic functions of accessing resources through HTTP, it is not flexible enough and cannot be customized as desired. Apache HttpClient 5 is an open source HTTP toolkit that can support the latest HTTP protocol standard, and has rich API and powerful extension features, which can be used to build any application that needs to handle HTTP protocol.

This article introduces some of the most common usages in Apache HttpClient 5. Through this article, you can quickly get started using HttpClient 5. The main content includes HttpClient 5's Get request, Post request, how to carry parameters, JSON parameters, setting timeouts, and asynchronous requests , operating cookies, form login, basic authentication, digest authentication, and custom HTTP request interceptors, etc.

HttpClient 5 dependencies

HttpClient 5 Maven dependency

<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents.client5/httpclient5 -->
<dependency>
    <groupId>org.apache.httpcomponents.client5</groupId>
    <artifactId>httpclient5</artifactId>
    <version>5.1.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents.client5/httpclient5-fluent -->
<dependency>
    <groupId>org.apache.httpcomponents.client5</groupId>
    <artifactId>httpclient5-fluent</artifactId>
    <version>5.1.3</version>
</dependency>

HttpClient 5 Gradle dependency

implementation 'org.apache.httpcomponents.client5:httpclient5:5.1.3'
implementation 'org.apache.httpcomponents.client5:httpclient5-fluent:5.1.3'

HttpClient 5 GET request

package com.wdbyte.httpclient;

import java.io.IOException;

import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;

/**
 * @author https://www.wdbyte.com
 */
public class HttpClient5Get {

    public static void main(String[] args) {
        String result = get("http://httpbin.org/get");
        System.out.println(result);
    }

    public static String get(String url) {
        String resultContent = null;
        HttpGet httpGet = new HttpGet(url);
        try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
            try (CloseableHttpResponse response = httpclient.execute(httpGet)) {
                // 获取状态码
                System.out.println(response.getVersion()); // HTTP/1.1
                System.out.println(response.getCode()); // 200
                System.out.println(response.getReasonPhrase()); // OK
                HttpEntity entity = response.getEntity();
                // 获取响应信息
                resultContent = EntityUtils.toString(entity);
            }
        } catch (IOException | ParseException e) {
            e.printStackTrace();
        }
        return resultContent;
    }

}

Response information:

HTTP/1.1
200
OK
{
  "args": {}, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/17)", 
    "X-Amzn-Trace-Id": "Root=1-62bb1891-5ab5e5376ed960471bf32f17"
  }, 
  "origin": "47.251.4.198", 
  "url": "http://httpbin.org/get"
}

HttpClient 5 Fluent GET

Using the Fluent API provided by Apache HttpClient 5 can initiate a GET request more conveniently, but there are fewer operable places.

rely:

<!-- https://mvnrepository.com/artifact/org.apache.httpcomponents.client5/httpclient5-fluent -->
<dependency>
    <groupId>org.apache.httpcomponents.client5</groupId>
    <artifactId>httpclient5-fluent</artifactId>
    <version>5.1.3</version>
</dependency>

Example:

package com.wdbyte.httpclient;

import java.io.IOException;

import org.apache.hc.client5.http.fluent.Request;
import org.apache.hc.client5.http.fluent.Response;

/**
* @author https://www.wdbyte.com
 */
public class HttpClient5GetFluent {

    public static void main(String[] args) {
        System.out.println(get("http://httpbin.org/get"));
    }

    public static String get(String url) {
        String result = null;
        try {
            Response response = Request.get(url).execute();
            result = response.returnContent().asString();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return result;
    }

}

Output information:

{
  "args": {}, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/17)", 
    "X-Amzn-Trace-Id": "Root=1-62bb190e-1ba46a92645843a04c55da32"
  }, 
  "origin": "47.251.4.198", 
  "url": "http://httpbin.org/get"
}

HttpClient5 GET request parameters

URIBuilder The method to  use  addParameters() to construct the parameters of the GET request.

package com.wdbyte.httpclient;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.BasicNameValuePair;
import org.apache.hc.core5.net.URIBuilder;

/**
* @author https://www.wdbyte.com
 */
public class HttpClient5GetParams {

    public static void main(String[] args) {
        String result = get("http://httpbin.org/get");
        System.out.println(result);
    }

    public static String get(String url) {
        String resultContent = null;
        HttpGet httpGet = new HttpGet(url);
        // 表单参数
        List<NameValuePair> nvps = new ArrayList<>();
        // GET 请求参数
        nvps.add(new BasicNameValuePair("username", "wdbyte.com"));
        nvps.add(new BasicNameValuePair("password", "secret"));
        // 增加到请求 URL 中
        try {
            URI uri = new URIBuilder(new URI(url))
                .addParameters(nvps)
                .build();
            httpGet.setUri(uri);
        } catch (URISyntaxException e) {
            throw new RuntimeException(e);
        }

        try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
            try (CloseableHttpResponse response = httpclient.execute(httpGet)) {
                // 获取状态码
                System.out.println(response.getVersion()); // HTTP/1.1
                System.out.println(response.getCode()); // 200
                System.out.println(response.getReasonPhrase()); // OK
                HttpEntity entity = response.getEntity();
                // 获取响应信息
                resultContent = EntityUtils.toString(entity);
            }
        } catch (IOException | ParseException e) {
            e.printStackTrace();
        }
        return resultContent;
    }
}

Output information:

{
  "args": {
    "password": "secret", 
    "username": "wdbyte.com"
  }, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/1.8.0_151)", 
    "X-Amzn-Trace-Id": "Root=1-62ecc660-69d58a226aefb1b6226541ec"
  }, 
  "origin": "42.120.75.185", 
  "url": "http://httpbin.org/get?username=wdbyte.com&password=secret"
}

The following is the format of the request response information obtained by capturing packets:

// 请求信息
GET /get?username=wdbyte.com&password=secret HTTP/1.1
Accept-Encoding: gzip, x-gzip, deflate
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/1.8.0_151)

// 响应信息
HTTP/1.1 200 OK
Date: Fri, 05 Aug 2022 07:27:30 GMT
Content-Type: application/json
Content-Length: 405
Connection: keep-alive
Server: gunicorn/19.9.0
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

{
  "args": {
    "password": "secret", 
    "username": "wdbyte.com"
  }, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/1.8.0_151)", 
    "X-Amzn-Trace-Id": "Root=1-62ecc660-69d58a226aefb1b6226541ec"
  }, 
  "origin": "42.120.75.185", 
  "url": "http://httpbin.org/get?username=wdbyte.com&password=secret"
}

HttpClient 5 POST request

The following demonstrates launching a POST request with form parameters.

parameter:username=wdbyte.com&password=secret

package com.wdbyte.httpclient;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.BasicNameValuePair;

/**
* @author https://www.wdbyte.com
 */
public class HttpClient5Post {

    public static void main(String[] args) {
        String result = post("http://httpbin.org/post");
        System.out.println(result);
    }
    public static String post(String url) {
        String result = null;
        HttpPost httpPost = new HttpPost(url);
        // 表单参数
        List<NameValuePair> nvps = new ArrayList<>();
        // POST 请求参数
        nvps.add(new BasicNameValuePair("username", "wdbyte.com"));
        nvps.add(new BasicNameValuePair("password", "secret"));
        httpPost.setEntity(new UrlEncodedFormEntity(nvps));
        try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
            try (CloseableHttpResponse response = httpclient.execute(httpPost)) {
                System.out.println(response.getVersion()); // HTTP/1.1
                System.out.println(response.getCode()); // 200
                System.out.println(response.getReasonPhrase()); // OK

                HttpEntity entity = response.getEntity();
                // 获取响应信息
                result = EntityUtils.toString(entity);
                // 确保流被完全消费
                EntityUtils.consume(entity);
            }
        } catch (IOException | ParseException e) {
            e.printStackTrace();
        }
        return result;
    }

}

Output information:

HTTP/1.1
200
OK
{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "password": "secret", 
    "username": "wdbyte.com"
  }, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Content-Length": "35", 
    "Content-Type": "application/x-www-form-urlencoded; charset=ISO-8859-1", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/17)", 
    "X-Amzn-Trace-Id": "Root=1-62bb1ac8-489b2100728c81d70797a482"
  }, 
  "json": null, 
  "origin": "183.128.136.89", 
  "url": "http://httpbin.org/post"
}

The following is the request information captured by Wireshark:

POST /post HTTP/1.1
Accept-Encoding: gzip, x-gzip, deflate
Content-Length: 35
Content-Type: application/x-www-form-urlencoded; charset=ISO-8859-1
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/17)

username=wdbyte.com&password=secret

HttpClient 5 Fluent POST

Using the Fluent API provided by Apache HttpClient 5 can initiate POST requests more conveniently, but there are fewer operable places.

Send a simple form parameter as well:username=wdbyte.com&password=secret

package com.wdbyte.httpclient;

import java.io.IOException;

import org.apache.hc.client5.http.fluent.Request;
import org.apache.hc.core5.http.message.BasicNameValuePair;

/**
* @author https://www.wdbyte.com
 */
public class HttpClient5PostFluent {

    public static void main(String[] args) {
        String result = post("http://httpbin.org/post");
        System.out.println(result);
    }

    public static String post(String url) {
        String result = null;
        Request request = Request.post(url);
        // POST 请求参数
        request.bodyForm(
            new BasicNameValuePair("username", "wdbyte.com"),
            new BasicNameValuePair("password", "secret"));
        try {
            result = request.execute().returnContent().asString();
        } catch (IOException e) {
            e.printStackTrace();
        }
        return result;
    }
}

Output information:

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "password": "secret", 
    "username": "wdbyte.com"
  }, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Content-Length": "35", 
    "Content-Type": "application/x-www-form-urlencoded; charset=ISO-8859-1", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/17)", 
    "X-Amzn-Trace-Id": "Root=1-62bb1c8a-7aee8c004f06919f31a2b533"
  }, 
  "json": null, 
  "origin": "183.128.136.89", 
  "url": "http://httpbin.org/post"
}

HttpClient5 POST JSON parameters

Use  StringEntity the class to store JSON parameters.

package com.wdbyte.httpclient;

import java.io.IOException;

import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;

/**
* @author https://www.wdbyte.com
 */
public class HttpClient5PostWithJson {

    public static void main(String[] args) {
        String json = "{"
            + "    \"password\": \"secret\","
            + "    \"username\": \"wdbyte.com\""
            + "}";
        String result = post("http://httpbin.org/post", json);
        System.out.println(result);
    }

    public static String post(String url, String jsonBody) {
        String result = null;
        HttpPost httpPost = new HttpPost(url);
        httpPost.setEntity(new StringEntity(jsonBody, ContentType.APPLICATION_JSON));
      
        try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
            try (CloseableHttpResponse response = httpclient.execute(httpPost)) {
                // 获取响应信息
                result = EntityUtils.toString(response.getEntity());
            }
        } catch (IOException | ParseException e) {
            e.printStackTrace();
        }
        return result;
    }

}

Output information:

{
  "args": {}, 
  "data": "{    \"password\": \"secret\",    \"username\": \"wdbyte.com\"}", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Content-Length": "55", 
    "Content-Type": "text/plain; charset=ISO-8859-1", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/17)", 
    "X-Amzn-Trace-Id": "Root=1-62bb1dbb-5a963c1d798b06be3ee1a15e"
  }, 
  "json": {
    "password": "secret", 
    "username": "wdbyte.com"
  }, 
  "origin": "183.128.136.89", 
  "url": "http://httpbin.org/post"
}

The following is the request response information captured by Wireshark:

// 请求信息
POST /post HTTP/1.1
Accept-Encoding: gzip, x-gzip, deflate
Content-Length: 55
Content-Type: application/json; charset=UTF-8
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/17)

{    "password": "secret",    "username": "wdbyte.com"}

// 响应信息
HTTP/1.1 200 OK
Date: Tue, 28 Jun 2022 15:30:17 GMT
Content-Type: application/json
Content-Length: 573
Connection: keep-alive
Server: gunicorn/19.9.0
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

{
  "args": {}, 
  "data": "{    \"password\": \"secret\",    \"username\": \"wdbyte.com\"}", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Content-Length": "55", 
    "Content-Type": "application/json; charset=UTF-8", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/17)", 
    "X-Amzn-Trace-Id": "Root=1-62bb1e89-64db55730a0361c720232ccd"
  }, 
  "json": {
    "password": "secret", 
    "username": "wdbyte.com"
  }, 
  "origin": "183.128.136.89", 
  "url": "http://httpbin.org/post"
}

HttpClient 5 setting timeout

Use  RequestConfig an object to configure the timeout.

package com.wdbyte.httpclient;

import java.io.IOException;

import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.config.RequestConfig;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.ParseException;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.util.Timeout;

/**
* @author https://www.wdbyte.com
 */
public class HttpClient5GetWithTimeout {

    public static void main(String[] args) {
        String result = get("http://httpbin.org/get");
        System.out.println(result);
    }

    public static String get(String url) {
        String resultContent = null;
        // 设置超时时间
        RequestConfig config = RequestConfig.custom()
            .setConnectTimeout(Timeout.ofMilliseconds(5000L))
            .setConnectionRequestTimeout(Timeout.ofMilliseconds(5000L))
            .setResponseTimeout(Timeout.ofMilliseconds(5000L))
            .build();
        // 请求级别的超时
        HttpGet httpGet = new HttpGet(url);
        //httpGet.setConfig(config);
        //try (CloseableHttpClient httpclient = HttpClients.createDefault()) {
        // 客户端级别的超时
        try (CloseableHttpClient httpclient = HttpClients.custom().setDefaultRequestConfig(config).build()) {
            try (CloseableHttpResponse response = httpclient.execute(httpGet)) {
                // 获取状态码
                System.out.println(response.getVersion()); // HTTP/1.1
                System.out.println(response.getCode()); // 200
                System.out.println(response.getReasonPhrase()); // OK
                HttpEntity entity = response.getEntity();
                // 获取响应信息
                resultContent = EntityUtils.toString(entity);
            }
        } catch (IOException | ParseException e) {
            e.printStackTrace();
        }
        return resultContent;
    }

}

HttpClient 5 asynchronous request

The following demonstrates three HttpClient 5 asynchronous request methods.

package com.wdbyte.httpclient;

import java.io.IOException;
import java.nio.CharBuffer;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

import org.apache.hc.client5.http.async.methods.AbstractCharResponseConsumer;
import org.apache.hc.client5.http.async.methods.SimpleHttpRequest;
import org.apache.hc.client5.http.async.methods.SimpleHttpRequests;
import org.apache.hc.client5.http.async.methods.SimpleHttpResponse;
import org.apache.hc.client5.http.impl.async.CloseableHttpAsyncClient;
import org.apache.hc.client5.http.impl.async.HttpAsyncClients;
import org.apache.hc.core5.concurrent.FutureCallback;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.HttpException;
import org.apache.hc.core5.http.HttpResponse;
import org.apache.hc.core5.http.nio.AsyncRequestProducer;
import org.apache.hc.core5.http.nio.support.AsyncRequestBuilder;

/**
 * HttpClient 5 异步请求
* @author https://www.wdbyte.com
 * @date 2022/06/25
 */
public class HttpClient5Async {

    public static void main(String[] args) {
        getAsync1("http://httpbin.org/get");
        getAsync2("http://httpbin.org/get");
        getAsync3("http://httpbin.org/get");
    }

    /**
     * 异步请求
     *
     * @param url
     * @return
     */
    public static String getAsync1(String url) {
        try (CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault()) {
            // 开始 http clinet
            httpclient.start();
            // 执行请求
            SimpleHttpRequest request1 = SimpleHttpRequests.get(url);
            Future<SimpleHttpResponse> future = httpclient.execute(request1, null);
            // 等待直到返回完毕
            SimpleHttpResponse response1 = future.get();
            System.out.println("getAsync1:" + request1.getRequestUri() + "->" + response1.getCode());
        } catch (IOException | ExecutionException | InterruptedException e) {
            throw new RuntimeException(e);
        }
        return null;
    }

    /**
     * 异步请求，根据响应情况回调
     *
     * @param url
     * @return
     */
    public static String getAsync2(String url) {
        try (CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault()) {
            // 开始 http clinet
            httpclient.start();
            // 根据请求响应情况进行回调操作
            CountDownLatch latch = new CountDownLatch(1);
            SimpleHttpRequest request = SimpleHttpRequests.get(url);
            httpclient.execute(request, new FutureCallback<SimpleHttpResponse>() {
                @Override
                public void completed(SimpleHttpResponse response2) {
                    latch.countDown();
                    System.out.println("getAsync2:" + request.getRequestUri() + "->" + response2.getCode());
                }

                @Override
                public void failed(Exception ex) {
                    latch.countDown();
                    System.out.println("getAsync2:" + request.getRequestUri() + "->" + ex);
                }

                @Override
                public void cancelled() {
                    latch.countDown();
                    System.out.println("getAsync2:" + request.getRequestUri() + " cancelled");
                }

            });
            latch.await();
        } catch (IOException | InterruptedException e) {
            throw new RuntimeException(e);
        }
        return null;
    }

    /**
     * 异步请求，对响应流做点什么
     *
     * @param url
     * @return
     */
    public static String getAsync3(String url) {
        try (CloseableHttpAsyncClient httpclient = HttpAsyncClients.createDefault()) {
            // 开始 http clinet
            httpclient.start();
            // 根据请求响应情况进行回调操作
            SimpleHttpRequest request = SimpleHttpRequests.get(url);

            CountDownLatch latch = new CountDownLatch(1);
            AsyncRequestProducer producer = AsyncRequestBuilder.get("http://httpbin.org/get").build();
            AbstractCharResponseConsumer<HttpResponse> consumer3 = new AbstractCharResponseConsumer<HttpResponse>() {

                HttpResponse response;

                @Override
                protected void start(HttpResponse response, ContentType contentType) throws HttpException, IOException {
                    System.out.println("getAsync3: 开始响应....");
                    this.response = response;
                }

                @Override
                protected int capacityIncrement() {
                    return Integer.MAX_VALUE;
                }

                @Override
                protected void data(CharBuffer data, boolean endOfStream) throws IOException {
                    System.out.println("getAsync3: 收到数据....");
                    // Do something useful
                }

                @Override
                protected HttpResponse buildResult() throws IOException {
                    System.out.println("getAsync3: 接收完毕...");
                    return response;
                }

                @Override
                public void releaseResources() {
                }

            };
            httpclient.execute(producer, consumer3, new FutureCallback<HttpResponse>() {

                @Override
                public void completed(HttpResponse response) {
                    latch.countDown();
                    System.out.println("getAsync3: "+request.getRequestUri() + "->" + response.getCode());
                }

                @Override
                public void failed(Exception ex) {
                    latch.countDown();
                    System.out.println("getAsync3: "+request.getRequestUri() + "->" + ex);
                }

                @Override
                public void cancelled() {
                    latch.countDown();
                    System.out.println("getAsync3: "+request.getRequestUri() + " cancelled");
                }

            });
            latch.await();
        } catch (IOException | InterruptedException e) {
            throw new RuntimeException(e);
        }
        return null;

    }
}

Output result:

getAsync1:/get->200
getAsync2:/get->200
getAsync3: 开始响应....
getAsync3: 收到数据....
getAsync3: 收到数据....
getAsync3: 收到数据....
getAsync3: 接收完毕...
getAsync3: /get->200

HttpClient 5 get cookie

The response of the request  http://httpbin.org/cookies/set/cookieName/www.wdbyte.com will have a cookie information, where the name is  cookieNameand the value is  www.wdbyte.com, we use this as a test.

Postman requests the test, and you can see that the cookie information is responded.

 

Write the Java code below for request testing

package com.wdbyte.httpclient;

import java.util.List;

import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.cookie.BasicCookieStore;
import org.apache.hc.client5.http.cookie.Cookie;
import org.apache.hc.client5.http.cookie.CookieStore;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.impl.cookie.BasicClientCookie;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.io.entity.EntityUtils;

/**
 * 这个例子演示了使用本地HTTP上下文填充, 自定义属性
 */
public class HttpClient5WithCookie {

    public static void main(final String[] args) throws Exception {
        try (final CloseableHttpClient httpclient = HttpClients.createDefault()) {
            // 创建一个本地的 Cookie 存储
            final CookieStore cookieStore = new BasicCookieStore();
            // BasicClientCookie clientCookie = new BasicClientCookie("name", "www.wdbyte.com");
            // clientCookie.setDomain("http://httpbin.org/cookies");
            // 过期时间
            // clientCookie.setExpiryDate(new Date());
            // 添加到本地 Cookie
            // cookieStore.addCookie(clientCookie);

            // 创建本地 HTTP 请求上下文 HttpClientContext
            final HttpClientContext localContext = HttpClientContext.create();
            // 绑定 cookieStore 到 localContext
            localContext.setCookieStore(cookieStore);

            final HttpGet httpget = new HttpGet("http://httpbin.org/cookies/set/cookieName/www.wdbyte.com");
            System.out.println("执行请求 " + httpget.getMethod() + " " + httpget.getUri());

            // 获取 Coolie 信息
            try (final CloseableHttpResponse response = httpclient.execute(httpget, localContext)) {
                System.out.println("----------------------------------------");
                System.out.println(response.getCode() + " " + response.getReasonPhrase());
                final List<Cookie> cookies = cookieStore.getCookies();
                for (int i = 0; i < cookies.size(); i++) {
                    System.out.println("Local cookie: " + cookies.get(i));
                }
                EntityUtils.consume(response.getEntity());
            }
        }
    }

}

Output result:

执行请求 GET http://httpbin.org/cookies/set/cookieName/www.wdbyte.com
----------------------------------------
200 OK
Local cookie: [name: cookieName; value: www.wdbyte.com; domain: httpbin.org; path: /; expiry: null]

HttpClient 5 read file content request

Prepare a file params.json with JSON content format.

{"name":"www.wdbyte.com"}

Read this file as a request parameter to initiate a request.

package com.wdbyte.httpclient;

import java.io.File;
import java.io.FileInputStream;

import org.apache.hc.client5.http.classic.methods.HttpPost;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.FileEntity;
import org.apache.hc.core5.http.io.entity.InputStreamEntity;

/**
 * 加载数据流作为 POST 请求参数
 */
public class HttpClient5ChunkEncodedPost {

    public static void main(final String[] args) throws Exception {
        String params = "/Users/darcy/params.json";

        try (final CloseableHttpClient httpclient = HttpClients.createDefault()) {
            final HttpPost httppost = new HttpPost("http://httpbin.org/post");

            final InputStreamEntity reqEntity = new InputStreamEntity(new FileInputStream(params), -1,
                ContentType.APPLICATION_JSON);
            // 也可以使用 FileEntity 的形式
            // FileEntity reqEntity = new FileEntity(new File(params), ContentType.APPLICATION_JSON);

            httppost.setEntity(reqEntity);

            System.out.println("执行请求 " + httppost.getMethod() + " " + httppost.getUri());
            try (final CloseableHttpResponse response = httpclient.execute(httppost)) {
                System.out.println("----------------------------------------");
                System.out.println(response.getCode() + " " + response.getReasonPhrase());
                System.out.println(EntityUtils.toString(response.getEntity()));
            }
        }
    }
}

Output result:

执行请求 POST http://httpbin.org/post
----------------------------------------
200 OK
{
  "args": {}, 
  "data": "{\"name\":\"www.wdbyte.com\"}\n", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Content-Length": "26", 
    "Content-Type": "application/json; charset=UTF-8", 
    "Host": "httpbin.org", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/1.8.0_151)", 
    "X-Amzn-Trace-Id": "Root=1-62ee4d95-1f956d4303cea09c52694c86"
  }, 
  "json": {
    "name": "www.wdbyte.com"
  }, 
  "origin": "42.120.74.238", 
  "url": "http://httpbin.org/post"
}

HttpClient 5 form login

Form login can be understood as the process of initiating a request carrying authentication information and then getting a response cookie. Of course, this is not only applicable to form login, but also simply initiate a request carrying form information.

The POST request should be used to send the form parameter test, but there is no corresponding interface for testing in httpbin.org, so it is replaced by a GET request

Sample code:

package com.wdbyte.httpclient;

import java.util.ArrayList;
import java.util.List;

import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.cookie.BasicCookieStore;
import org.apache.hc.client5.http.cookie.Cookie;
import org.apache.hc.client5.http.entity.UrlEncodedFormEntity;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.NameValuePair;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.message.BasicNameValuePair;

/**
 * 演示基于表单的登录
 * 
 * @author https://www.wdbyte.com
 */
public class HttpClient5FormLogin {

    public static void main(final String[] args) throws Exception {
        final BasicCookieStore cookieStore = new BasicCookieStore();
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCookieStore(cookieStore)
                .build()) {

            // 本应该使用 POST 请求发送表单参数，但是在 httpbin.org 中没有对应的接口用于测试，所以这里换成了 GET 请求
            // HttpPost httpPost = new HttpPost("http://httpbin.org/cookies/set/username/wdbyte.com");
            HttpGet httpPost = new HttpGet("http://httpbin.org/cookies/set/username/wdbyte.com");
            // POST 表单请求参数
            List<NameValuePair> nvps = new ArrayList<>();
            nvps.add(new BasicNameValuePair("username", "wdbyte.com"));
            nvps.add(new BasicNameValuePair("password", "secret"));
            httpPost.setEntity(new UrlEncodedFormEntity(nvps));

            try (final CloseableHttpResponse response2 = httpclient.execute(httpPost)) {
                final HttpEntity entity = response2.getEntity();

                System.out.println("Login form get: " + response2.getCode() + " " + response2.getReasonPhrase());
                System.out.println("当前响应信息 "+EntityUtils.toString(entity));;

                System.out.println("Post 登录 Cookie:");
                final List<Cookie> cookies = cookieStore.getCookies();
                if (cookies.isEmpty()) {
                    System.out.println("None");
                } else {
                    for (int i = 0; i < cookies.size(); i++) {
                        System.out.println("- " + cookies.get(i));
                    }
                }
            }
        }
    }
}

Output result:

Login form get: 200 OK
当前响应信息 {
  "cookies": {
    "username": "wdbyte.com"
  }
}

Post 登录 Cookie:
- [name: username; value: wdbyte.com; domain: httpbin.org; path: /; expiry: null]

HttpClient 5 Basic Authorization

HTTP Basic Authentication (Basic Authorization) is a relatively simple authentication implementation, the main process is as follows

1. 1. Request an HTTP interface that requires basic authentication, but does not carry authentication information.

2. 2. At this time, a 401 status code will be responded, and the WWW-Authenticate in the response header will prompt that basic authentication is required.

3. 3. The user concatenates the authentication information to be submitted with a colon, then encodes it in base64, and then concatenates Basic at the beginning of the obtained string and puts it in the request header Authorization.

4. 4. The authentication is successful and the response is successful.

You can open the following URL through your browser to perform the basic authentication test.

http://httpbin.org/basic-auth/admin/123456

Implementation in Apache HttpClient 5.

package com.wdbyte.httpclient;

import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.io.entity.EntityUtils;

/**
 * 一个简单的示例,它使用HttpClient执行HTTP请求;
 * 一个需要进行用户身份验证的目标站点。
 */
public class HttpClient5BasicAuthentication {

    public static void main(final String[] args) throws Exception {
        final BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
        credsProvider.setCredentials(
                new AuthScope("httpbin.org", 80),
                new UsernamePasswordCredentials("admin", "123456".toCharArray()));
        try (final CloseableHttpClient httpclient = HttpClients.custom()
                .setDefaultCredentialsProvider(credsProvider)
                .build()) {
            final HttpGet httpget = new HttpGet("http://httpbin.org/basic-auth/admin/123456");

            System.out.println("执行请求" + httpget.getMethod() + " " + httpget.getUri());
            try (final CloseableHttpResponse response = httpclient.execute(httpget)) {
                System.out.println("----------------------------------------");
                System.out.println(response.getCode() + " " + response.getReasonPhrase());
                System.out.println(EntityUtils.toString(response.getEntity()));
            }
        }
    }
}

Output result:

执行请求GET http://httpbin.org/basic-auth/user/passwd
----------------------------------------
200 OK
{
  "authenticated": true, 
  "user": "user"
}

You can see the complete HTTP request response process by capturing packets.

// 请求
GET /basic-auth/user/passwd HTTP/1.1
Accept-Encoding: gzip, x-gzip, deflate
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/1.8.0_151)
// 响应
HTTP/1.1 401 UNAUTHORIZED
Date: Sat, 06 Aug 2022 08:25:33 GMT
Content-Length: 0
Connection: keep-alive
Server: gunicorn/19.9.0
WWW-Authenticate: Basic realm="Fake Realm"
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true
// 请求
GET /basic-auth/user/passwd HTTP/1.1
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/1.8.0_151)
Authorization: Basic dXNlcjpwYXNzd2Q=
// 响应
HTTP/1.1 200 OK
Date: Sat, 06 Aug 2022 08:25:33 GMT
Content-Type: application/json
Content-Length: 47
Connection: keep-alive
Server: gunicorn/19.9.0
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

{
  "authenticated": true, 
  "user": "user"
}

HttpClient 5 Digest Authorization

The shortcomings of HTTP Basic Authorization are obvious. There are certain security risks in the transmission of passwords in plain text. The Digest Authorization authentication method solves the problem of plain text transmission. Here we will only introduce the relevant content of Digest, and use a simple diagram to illustrate the process of Digest authentication.

 

Digest Certification Process

Below is the code demo.

package com.wdbyte.httpclient;

import org.apache.hc.client5.http.auth.AuthExchange;
import org.apache.hc.client5.http.auth.AuthScheme;
import org.apache.hc.client5.http.auth.AuthScope;
import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
import org.apache.hc.client5.http.impl.auth.DigestScheme;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.client5.http.protocol.HttpClientContext;
import org.apache.hc.core5.http.HttpHost;
import org.apache.hc.core5.http.io.entity.EntityUtils;

/**
 *
 * HttpClient如何验证多个请求的示例
 * 使用相同的摘要方案。在初始请求/响应交换之后
 * 共享相同执行上下文的所有后续请求都可以重用
 * 要向服务器进行身份验证的最后一个摘要nonce值。
 */
public class HttpClient5PreemptiveDigestAuthentication {

    public static void main(final String[] args) throws Exception {
        try (final CloseableHttpClient httpclient = HttpClients.createDefault()) {

            final HttpHost target = new HttpHost("http", "httpbin.org", 80);

            final HttpClientContext localContext = HttpClientContext.create();
            final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider();
            credentialsProvider.setCredentials(
                    new AuthScope(target),
                    new UsernamePasswordCredentials("admin", "123456".toCharArray()));
            localContext.setCredentialsProvider(credentialsProvider);

            final HttpGet httpget = new HttpGet("http://httpbin.org/digest-auth/auth/admin/123456");

            System.out.println("执行请求 " + httpget.getMethod() + " " + httpget.getUri());
            for (int i = 0; i < 2; i++) {
                try (final CloseableHttpResponse response = httpclient.execute(target, httpget, localContext)) {
                    System.out.println("----------------------------------------");
                    System.out.println(response.getCode() + " " + response.getReasonPhrase());
                    EntityUtils.consume(response.getEntity());

                    final AuthExchange authExchange = localContext.getAuthExchange(target);
                    if (authExchange != null) {
                        final AuthScheme authScheme = authExchange.getAuthScheme();
                        if (authScheme instanceof DigestScheme) {
                            final DigestScheme digestScheme = (DigestScheme) authScheme;
                            System.out.println("Nonce: " + digestScheme.getNonce() +
                                    "; count: " + digestScheme.getNounceCount());
                        }
                    }
                }
            }
        }
    }

}

Through the packet capture tool, you can clearly see the process of the two requests. In the last request, the authentication information is directly shared, and there is no re-authentication process.

// 1. 请求
GET /digest-auth/auth/admin/123456 HTTP/1.1
Accept-Encoding: gzip, x-gzip, deflate
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/1.8.0_151)
// 2. 详情，提示认证，给出参数
HTTP/1.1 401 UNAUTHORIZED
Date: Fri, 12 Aug 2022 07:11:06 GMT
Content-Type: text/html; charset=utf-8
Content-Length: 0
Connection: keep-alive
Server: gunicorn/19.9.0
WWW-Authenticate: Digest realm="[email protected]", nonce="8dc5e7974a86a6fcc3cf73230b0c4a93", qop="auth", opaque="64b7f68b386c3acc38131f7472aa2079", algorithm=MD5, stale=FALSE
Set-Cookie: stale_after=never; Path=/
Set-Cookie: fake=fake_value; Path=/
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true
// 3. 参数+密码 加密后再次请求
GET /digest-auth/auth/admin/123456 HTTP/1.1
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/1.8.0_151)
Cookie: fake=fake_value; stale_after=never
Authorization: Digest username="admin", realm="[email protected]", nonce="8dc5e7974a86a6fcc3cf73230b0c4a93", uri="/digest-auth/auth/admin/123456", response="7c6726f8ac54c1ba28e19c71b2fc7338", qop=auth, nc=00000001, cnonce="2fa61501d47a9d39", algorithm=MD5, opaque="64b7f68b386c3acc38131f7472aa2079"
// 4. 认证成功，响应
HTTP/1.1 200 OK
Date: Fri, 12 Aug 2022 07:11:08 GMT
Content-Type: application/json
Content-Length: 48
Connection: keep-alive
Server: gunicorn/19.9.0
Set-Cookie: fake=fake_value; Path=/
Set-Cookie: stale_after=never; Path=/
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

{
  "authenticated": true, 
  "user": "admin"
}
// 5. 再次请求，共享了登录状态。
GET /digest-auth/auth/admin/123456 HTTP/1.1
Accept-Encoding: gzip, x-gzip, deflate
Host: httpbin.org
Connection: keep-alive
User-Agent: Apache-HttpClient/5.1.3 (Java/1.8.0_151)
Cookie: fake=fake_value; stale_after=never
Authorization: Digest username="admin", realm="[email protected]", nonce="8dc5e7974a86a6fcc3cf73230b0c4a93", uri="/digest-auth/auth/admin/123456", response="9955ac79f6a51a876a326449447f549d", qop=auth, nc=00000002, cnonce="2fa61501d47a9d39", algorithm=MD5, opaque="64b7f68b386c3acc38131f7472aa2079"
// 5. 认证成功，响应
HTTP/1.1 200 OK
Date: Fri, 12 Aug 2022 07:11:09 GMT
Content-Type: application/json
Content-Length: 48
Connection: keep-alive
Server: gunicorn/19.9.0
Set-Cookie: fake=fake_value; Path=/
Set-Cookie: stale_after=never; Path=/
Access-Control-Allow-Origin: *
Access-Control-Allow-Credentials: true

{
  "authenticated": true, 
  "user": "admin"
}

HttpClient 5 interceptor

The interceptor in HttpClient 5 can intercept and process each stage of the request process, and you   can see the nodes that can be intercepted through HttpClientBuilder the about method in HttpClient  5.Interceptor

 

HttpClient5 interceptor

Write an example below to initiate three requests, each request adds a parameter in the request header header  request-id , and then  request-id directly responds to the request with a value of 2 with 404 to end.

package com.wdbyte.httpclient;

import java.io.IOException;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.hc.client5.http.classic.ExecChain;
import org.apache.hc.client5.http.classic.ExecChain.Scope;
import org.apache.hc.client5.http.classic.ExecChainHandler;
import org.apache.hc.client5.http.classic.methods.HttpGet;
import org.apache.hc.client5.http.impl.ChainElement;
import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
import org.apache.hc.client5.http.impl.classic.HttpClients;
import org.apache.hc.core5.http.ClassicHttpRequest;
import org.apache.hc.core5.http.ClassicHttpResponse;
import org.apache.hc.core5.http.ContentType;
import org.apache.hc.core5.http.EntityDetails;
import org.apache.hc.core5.http.Header;
import org.apache.hc.core5.http.HttpEntity;
import org.apache.hc.core5.http.HttpException;
import org.apache.hc.core5.http.HttpRequest;
import org.apache.hc.core5.http.HttpRequestInterceptor;
import org.apache.hc.core5.http.HttpStatus;
import org.apache.hc.core5.http.io.entity.EntityUtils;
import org.apache.hc.core5.http.io.entity.StringEntity;
import org.apache.hc.core5.http.message.BasicClassicHttpResponse;
import org.apache.hc.core5.http.protocol.HttpContext;

/**
 * 展示如何在请求和响应时进行拦截进行自定义处理。
 */
public class HttpClient5Interceptors {

    public static void main(final String[] args) throws Exception {
        try (final CloseableHttpClient httpclient = HttpClients.custom()
            // 添加一个请求 id 到请求 header
            .addRequestInterceptorFirst(new HttpRequestInterceptor() {
                private final AtomicLong count = new AtomicLong(0);
                @Override
                public void process(
                    final HttpRequest request,
                    final EntityDetails entity,
                    final HttpContext context) throws HttpException, IOException {
                    request.setHeader("request-id", Long.toString(count.incrementAndGet()));
                }
            })
            .addExecInterceptorAfter(ChainElement.PROTOCOL.name(), "custom", new ExecChainHandler() {
                // 请求 id 为 2 的，模拟 404 响应，并自定义响应的内容。
                @Override
                public ClassicHttpResponse execute(
                    final ClassicHttpRequest request,
                    final Scope scope,
                    final ExecChain chain) throws IOException, HttpException {

                    final Header idHeader = request.getFirstHeader("request-id");
                    if (idHeader != null && "2".equalsIgnoreCase(idHeader.getValue())) {
                        final ClassicHttpResponse response = new BasicClassicHttpResponse(HttpStatus.SC_NOT_FOUND,
                            "Oppsie");
                        response.setEntity(new StringEntity("bad luck", ContentType.TEXT_PLAIN));
                        return response;
                    } else {
                        return chain.proceed(request, scope);
                    }
                }
            })
            .build()) {

            for (int i = 0; i < 3; i++) {
                final HttpGet httpget = new HttpGet("http://httpbin.org/get");

                try (final CloseableHttpResponse response = httpclient.execute(httpget)) {
                    System.out.println("----------------------------------------");
                    System.out.println("执行请求 " + httpget.getMethod() + " " + httpget.getUri());
                    System.out.println(response.getCode() + " " + response.getReasonPhrase());
                    System.out.println(EntityUtils.toString(response.getEntity()));
                }
            }
        }
    }

}

Output the result.

----------------------------------------
执行请求 GET http://httpbin.org/get
200 OK
{
  "args": {}, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Host": "httpbin.org", 
    "Request-Id": "1", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/1.8.0_151)", 
    "X-Amzn-Trace-Id": "Root=1-62f615ba-658ccd42182d22534dbba82c"
  }, 
  "origin": "42.120.75.221", 
  "url": "http://httpbin.org/get"
}

----------------------------------------
执行请求 GET http://httpbin.org/get
404 Oppsie
bad luck
----------------------------------------
执行请求 GET http://httpbin.org/get
200 OK
{
  "args": {}, 
  "headers": {
    "Accept-Encoding": "gzip, x-gzip, deflate", 
    "Host": "httpbin.org", 
    "Request-Id": "3", 
    "User-Agent": "Apache-HttpClient/5.1.3 (Java/1.8.0_151)", 
    "X-Amzn-Trace-Id": "Root=1-62f615bb-4eb6ba10736ace0e21d0cb8c"
  }, 
  "origin": "42.120.75.221", 
  "url": "http://httpbin.org/get"
}