Reptile entry procedures and the use of HttpClient

Getting Case:

　　1. Create a maven project, introduction of dependence:

<dependencies>
    <!-- HttpClient -->
    <dependency>
        <groupId>org.apache.httpcomponents</groupId>
        <artifactId>httpclient</artifactId>
        <version>4.5.3</version>
    </dependency>
    <!-- 日志 -->
    <dependency>
        <groupId>org.slf4j</groupId>
        <artifactId>slf4j-log4j12</artifactId>
        <version>1.7.25</version>
    </dependency>
    <!-- junit -->
    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
    </dependency>
</dependencies>

　　2. Add log4j.properties

log4j.rootLogger=DEBUG,A1
log4j.logger.com.fgy = DEBUG

log4j.appender.A1=org.apache.log4j.ConsoleAppender
log4j.appender.A1.layout=org.apache.log4j.PatternLayout
log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%t] [%c]-[%p] %m%n

　　3. Write the test code, data capture

@Test
public void testFirst() throws IOException {
    CloseableHttpClient client = HttpClients.createDefault();
    HttpGet httpGet = new HttpGet("https://www.cnblogs.com/roadlandscape/");
    CloseableHttpResponse response = client.execute(httpGet);
    if (response.getStatusLine().getStatusCode() == 200) {
        String context = EntityUtils.toString(response.getEntity(), "utf-8");
        System.out.println(context);
    }
}

GET request with parameters:

@Test
public void testGetParam() {
    CloseableHttpClient client = HttpClients.createDefault();
    // HttpGet httpGet = new HttpGet("https://s.taobao.com/search?q=vivo");
    CloseableHttpResponse response = null;
    try {
        URIBuilder builder = new URIBuilder("https://s.taobao.com/search");
        builder.addParameter("q", "vivo");
        HttpGet httpGet = new HttpGet(builder.build());
        response = client.execute(httpGet);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

POST request:

@Test
public void testPost() {
    CloseableHttpClient client = HttpClients.createDefault();
    HttpPost httpPost = new HttpPost("https://www.cnblogs.com/roadlandscape/");
    CloseableHttpResponse response = null;
    try {
        response = client.execute(httpPost);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    } catch (IOException e) {
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

POST request with parameters:

@Test
public void testPostParam() {
    CloseableHttpClient client = HttpClients.createDefault();
    CloseableHttpResponse response = null;
    try {
        HttpPost httpPost = new HttpPost("https://s.taobao.com/search");
        // 创建存放参数的list
        List<NameValuePair> params = new ArrayList<>();
        params.add(new BasicNameValuePair("q", "vivo"));
        // 创建表单数据 Entity
        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(params, "utf-8");
        // 设置表单 Entity 到 httpPost 中
        httpPost.setEntity(entity);
        response = client.execute(httpPost);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        try {
            client.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}

connection pool:

　　If each request must create HttpClient, there will be problems frequently create and destroy, you can use connection pooling to solve this problem.

public  class ConPool {
     public  static  void main (String [] args) throws IOException { 
        PoolingHttpClientConnectionManager cm & lt = new new PoolingHttpClientConnectionManager ();
         // set the maximum number of connections 
        cm.setMaxTotal (200 is );
         // Set the number of concurrent each host 
        cm.setDefaultMaxPerRoute (20 is ); 

        the doGet (cm & lt); 
    } 

    Private  static  void the doGet (cm & lt PoolingHttpClientConnectionManager) throws IOException { 
        CloseableHttpClient Client =. HttpClients.custom () setConnectionManager (cm & lt) .build (); 
        HttpGet HttpGet = new new HttpGet ( "https://www.cnblogs.com/roadlandscape/" ); 

        // set the request parameter 
        RequestConfig requestConfig = RequestConfig.custom () 
                .setConnectTimeout ( 1000) // set the maximum time to create connections 
                .setConnectionRequestTimeout (500) // set the retrieved maximum time connected 
                .setSocketTimeout (10 * 1000) // maximum time of design data transmitted 
                .build (); 
        HttpGet .setConfig (requestConfig); 

        CloseableHttpResponse Response = client.execute (HttpGet);
        if (response.getStatusLine().getStatusCode() == 200) {
            String context = EntityUtils.toString(response.getEntity(), "utf-8");
            System.out.println(context);
        }
    }
}

 

1.1.1. Write code