自己写的代码比较乱,所以使用的是老师源码
创建log4j文件:
pom.xml中所需的坐标:
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>PaChong</groupId>
<artifactId>PaChong</artifactId>
<version>0.0.1-SNAPSHOT</version>
<dependencies>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.25</version>
<!-- <scope>test</scope> -->
</dependency>
<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.10.2</version>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-io/commons-io -->
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.6</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.7</version>
</dependency>
</dependencies>
</project>
不带参的Get请求:
public class CrawlerFirst {
public static void main(String[] args) throws Exception {
//1. 打开浏览器,创建HttpClient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
//2. 输入网址,发起get请求创建HttpGet对象
HttpGet httpGet = new HttpGet("http://www.itcast.cn");
//3.按回车,发起请求,返回响应,使用HttpClient对象发起请求
CloseableHttpResponse response = httpClient.execute(httpGet);
//4. 解析响应,获取数据
//判断状态码是否是200
if (response.getStatusLine().getStatusCode() == 200) {
HttpEntity httpEntity = response.getEntity();
String content = EntityUtils.toString(httpEntity, "utf8");
System.out.println(content);
}
}
}
带参数的GET请求
public class HttpGetParamTest {
public static void main(String[] args) throws Exception {
//创建HttpClient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
//设置请求地址是:http://yun.itheima.com/search?keys=Java
//创建URIBuilder
URIBuilder uriBuilder = new URIBuilder("http://yun.itheima.com/search");
//设置参数
uriBuilder.setParameter("keys","Java");
//创建HttpGet对象,设置url访问地址
HttpGet httpGet = new HttpGet(uriBuilder.build());
System.out.println("发起请求的信息:"+httpGet);
CloseableHttpResponse response = null;
try {
//使用HttpClient发起请求,获取response
response = httpClient.execute(httpGet);
//解析响应
if (response.getStatusLine().getStatusCode() == 200) {
String content = EntityUtils.toString(response.getEntity(), "utf8");
System.out.println(content.length());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
//关闭response
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
try {
httpClient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
不带参数的Post请求
public static void main(String[] args) throws IOException {
//创建HttpClient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
//创建HttpGet请求
HttpPost httpPost = new HttpPost("http://www.itcast.cn/");
CloseableHttpResponse response = null;
try {
//使用HttpClient发起请求
response = httpClient.execute(httpPost);
//判断响应状态码是否为200
if (response.getStatusLine().getStatusCode() == 200) {
//如果为200表示请求成功,获取返回数据
String content = EntityUtils.toString(response.getEntity(), "UTF-8");
//打印数据长度
System.out.println(content);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//释放连接
if (response == null) {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
httpClient.close();
}
}
}
带参的Post请求
public static void main(String[] args) throws IOException {
//创建HttpClient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
//创建HttpGet请求
HttpPost httpPost = new HttpPost("http://www.itcast.cn/");
//声明存放参数的List集合
List<NameValuePair> params = new ArrayList<NameValuePair>();
params.add(new BasicNameValuePair("keys", "java"));
//创建表单数据Entity
UrlEncodedFormEntity formEntity = new UrlEncodedFormEntity(params, "UTF-8");
//设置表单Entity到httpPost请求对象中
httpPost.setEntity(formEntity);
CloseableHttpResponse response = null;
try {
//使用HttpClient发起请求
response = httpClient.execute(httpPost);
//判断响应状态码是否为200
if (response.getStatusLine().getStatusCode() == 200) {
//如果为200表示请求成功,获取返回数据
String content = EntityUtils.toString(response.getEntity(), "UTF-8");
//打印数据长度
System.out.println(content);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//释放连接
if (response == null) {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
httpClient.close();
}
}
}
连接池管理器
public class HttpClientPoolTest {
public static void main(String[] args) {
//创建连接池管理器
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
//设置最大连接数
cm.setMaxTotal(100);
//设置每个主机的最大连接数
cm.setDefaultMaxPerRoute(10);
//使用连接池管理器发起请求
doGet(cm);
doGet(cm);
}
private static void doGet(PoolingHttpClientConnectionManager cm) {
//不是每次创建新的HttpClient,而是从连接池中获取HttpClient对象
CloseableHttpClient httpClient = HttpClients.custom().setConnectionManager(cm).build();
HttpGet httpGet = new HttpGet("http://www.itcast.cn");
CloseableHttpResponse response = null;
try {
response = httpClient.execute(httpGet);
if (response.getStatusLine().getStatusCode() == 200) {
String content = EntityUtils.toString(response.getEntity(), "utf8");
System.out.println(content.length());
}
} catch (IOException e) {
e.printStackTrace();
}finally {
if (response != null) {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
//不能关闭HttpClient,由连接池管理HttpClient
//httpClient.close();
}
}
}
}
请求参数:有时候因为网络,或者目标服务器的原因,请求需要更长的时间才能完成,我们需要自定义相关时间
public static void main(String[] args) throws IOException {
//创建HttpClient对象
CloseableHttpClient httpClient = HttpClients.createDefault();
//创建HttpGet请求
HttpGet httpGet = new HttpGet("http://www.itcast.cn/");
//设置请求参数
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(1000)//设置创建连接的最长时间
.setConnectionRequestTimeout(500)//设置获取连接的最长时间
.setSocketTimeout(10 * 1000)//设置数据传输的最长时间
.build();
httpGet.setConfig(requestConfig);
CloseableHttpResponse response = null;
try {
//使用HttpClient发起请求
response = httpClient.execute(httpGet);
//判断响应状态码是否为200
if (response.getStatusLine().getStatusCode() == 200) {
//如果为200表示请求成功,获取返回数据
String content = EntityUtils.toString(response.getEntity(), "UTF-8");
//打印数据长度
System.out.println(content);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
//释放连接
if (response == null) {
try {
response.close();
} catch (IOException e) {
e.printStackTrace();
}
httpClient.close();
}
}
}
使用dom方式遍历文档
1.根据id查询元素getElementById
2.根据标签获取元素getElementsByTag
3.根据class获取元素getElementsByClass
4.根据属性获取元素getElementsByAttribute
//获取元素
Element element = document.getElementById("test");
//1. 从元素中获取id
String str = element.id();
//2. 从元素中获取className
str = element.className();
//3. 从元素中获取属性的值attr
str = element.attr("id");
//4. 从元素中获取所有属性attributes
str = element.attributes().toString();
//5. 从元素中获取文本内容text
str = element.text();
Selector选择器概述
tagname: 通过标签查找元素,比如:span
#id: 通过ID查找元素,比如:# city_bj
.class: 通过class名称查找元素,比如:.class_a
[attribute]: 利用属性查找元素,比如:[abc]
[attr=value]: 利用属性值来查找元素,比如:[class=s_name]
//tagname: 通过标签查找元素,比如:span
Elements span = document.select("span");
for (Element element : span) {
System.out.println(element.text());
}
//#id: 通过ID查找元素,比如:#city_bjj
String str = document.select("#city_bj").text();
//.class: 通过class名称查找元素,比如:.class_a
str = document.select(".class_a").text();
//[attribute]: 利用属性查找元素,比如:[abc]
str = document.select("[abc]").text();
//[attr=value]: 利用属性值来查找元素,比如:[class=s_name]
str = document.select("[class=s_name]").text();
Selector选择器组合使用
el#id: 标签+ID,比如: h3#city_bj
el.class: 标签+class,比如: li.class_a
el[attr]: 标签+属性名,比如: span[abc]
任意组合: 比如:span[abc].s_name
ancestor child: 查找某个元素下子元素,比如:.city_con li 查找"city_con"下的所有li
parent > child: 查找某个父元素下的直接子元素,比如:
.city_con > ul > li 查找city_con第一级(直接子元素)的ul,再找所有ul下的第一级li
parent > : 查找某个父元素下所有直接子元素
//el#id: 标签+ID,比如: h3#city_bj
String str = document.select("h3#city_bj").text();
//el.class: 标签+class,比如: li.class_a
str = document.select("li.class_a").text();
//el[attr]: 标签+属性名,比如: span[abc]
str = document.select("span[abc]").text();
//任意组合,比如:span[abc].s_name
str = document.select("span[abc].s_name").text();
//ancestor child: 查找某个元素下子元素,比如:.city_con li 查找"city_con"下的所有li
str = document.select(".city_con li").text();
//parent > child: 查找某个父元素下的直接子元素,
//比如:.city_con > ul > li 查找city_con第一级(直接子元素)的ul,再找所有ul下的第一级li
str = document.select(".city_con > ul > li").text();
//parent > * 查找某个父元素下所有直接子元素.city_con > *
str = document.select(".city_con > *").text();