JAVA crawl data


/**
* httClient 请求 GET
* 获取数据测试
*/
@Test
public void testDepartmentList() {
//数据链接
String url = "https://search-x.jd.com/Search?callback=jQuery1352084&area=22&enc=utf-8&keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC&adType=7&urlcid3=672&page=1&ad_ids=291%3A33&xtest=new_search&_=1585561488653";
//网页链接
String referer = "https://search.jd.com/search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC&enc=utf-8&qrst=1&rt=1&stop=1&spm=2.1.1&vt=2&cid2=671&cid3=672&ev=exbrand_%E8%81%94%E6%83%B3%EF%BC%88Lenovo%EF%BC%89%5E&uc=0";
System.out.println("请求url:" + url);
HttpRequest request = HttpUtil.createGet(url);
request.header("Referer", referer);
String str2 = request.execute().body();

int length = str2.length();
int i = str2.indexOf("(");
System.out.println(i);
str2 = str2.substring(i+1, length - 1);
System.out.println("请求响应:" + str2);

JSONObject jsonObject = JSON.parseObject(str2);
System.out.println(jsonObject);
JSONArray jsonArray = jsonObject.getJSONArray("291");
int size = jsonArray.size();
for (int j = 0; j <size ; j++) {
JSONObject jsonObject1 = jsonArray.getJSONObject(j);
String ad_title = jsonObject1.getString("ad_title");

System.out.println(j+"ad_title="+ad_title);
}

System.out.println("Request Response: "+ str2);
}

Guess you like

Origin www.cnblogs.com/zhetiankj/p/12663020.html