搬运自cdp4j官网。。。
cdp4j
cdp4j是一个Java库，它提供了高级API来通过DevTools协议控制Chrome或Chromium。它可以用于自动使用网页和测试网页。cdp4j默认情况下可以完全运行，但可以配置为运行无头Chrome或Chromium。
cdp4j能做什么
您可以在浏览器中手动执行的大多数操作都可以使用cdp4j完成。以下是一些入门示例：
- 生成页面的屏幕截图和PDF。
- 爬网单页应用程序并生成预渲染的内容（服务器端渲染）。
- 自动执行表单提交，UI测试，键盘输入等。
- 创建最新的自动化测试环境。
- 使用最新的JavaScript和浏览器功能，可以在最新版本的Chromium中直接运行测试。
特征
- 支持Chrome DevTools协议（树梢）的全部功能
- 评估JavaScript
- 调用JavaScript函数
- 支持本地CSS选择器引擎
- 支持Sizzle选择器引擎
- 支持XPath查询
- 隐身浏览（私有标签）
- 全页截屏
- 触发鼠标事件（单击等）
- 发送键（文本，标签，回车等）
- 将浏览器中的日志条目（javascript，网络，存储等）重定向到slf4j
- 拦截网络（请求和响应）
- 以编程方式上传文件而无需第三方解决方案（不需要AWT机械手等）
- 获取和设置元素属性
- 支持无头铬/铬
- 向后导航，前进，停止，重新加载
- 清除缓存，清除cookie，列出cookie
- 设置并获取表单元素的值
- 支持事件处理
maven依赖jar包
<!-- https://mvnrepository.com/artifact/io.webfolder/cdp4j --> <dependency> <groupId>io.webfolder</groupId> <artifactId>cdp4j</artifactId> <version>3.0.13</version> </dependency> <!-- https://mvnrepository.com/artifact/org.jvnet.winp/winp --> <dependency> <groupId>org.jvnet.winp</groupId> <artifactId>winp</artifactId> <version>1.28</version> </dependency>
所有官网的例子都拉过来了，用啥自己找：
Attributes.java
import java.net.URL;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class Attributes {
public static void main(String[] args) {
URL url = Attributes.class.getResource("/attributes.html");
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate(url.toString());
session.waitDocumentReady();
String value = session.getAttribute("input", "value");
System.out.println("Old Value: " + value);
session.setAttribute("input", "value", "bar");
value = session.getAttribute("input", "value");
System.out.println("New Value: " + value);
}
}
}BasicAuthentication.java
import static io.webfolder.cdp.event.Events.NetworkRequestIntercepted;
import static java.util.Arrays.asList;
import java.util.Map;
import com.google.gson.Gson;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Network;
import io.webfolder.cdp.event.network.RequestIntercepted;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
import io.webfolder.cdp.type.constant.AuthResponse;
import io.webfolder.cdp.type.network.AuthChallengeResponse;
import io.webfolder.cdp.type.network.RequestPattern;
import io.webfolder.cdp.type.network.ResourceType;
public class BasicAuthentication {
@SuppressWarnings("unchecked")
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
Network network = session.getCommand().getNetwork();
network.enable();
RequestPattern pattern = new RequestPattern();
pattern.setUrlPattern("*");
pattern.setResourceType(ResourceType.Document);
network.setRequestInterception(asList(pattern));
// Disable newtork caching when intercepting
// https://github.com/GoogleChrome/puppeteer/pull/1154
network.setCacheDisabled(Boolean.TRUE);
session.addEventListener((e, v) -> {
if (NetworkRequestIntercepted.equals(e)) {
RequestIntercepted ri = (RequestIntercepted) v;
if (ri.getAuthChallenge() != null) {
AuthChallengeResponse acr = new AuthChallengeResponse();
acr.setUsername("user");
acr.setPassword("password");
acr.setResponse(AuthResponse.ProvideCredentials);
network.continueInterceptedRequest(ri.getInterceptionId(), null,
null, null, null, null, null, acr);
} else {
network.continueInterceptedRequest(ri.getInterceptionId());
}
}
});
session.navigate("https://httpbin.org/basic-auth/user/password");
session.wait(1000);
String content = (String) session
.evaluate("window.document.body.textContent");
System.out.println(content);
Map map = new Gson().fromJson(content, Map.class);
Object authenticated = map.get("authenticated");
Object user = map.get("user");
System.out.println("Authenticated: " + authenticated);
System.out.println("User:" + user);
}
}
}Bing.java
import static java.util.Locale.ENGLISH;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class Bing {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate("https://www.bing.com").waitDocumentReady()
.enableNetworkLog().click("input[type='search']")
.sendKeys("Microsoft").sendEnter().wait(1000);
String firstResult = session.getText("strong").toLowerCase(ENGLISH);
System.out.println("Query String : " + session.getQueryString());
System.out.println("Path name : " + session.getPathname());
System.out.println(firstResult);
}
}
}BingTranslator.java
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Option;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class BingTranslator {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate("https://www.bing.com/translator").waitDocumentReady()
.enableConsoleLog().enableDetailLog().enableNetworkLog();
Option en = session.getOptions("#t_sl").stream()
.filter(p -> "en".equals(p.getValue())).findFirst().get();
Option et = session.getOptions("#t_tl").stream()
.filter(p -> "et".equals(p.getValue())).findFirst().get();
session.click("#t_sl") // click source language
.wait(500).setSelectedIndex("#t_sl", en.getIndex()) // choose
// English
.wait(500).click("#t_tl") // click destination language
.wait(500).setSelectedIndex("#t_tl", et.getIndex()) // choose
// Estonian
.wait(500);
session.focus("#t_sv").sendKeys("hello world").wait(1000);
System.out.println(session.getValue("#t_tv"));
}
}
}CheckBox.java
import java.net.URL;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class CheckBox {
public static void main(String[] args) {
Launcher launcher = new Launcher();
URL url = CheckBox.class.getResource("/checkbox.html");
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate(url.toString());
session.waitDocumentReady();
System.out.println("Checked: " + session.isChecked("input[name='red']"));
session.setChecked("input[name='red']", true);
System.out.println("Checked: " + session.isChecked("input[name='red']"));
session.wait(1000);
session.setChecked("input[name='red']", false);
System.out.println("Checked: " + session.isChecked("input[name='red']"));
session.wait(1000);
}
}
}CloseSessionOnRedirect.java
import static io.webfolder.cdp.event.Events.NetworkRequestWillBeSent;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Network;
import io.webfolder.cdp.event.network.RequestWillBeSent;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
import io.webfolder.cdp.type.network.Response;
public class CloseSessionOnRedirect {
private static boolean terminateSession;
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
Network network = session.getCommand().getNetwork();
network.enable();
session.addEventListener((e, d) -> {
if (NetworkRequestWillBeSent.equals(e)) {
RequestWillBeSent rws = (RequestWillBeSent) d;
Response rr = rws.getRedirectResponse();
boolean isRedirect = rr != null && rr.getStatus() != null;
if (isRedirect) {
terminateSession = true;
session.stop();
session.close();
System.out.println("");
System.out.println(
"Redirect URL : " + rws.getRequest().getUrl());
System.out.println("Redirect Status Code : " + rr.getStatus());
System.out.println("Redirect Header : "
+ rws.getRedirectResponse().getHeaders());
System.out.println("");
}
}
});
session.navigate(
"https://httpbin.org/redirect-to?url=https://webfolder.io?cdp4j");
if (!terminateSession) {
session.waitDocumentReady();
}
}
}
}CodeCoverage.java
import static java.lang.String.format;
import java.net.URL;
import java.util.List;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Profiler;
import io.webfolder.cdp.session.Command;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
import io.webfolder.cdp.type.profiler.CoverageRange;
import io.webfolder.cdp.type.profiler.FunctionCoverage;
import io.webfolder.cdp.type.profiler.ScriptCoverage;
public class CodeCoverage {
public static void main(String[] args) {
URL url = CodeCoverage.class.getResource("/code-coverage.html");
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
Command command = session.getCommand();
Profiler profiler = command.getProfiler();
session.navigate(url.toString());
session.waitDocumentReady();
profiler.enable();
profiler.start();
session.callFunction("fibonacci", Double.class, 20);
profiler.stop();
List list = profiler.getBestEffortCoverage();
for (ScriptCoverage coverage : list) {
if (!coverage.getUrl().endsWith("test-lib.js")) {
continue;
}
String libName = coverage.getUrl().substring(
coverage.getUrl().lastIndexOf("/") + 1,
coverage.getUrl().length());
List functions = coverage.getFunctions();
System.out.println("");
System.out.println(
"======================================================================");
System.out.println(String.format(" %-40s %s %s", "[Function]",
"[Start Offset]", "[End Offset]"));
System.out.println(
"======================================================================");
for (FunctionCoverage functionCoverage : functions) {
if (functionCoverage.getFunctionName().isEmpty()) {
continue;
}
for (CoverageRange range : functionCoverage.getRanges()) {
System.out.println(format(" %-46s %-11d %d",
libName + "/" + functionCoverage.getFunctionName(),
range.getStartOffset(), range.getEndOffset()));
}
}
System.out.println("");
}
profiler.disable();
}
}
}Crawler.java
import static io.webfolder.cdp.event.Events.NetworkLoadingFinished;
import static io.webfolder.cdp.event.Events.NetworkResponseReceived;
import static java.util.Base64.getDecoder;
import static java.util.Collections.emptyList;
import static java.util.Collections.synchronizedSet;
import java.io.ByteArrayInputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.event.network.LoadingFinished;
import io.webfolder.cdp.event.network.ResponseReceived;
import io.webfolder.cdp.exception.CdpException;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
import io.webfolder.cdp.type.network.GetResponseBodyResult;
import io.webfolder.cdp.type.network.ResourceType;
public class Crawler {
public static class SitemapParser {
public List parse(String content) throws Exception {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
Document document = builder
.parse(new ByteArrayInputStream(content.getBytes()));
NodeList nodes = document.getElementsByTagName("loc");
List urls = new ArrayList<>();
for (int i = 0; i < nodes.getLength(); i++) {
Node node = nodes.item(i);
String tc = node.getTextContent();
if (tc != null) {
urls.add(tc);
}
}
return urls;
}
}
public static class Resource {
private String document;
private byte[] content;
private int status;
private Map responseHeaders;
private String url;
private String requestId;
public String getDocument() {
return document;
}
public void setDocument(String document) {
this.document = document;
}
public byte[] getContent() {
return content;
}
public void setContent(byte[] content) {
this.content = content;
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public Map getResponseHeaders() {
return responseHeaders;
}
public void setResponseHeaders(Map responseHeaders) {
this.responseHeaders = responseHeaders;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getRequestId() {
return requestId;
}
public void setRequestId(String requestId) {
this.requestId = requestId;
}
}
public static class HttpClient implements AutoCloseable {
private SessionFactory factory;
private Session session;
private int pageLoadTimeout;
private Set finishedResources = synchronizedSet(new HashSet<>());
public HttpClient(SessionFactory factory, int pageLoadTimeout) {
this.factory = factory;
this.pageLoadTimeout = pageLoadTimeout;
}
public Resource fetch(String url) {
Resource webResource = new Resource();
session = factory.create();
session.getCommand().getNetwork().enable();
session.addEventListener((e, d) -> {
if (NetworkLoadingFinished.equals(e)) {
LoadingFinished lf = (LoadingFinished) d;
finishedResources.add(lf.getRequestId());
}
if (NetworkResponseReceived.equals(e)) {
ResponseReceived rr = (ResponseReceived) d;
if (!rr.getResponse().getUrl().startsWith("https://")
&& !rr.getResponse().getUrl().startsWith("http://")) {
return;
}
ResourceType type = rr.getType();
if (ResourceType.Document.equals(type)) {
webResource.setResponseHeaders(
new ConcurrentHashMap<>(rr.getResponse().getHeaders()));
webResource.setStatus(rr.getResponse().getStatus().intValue());
webResource.setUrl(rr.getResponse().getUrl());
webResource.setRequestId(rr.getRequestId());
}
}
});
session.navigate(url);
session.waitDocumentReady(pageLoadTimeout);
for (String requestId : new ArrayList<>(finishedResources)) {
try {
GetResponseBodyResult rb = session.getCommand().getNetwork()
.getResponseBody(requestId);
if (rb.getBase64Encoded()) {
webResource.setContent(getDecoder().decode(rb.getBody()));
} else {
webResource.setDocument(rb.getBody());
}
} catch (CdpException e) {
// ignore
}
}
String ct = (String) webResource.getResponseHeaders().get("content-type");
if (ct != null && "text/html".equals(ct)) {
webResource.setDocument(session.getContent());
}
return webResource;
}
@Override
public void close() {
if (session != null) {
session.close();
}
}
}
public static void main(String[] args) throws Exception {
Launcher launcher = new Launcher();
SessionFactory factory = launcher.launch();
Session dummySession = factory.create();
dummySession.navigate("about:blank");
List urls = emptyList();
final int pageLoadTimeout = 60 * 1000; // 60 seconds
try (HttpClient client = new HttpClient(factory, pageLoadTimeout)) {
Resource resource = client.fetch("https://webfolder.io/sitemap.xml");
urls = new SitemapParser().parse(resource.getDocument());
}
List resources = new ArrayList<>();
for (String url : urls) {
try (HttpClient client = new HttpClient(factory, pageLoadTimeout)) {
Resource resource = client.fetch(url);
resources.add(resource);
}
}
for (Resource resource : resources) {
System.out.println("URL: " + resource.getUrl() + ", Status Code: "
+ resource.getStatus() + ", Content Length: "
+ resource.getDocument().length());
}
dummySession.close();
}
}DownloadFile.java
import static io.webfolder.cdp.type.constant.DownloadBehavior.Allow;
import java.nio.file.Path;
import java.nio.file.Paths;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Page;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class DownloadFile {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate(
"https://www.chiark.greenend.org.uk/~sgtatham/putty/latest.html");
session.waitDocumentReady();
session.getCommand().getNetwork().enable();
Page page = session.getCommand().getPage();
Path downloadPath = Paths.get(".").toAbsolutePath();
page.setDownloadBehavior(Allow, downloadPath.toString());
// link must be visible before downloading the file
session.evaluate("document.querySelector(\"code\").scrollIntoView()");
// click the download link
session.click("code");
session.wait(2000 * 200);
}
}
}EvaluateOnNewDocument.java
import java.net.URL;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Page;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class EvaluateOnNewDocument {
public static void main(String[] args) {
URL url = Select.class.getResource("/inject-script.html");
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
Page page = session.getCommand().getPage();
// enable Page domain before using the addScriptToEvaluateOnNewDocument()
page.enable();
// addScriptToEvaluateOnNewDocument() must be called before
// Session.navigate()
page.addScriptToEvaluateOnNewDocument(
"window.dummyMessage = 'hello, world!'");
session.enableConsoleLog();
session.navigate(url.toString());
session.wait(500);
}
}
}ExecuteJavascript.java
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class ExecuteJavascript {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.waitDocumentReady();
Double result = (Double) session
.evaluate("var foo = function() { return 2 + 2; }; foo();");
System.out.println(result);
session.evaluate(
"var bar = {}; bar.myFunc = function(s1, s2) { return s1 + ' ' + s2; }");
String message = session.callFunction("bar.myFunc", String.class, "hello",
"world");
System.out.println(message);
Integer intResult = session.callFunction("foo", Integer.class);
System.out.println(intResult);
}
}
}FollowRedirects.java
import static io.webfolder.cdp.event.Events.NetworkRequestIntercepted;
import static io.webfolder.cdp.type.network.ResourceType.Document;
import static java.util.Arrays.asList;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Network;
import io.webfolder.cdp.event.network.RequestIntercepted;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
import io.webfolder.cdp.type.network.RequestPattern;
public class FollowRedirects {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
Network network = session.getCommand().getNetwork();
network.enable();
// Disable newtork caching when intercepting
// https://github.com/GoogleChrome/puppeteer/pull/1154
network.setCacheDisabled(Boolean.TRUE);
RequestPattern pattern = new RequestPattern();
pattern.setUrlPattern("*");
pattern.setResourceType(Document);
network.setRequestInterception(asList(pattern));
boolean followRedirect = true;
session.addEventListener((e, d) -> {
if (NetworkRequestIntercepted.equals(e)) {
RequestIntercepted ri = (RequestIntercepted) d;
boolean isRedirect = ri.getRedirectUrl() != null
&& !ri.getRedirectUrl().isEmpty();
if (isRedirect) {
System.out.println("");
System.out.println(
"Redirect URL : " + ri.getRedirectUrl());
System.out.println(
"Redirect Status Code : " + ri.getResponseStatusCode());
System.out.println(
"Redirect Header : " + ri.getResponseHeaders());
System.out.println("");
if (!followRedirect) {
return;
}
}
network.continueInterceptedRequest(ri.getInterceptionId());
}
});
session.navigate(
"https://httpbin.org/redirect-to?url=https://webfolder.io?cdp4j");
session.waitDocumentReady();
System.out.println(session.evaluate("document.location.href"));
}
}
}HelloWorld.java
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class HelloWorld {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate("https://webfolder.io?cdp4j");
session.waitDocumentReady();
String content = session.getContent();
System.out.println(content);
}
}
}IncognitoBrowsing.java
import static java.util.Arrays.asList;
import io.webfolder.cdp.AdaptiveProcessManager;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class IncognitoBrowsing {
// Requires Headless Chrome
// https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md
public static void main(String[] args) {
Launcher launcher = new Launcher();
launcher.setProcessManager(new AdaptiveProcessManager());
try (SessionFactory factory = launcher
.launch(asList("--headless", "--disable-gpu"))) {
String firstContext = null;
try (Session firstSession = factory.create()) {
firstContext = firstSession.getBrowserContextId();
firstSession.navigate("https://httpbin.org/cookies/set?SESSION_ID=1");
firstSession.waitDocumentReady();
String session1Content = (String) firstSession
.evaluate("window.document.body.textContent");
System.err.println(session1Content);
}
// firstSession & anotherSession share same SESSSION_ID value
try (Session anotherSession = factory.create(firstContext)) {
anotherSession.navigate("https://httpbin.org/cookies");
anotherSession.waitDocumentReady();
String anotherSessionContent = (String) anotherSession
.evaluate("window.document.body.textContent");
System.err.println(anotherSessionContent);
}
String secondContext = factory.createBrowserContext();
try (Session secondSession = factory.create(secondContext)) {
secondSession.navigate("https://httpbin.org/cookies");
secondSession.waitDocumentReady();
String session2Content = (String) secondSession
.evaluate("window.document.body.textContent");
System.err.println(session2Content);
}
// Dispose first context
factory.disposeBrowserContext(firstContext);
// Dispose second context
factory.disposeBrowserContext(secondContext);
launcher.getProcessManager().kill();
}
}
}
InvokeJavaFromJs.java
import static io.webfolder.cdp.event.Events.RuntimeBindingCalled;
import static io.webfolder.cdp.session.WaitUntil.DomContentLoad;
import static java.lang.String.format;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Page;
import io.webfolder.cdp.event.runtime.BindingCalled;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class InvokeJavaFromJs {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.enableConsoleLog();
Page page = session.getCommand().getPage();
// enable Page domain before using the addScriptToEvaluateOnNewDocument()
page.enable();
// addScriptToEvaluateOnNewDocument() must be called before
// Session.navigate()
page.addScriptToEvaluateOnNewDocument("sendMessage = send");
session.getCommand().getRuntime().addBinding("send");
session.navigateAndWait("about:blank", DomContentLoad);
session.addEventListener((event, value) -> {
if (RuntimeBindingCalled.equals(event)) {
BindingCalled binding = (BindingCalled) value;
System.out.println(format("name: [%s] payload: [%s]",
binding.getName(), binding.getPayload()));
}
});
session.evaluate("sendMessage(JSON.stringify({ 'foo' : 'bar' }));");
session.evaluate("sendMessage(JSON.stringify({ 'hello' : 'world' }));");
}
}
}InvokeJsFromJava.java
import static io.webfolder.cdp.session.WaitUntil.DomContentLoad;
import static java.nio.file.Paths.get;
import static java.util.Arrays.asList;
import java.util.List;
import io.webfolder.cdp.JsFunction;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class InvokeJsFromJava {
public static interface MyJsFunction {
@JsFunction("let attributes = []; "
+ "document.querySelectorAll(selector).forEach(e => "
+ "{ attributes.push(e.getAttribute(attributeName)); "
+ "}); return attributes;")
List listAttributes(String selector, String attributeName);
@JsFunction("")
void dummy();
@JsFunction("console.error(message);")
void consoleError(String message);
@JsFunction("return a + b")
int sum(int a, int b);
@JsFunction("return str1 + str2")
String concat(String str1, String str2);
@JsFunction("let list = []; numbers.forEach(i => "
+ "list.push(i + inc)); return list;")
List increment(List numbers, int inc);
}
public static void main(String[] args) {
Launcher launcher = new Launcher();
String uri = get("src/test/resources/js-function-test.html").toAbsolutePath()
.toUri().toString();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
// Important!
// Register the JsFunction before the navigate method
session.registerJsFunction(MyJsFunction.class);
session.enableConsoleLog();
session.navigateAndWait(uri, DomContentLoad);
MyJsFunction utility = session.getJsFunction(MyJsFunction.class);
List attributes = utility.listAttributes("img", "src");
System.out.println(attributes);
utility.dummy();
utility.consoleError("panic!");
System.out.println(utility.sum(2, 2));
System.out.println(utility.concat("foo", "bar"));
List list = utility.increment(asList(0, 1, 2, 3), 1);
System.out.println(list);
session.wait(500);
}
}
}Logging.java
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class Logging {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate("about:blank");
session.waitDocumentReady();
// logs javascript messages
session.enableConsoleLog();
session.evaluate("console.info('info message')");
session.evaluate("console.error('error message')");
session.evaluate("console.warn('warning message')");
// logs newtwork, violation, security, storage and deprecation messages
session.enableDetailLog();
session.evaluate("fetch('https://google.com')");
session.wait(2000);
}
}
}MultiProcess.java
import static io.webfolder.cdp.session.SessionFactory.DEFAULT_PORT;
import java.io.IOException;
import java.net.ServerSocket;
import java.nio.file.Path;
import java.util.Random;
import static java.lang.System.getProperty;
import static java.nio.file.Paths.get;
import static java.util.Arrays.asList;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class MultiProcess {
// port number and user-data-dir must be different for each chrome process
// As an alternative @see IncognitoBrowsing.java for incognito mode (private
// browsing).
public static void main(String[] args) {
new Thread() {
public void run() {
Launcher launcher = new Launcher(getFreePort(DEFAULT_PORT));
Path remoteProfileData = get(getProperty("java.io.tmpdir"))
.resolve("remote-profile-" + new Random().nextInt());
SessionFactory factory = launcher.launch(
asList("--user-data-dir=" + remoteProfileData.toString()));
try (SessionFactory sf = factory) {
try (Session session = sf.create()) {
session.navigate("https://webfolder.io");
session.waitDocumentReady();
System.err.println(
"Content Length: " + session.getContent().length());
}
}
}
}.start();
new Thread() {
public void run() {
Launcher launcher = new Launcher(getFreePort(DEFAULT_PORT));
Path remoteProfileData = get(getProperty("java.io.tmpdir"))
.resolve("remote-profile-" + new Random().nextInt());
SessionFactory factory = launcher.launch(
asList("--user-data-dir=" + remoteProfileData.toString()));
try (SessionFactory sf = factory) {
try (Session session = sf.create()) {
session.navigate("https://webfolder.io");
session.waitDocumentReady();
System.err.println(
"Content Length: " + session.getContent().length());
}
}
}
}.start();
}
protected static int getFreePort(int portNumber) {
try (ServerSocket socket = new ServerSocket(portNumber)) {
int freePort = socket.getLocalPort();
return freePort;
} catch (IOException e) {
return getFreePort(portNumber + 1);
}
}
}MultiSelect.java
import static java.util.Arrays.asList;
import java.net.URL;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class MultiSelect {
public static void main(String[] args) {
URL url = MultiSelect.class.getResource("/multi-select.html");
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate(url.toString());
session.waitDocumentReady();
System.out.println("Selected options:");
session.getOptions("select").stream().forEach(o -> {
if (o.isSelected()) {
System.out.println(o);
}
});
session.wait(1000);
session.clearOptions("select");
System.out.println("Selected Option count: " + session.getOptions("select")
.stream().filter(o -> o.isSelected()).count());
session.wait(1000);
session.setSelectedOptions("select", asList(0, 1, 2));
System.out.println("Selected options:");
session.getOptions("select").stream().forEach(o -> {
if (o.isSelected()) {
System.out.println(o);
}
});
System.out.println("Selected Option count: " + session.getOptions("select")
.stream().filter(o -> o.isSelected()).count());
session.wait(1000);
}
}
}NetworkResponse.java
import static io.webfolder.cdp.event.Events.NetworkLoadingFinished;
import static io.webfolder.cdp.event.Events.NetworkResponseReceived;
import java.util.HashSet;
import java.util.Set;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.event.network.LoadingFinished;
import io.webfolder.cdp.event.network.ResponseReceived;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
import io.webfolder.cdp.type.network.GetResponseBodyResult;
import io.webfolder.cdp.type.network.Response;
import io.webfolder.cdp.type.network.ResourceType;
public class NetworkResponse {
public static void main(String[] args) {
Launcher launcher = new Launcher();
Set finished = new HashSet<>();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.getCommand().getNetwork().enable();
session.addEventListener((e, d) -> {
if (NetworkLoadingFinished.equals(e)) {
LoadingFinished lf = (LoadingFinished) d;
finished.add(lf.getRequestId());
}
if (NetworkResponseReceived.equals(e)) {
ResponseReceived rr = (ResponseReceived) d;
Response response = rr.getResponse();
System.out.println("----------------------------------------");
System.out.println("URL : " + response.getUrl());
System.out.println(
"Status : HTTP " + response.getStatus().intValue() + " "
+ response.getStatusText());
System.out.println("Mime Type : " + response.getMimeType());
if (finished.contains(rr.getRequestId())
&& ResourceType.Document.equals(rr.getType())) {
GetResponseBodyResult rb = session.getCommand().getNetwork()
.getResponseBody(rr.getRequestId());
if (rb != null) {
String body = rb.getBody();
System.out.println("Content : " + body.substring(0,
body.length() > 1024 ? 1024 : body.length()));
}
}
}
});
session.navigate("http://cnn.com");
session.waitDocumentReady();
}
}
}PdfPrinter.java
import static java.lang.Thread.sleep;
import static java.util.Arrays.asList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.function.Consumer;
import io.webfolder.cdp.AdaptiveProcessManager;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class PdfPrinter implements AutoCloseable {
private SessionFactory factory;
private Timer timer = new Timer("cdp4j-task-manager", true);
private Launcher launcher;
private ExecutorService pool = Executors.newCachedThreadPool();
private List arguments;
public PdfPrinter(List arguments) {
this.arguments = Collections.unmodifiableList(arguments);
}
public void execute(Consumer func) {
pool.execute(() -> {
String context = factory.createBrowserContext();
Session session = factory.create(context);
try {
func.accept(session);
} catch (Throwable t) {
throw new RuntimeException(t);
} finally {
if (session != null) {
try {
session.close();
} catch (Throwable t) {
// ignore
}
}
if (factory.ping()) {
factory.disposeBrowserContext(context);
}
}
});
}
public void init() {
launcher = new Launcher();
launcher.setProcessManager(new AdaptiveProcessManager());
factory = launcher.launch(arguments);
int delay = 1000;
timer.scheduleAtFixedRate(new TimerTask() {
@Override
public void run() {
if (factory != null) {
int retryCount = 0;
boolean connected = factory.ping();
while (!(connected = factory.ping()) && retryCount < 50) {
try {
sleep(100);
} catch (InterruptedException e) {
// ignore
}
retryCount += 1;
}
if (!connected) {
restart();
}
}
}
}, delay, delay);
}
protected void restart() {
SessionFactory oldFactory = factory;
Thread shutdownThread = new Thread(() -> {
try {
oldFactory.close();
} catch (Throwable t) {
// ignore
}
});
shutdownThread.setDaemon(true);
shutdownThread.setName("cdp4j-task-manager");
shutdownThread.start();
launcher.kill();
launcher = new Launcher();
factory = launcher.launch();
}
@Override
public void close() {
pool.shutdown();
timer.cancel();
factory.close();
launcher.kill();
}
public static void main(String[] args) {
PdfPrinter manager = new PdfPrinter(
Arrays.asList("--disable-gpu", "--headless"));
manager.init();
List urls = asList("http://www.google.com", "http://www.bing.com");
CountDownLatch latch = new CountDownLatch(urls.size());
for (String next : urls) {
manager.execute(new Consumer() {
@Override
public void accept(Session s) {
s.navigate(next);
try {
s.waitDocumentReady();
byte[] content = s.getCommand().getPage().printToPDF();
if (content != null) {
System.out.println(
"PDF size: " + content.length + " (" + next + ")");
}
} catch (Throwable e) {
//
} finally {
latch.countDown();
}
}
});
}
try {
latch.await();
} catch (InterruptedException e) {
throw new RuntimeException(e);
} finally {
manager.close();
}
}
}PrintToPDF.java
import static java.awt.Desktop.getDesktop;
import static java.awt.Desktop.isDesktopSupported;
import static java.nio.file.Files.createTempFile;
import static java.nio.file.Files.write;
import static java.util.Arrays.asList;
import java.io.IOException;
import java.nio.file.Path;
import io.webfolder.cdp.AdaptiveProcessManager;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class PrintToPDF {
// Requires Headless Chrome
// https://chromium.googlesource.com/chromium/src/+/lkgr/headless/README.md
public static void main(String[] args) throws IOException {
Launcher launcher = new Launcher();
launcher.setProcessManager(new AdaptiveProcessManager());
Path file = createTempFile("cdp4j", ".pdf");
try (SessionFactory factory = launcher
.launch(asList("--disable-gpu", "--headless"))) {
String context = factory.createBrowserContext();
try (Session session = factory.create(context)) {
session.navigate("https://webfolder.io/cdp4j.html");
session.waitDocumentReady();
byte[] content = session.getCommand().getPage().printToPDF();
write(file, content);
}
factory.disposeBrowserContext(context);
}
if (isDesktopSupported()) {
getDesktop().open(file.toFile());
}
launcher.getProcessManager().kill();
}
}Readability.java
import static io.webfolder.cdp.session.WaitUntil.DomReady;
import static java.nio.file.Files.readAllBytes;
import static java.nio.file.Paths.get;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.command.Page;
import io.webfolder.cdp.logger.CdpLoggerType;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
// Remove noise and ads before extracting the content
// @see https://github.com/mozilla/readability
public class Readability {
public static void main(String[] args) throws Exception {
Launcher launcher = new Launcher(CdpLoggerType.Null);
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
Page page = session.getCommand().getPage();
// enable Page domain before using the addScriptToEvaluateOnNewDocument()
page.enable();
// addScriptToEvaluateOnNewDocument() must be called before
// Session.navigate()
page.addScriptToEvaluateOnNewDocument(
new String(readAllBytes(get("src/test/resources/readability.js"))));
session.navigateAndWait(
"https://www.theregister.co.uk/2018/07/27/lower_saxony_to_dump_linux",
DomReady);
String content = (String) session
.evaluate("new Readability(window.document).parse().content");
System.out.println(content);
} finally {
launcher.kill();
}
}
}RemoteLaunching.java
import io.webfolder.cdp.RemoteLauncher;
import io.webfolder.cdp.RemoteLauncherBuilder;
import io.webfolder.cdp.session.SessionFactory;
import java.io.*;
import java.nio.charset.StandardCharsets;
import static java.util.Arrays.asList;
public class RemoteLaunching {
@SuppressWarnings("unused")
public static void main(String[] args) throws IOException {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
try (BufferedReader br = new BufferedReader(new InputStreamReader(
new FileInputStream(new File("path/to/your/private/key")),
StandardCharsets.UTF_8))) {
String s;
while ((s = br.readLine()) != null) {
pw.println(s);
}
}
RemoteLauncher l = new RemoteLauncherBuilder().withHost("1.2.3.4")
.withChromePort(12345).withUser("chromeuser")
.withPrivateKey(sw.toString()).create();
SessionFactory sf = l.launch(asList("--headless", "--disable-gpu"));
l.kill();
}
}Screenshot.java
import static java.awt.Desktop.getDesktop;
import static java.awt.Desktop.isDesktopSupported;
import static java.nio.file.Files.createTempFile;
import static java.nio.file.Files.write;
import java.io.IOException;
import java.nio.file.Path;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class Screenshot {
public static void main(String[] args) throws IOException, InterruptedException {
Launcher launcher = new Launcher();
Path file = createTempFile("screenshot", ".png");
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate("https://news.ycombinator.com");
session.waitDocumentReady();
// activate the tab/session before capturing the screenshot
session.activate();
byte[] data = session.captureScreenshot();
write(file, data);
}
if (isDesktopSupported()) {
getDesktop().open(file.toFile());
}
}
}Select.java
import java.net.URL;
import java.util.Optional;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Option;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class Select {
public static void main(String[] args) {
URL url = Select.class.getResource("/select.html");
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate(url.toString());
session.waitDocumentReady();
int selectedIndex = session.getSelectedIndex("select");
System.out.println("Selected index : " + selectedIndex);
System.out.println("options : " + session.getOptions("select"));
session.wait(2000);
session.setSelectedIndex("select", 2);
session.wait(2000);
selectedIndex = session.getSelectedIndex("select");
System.out.println("Selected index : " + selectedIndex);
Optional selected = session.getOptions("select").stream()
.filter(o -> o.isSelected()).findFirst();
System.out.println("Selected : " + selected.get().getText());
}
}
}SendKeys.java
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class SendKeys {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.getCommand().getNetwork().enable();
session.navigate("https://google.com");
session.waitDocumentReady();
session.sendKeys("webfolder.io");
session.sendEnter();
session.wait(2000);
}
}
}SharedSession.java
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class SharedSession {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
try (Session firstSession = factory.create()) {
firstSession.navigate("https://httpbin.org/cookies/set?SESSION_ID=1");
firstSession.wait(1000);
String session1 = (String) firstSession
.evaluate("window.document.body.textContent");
System.out.println(session1);
}
try (Session secondSession = factory.create()) {
secondSession.navigate("https://httpbin.org/cookies");
String session2 = (String) secondSession
.evaluate("window.document.body.textContent");
secondSession.wait(1000);
System.out.println(session2);
}
}
}
}UserAgent.java
import java.util.Map;
import com.google.gson.Gson;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class UserAgent {
@SuppressWarnings("rawtypes")
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.setUserAgent("My Browser");
session.navigate("https://httpbin.org/headers");
session.waitDocumentReady();
String response = (String) session.evaluate("document.body.textContent");
Gson gson = new Gson();
Map json = gson.fromJson(response, Map.class);
Map headers = (Map) json.get("headers");
System.out.println(headers.get("User-Agent"));
}
}
}WaitUntil.java
import java.net.URL;
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class WaitUntil {
public static void main(String[] args) {
URL url = WaitUntil.class.getResource("/wait-until.html");
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate(url.toString());
session.waitDocumentReady();
boolean succeed = session.waitUntil(s -> {
return s.matches("#time");
}, 10 * 1000);
if (succeed) {
String time = session.getText("#time");
System.out.println(time);
}
}
}
}XPathSelector.java
import io.webfolder.cdp.Launcher;
import io.webfolder.cdp.session.Session;
import io.webfolder.cdp.session.SessionFactory;
public class XPathSelector {
public static void main(String[] args) {
Launcher launcher = new Launcher();
try (SessionFactory factory = launcher.launch();
Session session = factory.create()) {
session.navigate("https://webfolder.io").waitDocumentReady();
String title = session.getText("/html/head/title");
System.out.println(title);
}
}
}