Java Crawler(1)HTMLUnit

Java Crawler(1)HTMLUnit

pom.xml Add Few Dependencies

+              <dependency>
+                  <groupId>net.sourceforge.htmlunit</groupId>
+                  <artifactId>htmlunit</artifactId>
+              </dependency>
+              <dependency>
+                  <groupId>net.sourceforge.htmlunit</groupId>
+                  <artifactId>htmlunit-core-js</artifactId>
+                  <version>2.27</version>
+              </dependency>
+              <dependency>
+                  <groupId>net.sourceforge.htmlunit</groupId>
+                  <artifactId>neko-htmlunit</artifactId>
+                  <version>2.27</version>
+              </dependency>
+              <dependency>
+                  <groupId>org.w3c.css</groupId>
+                  <artifactId>sac</artifactId>
+                  <version>1.3</version>
+              </dependency><dependency>
+                  <groupId>commons-codec</groupId>
+                  <artifactId>commons-codec</artifactId>
+              </dependency>
+              <dependency>
+                  <groupId>xalan</groupId>
+                  <artifactId>xalan</artifactId>
+                  <version>2.7.2</version>
+              </dependency>
+              <dependency>
+                  <groupId>net.sourceforge.cssparser</groupId>
+                  <artifactId>cssparser</artifactId>
+                  <version>0.9.23</version>
+              </dependency>
+              <dependency>
+                  <groupId>org.eclipse.jetty.websocket</groupId>
+                  <artifactId>websocket-client</artifactId>
+              </dependency>
+              <dependency>
+                      <groupId>xerces</groupId>
+                      <artifactId>xercesImpl</artifactId>
+                      <version>2.11.0</version>
+              </dependency>

Most easiest Test Class CrawlerTest.java

package com.sillycat.jobsmonitorapi.service;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.net.MalformedURLException;
import java.util.Iterator;
import java.util.List;
import org.junit.Test;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.DomNode;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlDivision;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;

public class CrawlerTest {

@Test
public void testCrawlWalmart() throws FailingHttpStatusCodeException, MalformedURLException, IOException {
    try (final WebClient webClient = new WebClient(BrowserVersion.INTERNET_EXPLORER)) {
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        webClient.getOptions().setJavaScriptEnabled(true);
        webClient.getOptions().setRedirectEnabled(true);
        webClient.getOptions().setTimeout(30000);
        webClient.setJavaScriptTimeout(30000);
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        HtmlPage page = webClient.getPage("https://www.walmart.com/search/?grid=false&page=2&query=computer#searchProductResult");
        webClient.waitForBackgroundJavaScript(10000);
        //String htmlContent = page.asXml();
        //File htmlFile = new File("/Users/carl/Downloads/products.html");
        //PrintWriter pw = new PrintWriter(htmlFile);
        //pw.print(htmlContent);
        //pw.close();
        HtmlDivision div = page.getFirstByXPath("//div[@class='search-result-listview-items']");
        Iterator<DomElement> itDivs = div.getChildElements().iterator();
        int count = 0;
        for(;itDivs.hasNext();){
            count++;
            DomElement helloDiv = itDivs.next();
            //helloDiv.click();
            System.out.println("Print==== " + count + " " + helloDiv.asText());
         }
        //List<?> links = page.getByXPath("//a[@class='product-title-link']");
        //for(int i = 0;i<links.size();i++){
            //System.out.println("Link===========" + (i+1) + " " + links.get(i));
        //}
        //List<HtmlAnchor> anchors = page.getAnchors();
        //int count = 0;
        //for(int i = 0;i<anchors.size();i++){
            //HtmlAnchor anchor = anchors.get(i);
            //String url = anchor.getHrefAttribute();
            //if(url.startsWith("/ip")){
                //count++;
                //System.out.println("URL ============== " + count + " " + url);
            //}
        //}
       
//          Detail Page
//HtmlPage page = webClient.getPage("https://www.walmart.com/ip/HP-15-ay041wm-15-6-Silver-Fusion-Laptop-Touch-Screen-Windows-10-Intel-Core-i3-6100U-Processor-8GB-Memory-1TB-Hard-Drive/51397784");
//HtmlSpan priceSpan = page.getFirstByXPath("//span[@class='Price-characteristic']");
//System.out.println("Price========" + priceSpan.getTextContent());
}
}
}



References: