Java SE web crawler①

A description of needs

　　Give a url, find out all the links in the url, and supplement it with an absolute path

Two program implementation

/**
 *
 * @author Zen Johnny
 *@date April 29, 2018 at 7:22:44 PM
 *
 */
package spider;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class WebPageSpider {
	private static Pattern pattern = null;
	private static Matcher matches = null;
	private static BufferedReader br = null;
	private static StringBuffer text = null;
	private static URL _url = null;
	private static List<String> links = null;
	
	public static String captureWebPageContent(String url) throws UnsupportedEncodingException, IOException {
		text = new StringBuffer();
		_url = new URL(url);
		br = new BufferedReader(new InputStreamReader(_url.openStream(), "utf-8"));
		String line = null;
		while( (line = br.readLine()) != null) {
			text.append(line);
		}
		return text.toString();
	}
	
	/*
	 	Retrieves and returns all links in a webpage's text
	 	@param refUrl: the absolute path of the reference
	 	Eg：refAbsoluteUrl -> xxxx.com/X/Y/M/L/test.html?key=35435
	 */
	public static List<String> findLinks(String text, String refAbsoluteUrl) throws MalformedURLException {
		links = new LinkedList<String>();
		String regex = "((href)|(src)){1}=([\"\'])(.*?)\\4";//\\4: If it is preceded by double (single) quotation marks, The end must also be double (single) quotation marks
		pattern = Pattern.compile(regex);
		matcher = pattern.matcher (text);
		String link = null;
		while(matcher.find()) {
			link = matcher.group().replaceAll("((href=)|(src=)[\"\'])|([\'\"])", "");
			if(link.startsWith(".")) {//If it starts with a relative path, add the base URL by default
				links.add(revertToAbsolutePath(refAbsoluteUrl, link));
			} else if(link.startsWith("/")){//Start with the root path
				URL tmp_url = new URL(refAbsoluteUrl);
				links.add(tmp_url.getHost() + link);
			} else if(link.endsWith("#")){//Use # as the path, that is, the current path (reference path)
				links.add(refAbsoluteUrl);
			} else {
				links.add(link);
			}
			
		}
		return links;
	}
	
	/*
	 	Convert URL path to path list
	 	Eg:xxxx.com/X/Y/M/L/test.html?key=35435 => xxxx.com X Y M L test.html?key=35435
	 */
	public static List<String> dirs(String url) {
		java.util.List<String> dirs = new LinkedList<String>();
		String [] dirsArray = url.split("/+");
		for(String item : dirsArray) {
//			System.out.println(item);//test
			dirs.add(item.trim());
		}
		return dirs;
	}
	
	/*
	 	Revert relative path to absolute path
	 	@param:curAbsoluteUrl refers to the absolute path
	 	@param:relativeSubUrl relative sub path
	 	
	 	xxxx.com/X/Y/M/L/test.html?key=35435	/hr/ry/ry			=> 	xxxx.com/X/Y/M/L/hr/ry/ry
	 	xxxx.com/X/Y/M/L/						./../../G/J/d.x		=>	xxxx.com/X/Y/G/J/d.x		
	 */
	public static String revertToAbsolutePath(String refAbsoluteUrl, String relativeSubUrl) {
		List<String> refPaths = dirs(refAbsoluteUrl);//Reference path list
		List<String> relativePaths = dirs(relativeSubUrl);//relative path list
		List<String> dirs = new LinkedList<String>();
		StringBuffer path = new StringBuffer();

		if(refPaths.get(refPaths.size() - 1).matches("(.)*[\\.\\?](.)*")) {//If the last item of the reference path is a file or At the end of the query form, delete the last item
//			System.out.println(refPaths.get(dirs.size() - 1));//test
			refPaths.remove(refPaths.size() - 1);
		}
		
		for(String item : relativePaths) {
//			System.out.println("item:"+item);//test
			if(item.equals("..")) {//The upper level path, then: delete the last item of refPaths
				refPaths.remove(refPaths.size() - 1);
			} else if(!item.equals(".")) {//Not the current path. Or an empty path, that is: a real path
				if(!item.equals("")) {
					refPaths.add(item);
				} else {
//					System.out.println("【空】");//test
				}
			}
		}
		for(int item = 0,size = refPaths.size();item<size;item++) {//test
			path.append(refPaths.get(item) + (item+1 == size?"":"/"));
		}
		return path.toString();
	}
}