网络爬虫测试


/**
 * 网络爬虫测试
 * 
 * @author Tunny
 * @2018年8月11日
 */
public class WebCrawler {
	public static void main(String[] args) {
		Scanner can = new Scanner(System.in);
		String url = "http://www.zysj.com.cn/lilunshuji/jichulilun/index.html";
		url = can.nextLine();
		
		CloseableHttpClient httpClient = HttpClients.createDefault();
		HttpGet httpGet = new HttpGet(url);
		httpGet.addHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
		// 执行请求
		HttpResponse response;
		String line;
		try {
 
			response = httpClient.execute(httpGet);
			HttpEntity httpEntity = response.getEntity();
			BufferedReader bufferedReader = null;
			bufferedReader = new BufferedReader(new InputStreamReader(
					httpEntity.getContent(), "utf-8"), 8 * 1024);
			StringBuilder entityStringBuilder = new StringBuilder();
			while ((line = bufferedReader.readLine()) != null) {
				entityStringBuilder.append(line + "\n");
			}
		//	System.out.println(entityStringBuilder.toString());
		// appendMethodB("f:/中医基础理论.html",entityStringBuilder.toString());
			savaFile("C:/Users/tunny/Desktop/index.html",entityStringBuilder.toString(),"UTF-8");
 
		} catch (ClientProtocolException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
 
	/**
	 * 保存文件
	 * @param fileName 文件名称:绝对路径
	 * @param content 要保存的内容
	 * @param format 以某种格式保存文件
	 */
	public static void savaFile(String fileName, String content,String format) {
		BufferedWriter rd=null;
		OutputStream out=null;
		File file = new File(fileName);
		try {
			out = new FileOutputStream(file);
			rd = new BufferedWriter(new OutputStreamWriter(out,format));
			rd.write(content);
		} catch (IOException e) {
			e.printStackTrace();
		}finally{
			if(null!=rd){
				try {
					rd.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			if(null!=out){
				try {
					out.close();
				} catch (IOException e) {
					e.printStackTrace();
				}
			}
			
		}
	}
 
	/**
	 * 
	 * @param fileName
	 * @param content
	 */
	public static void appendMethodB(String fileName, String content) {
		FileWriter writer=null;
		try {
			writer = new FileWriter(fileName, false);
			// 打开一个写文件器,构造函数中的第二个参数true表示以追加形式写文件
			writer.write(content);
		} catch (IOException e) {
			e.printStackTrace();
		}finally{
			try {
				if(null!=writer){
					writer.close();
				}
			} catch (IOException e) {
				e.printStackTrace();
			}
			
		}
	}
}

猜你喜欢

转载自blog.csdn.net/tun52099/article/details/81706612