Java抓取网页爬虫

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/weijianpeng2013_2015/article/details/71332740

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

import org.apache.commons.httpclient.Header;
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpException;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.NameValuePair;
import org.apache.commons.httpclient.methods.PostMethod;


public class RetrivePage {
    private static HttpClient httpClient=new HttpClient();
    //设置代理服务器
    static{
        //设置代理服务器的IP地址和端口
        //httpClient.getHostConfiguration().setProxy("192.168.28.137", 8080);
    }
    public static boolean downloadPage(String path) throws HttpException, IOException{
        InputStream input=null;
        OutputStream output=null;
        //得到post方法
        PostMethod postMethod=new PostMethod(path);
        //设置post方法的参数
        /*NameValuePair[] postData=new NameValuePair[2];
        postData[0]=new NameValuePair("name","lietu");
        postData[1]=new NameValuePair("password","*****");
        postMethod.addParameters(postData);*/
        //执行，返回状态码
        int statusCode=httpClient.executeMethod(postMethod);
        //针对状态码进行处理（简单期间，只处理返回值为200的状态码）
        if(statusCode==HttpStatus.SC_OK){
            input=postMethod.getResponseBodyAsStream();
            //得到文件名
            //String filename="G://pachon//"+path.substring(path.lastIndexOf('/')+1);
            String filename="G:\\pachon\\"+"hello.html";
            System.out.println("filtname="+filename);
            //获得文件输出流
            output=new FileOutputStream(filename);

            //输出到文件
            int tempBytes=-1;
            while((tempBytes=input.read())>0){
                output.write(tempBytes);
            }
            //关闭输入输出流
            if(input!=null){
                input.close();
            }
            if(output!=null){
                output.close();
            }
            return true;
        }
        //若需要转向，则进行转向操作
        if((statusCode==HttpStatus.SC_MOVED_TEMPORARILY)||
                (statusCode==HttpStatus.SC_MOVED_PERMANENTLY)||
                (statusCode==HttpStatus.SC_TEMPORARY_REDIRECT)||
                statusCode==HttpStatus.SC_SEE_OTHER){
            //获取新的URL地址
            Header header=postMethod.getResponseHeader("location");
            if(header==null){
                String newUrl=header.getValue();
                if(newUrl==null||newUrl.equals("")){
                    newUrl="/";
                    //使用post转向
                    PostMethod redirect=new PostMethod(newUrl);
                    //发送请求，做进一步处理。。。。
                }
            }
        }
        return false;

    }
    //测试代码
    public static void main(String[] args) {
        //抓取猎兔首页，输出
        try {
            RetrivePage.downloadPage("http://www.lietuw.com/");
            System.out.println("执行成功!");
        } catch (HttpException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}