Java 对HDFS接口调用

版权声明：本文为博主原创文章，转载请标明出处http://blog.csdn.net/leoe_ https://blog.csdn.net/LEoe_/article/details/79748209

使用HDFS提供的API来对分布式文件系统中的文件进行基本的操作，在linux下是不需要连接的，在windows下连接hdfs有三种方式：

Configuration  conf = new Configuration();
//第一种方式
conf.set("fs.defaultFS", "hdfs://10.49.85.152:9000");
//第二种设置configuration方式
conf.addResource(new Path("D:\\hadoop\\1.xml"));    
FileSystem hdfs = FileSystem.get(conf);
//第三种方式
FileSystem hdfs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);

---

下面是对基本的API的调用，以供参考：

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSDemo2 {
    static Configuration  conf;
    static FileSystem fs;

    static{
        conf = new Configuration();
        try {
            fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (URISyntaxException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

//  public HDFSDemo2(){
//      conf = new Configuration();
//      try {
//          fs = FileSystem.get(new URI("hdfs://10.49.85.152:9000"), conf);
//      } catch (IOException e) {
//          // TODO Auto-generated catch block
//          e.printStackTrace();
//      } catch (URISyntaxException e) {
//          // TODO Auto-generated catch block
//          e.printStackTrace();
//      }
//  }
    //创建文件夹
    public static void mkdir(String name){
        boolean flag = false; 
        try {
             if (!fs.exists(new Path(name+"1"))) {  
                 flag = fs.mkdirs(new Path(name));
                }  
            System.out.println(flag);
            if(flag){
                System.out.println("文件夹创建成功");
            }else{
                System.out.println("文件夹创建失败");
            }
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }
    }
    //创建文件
    public static void createFile(String file, String text){
        FSDataOutputStream outputStream = null;
        byte[] arg0 = text.getBytes();
        try {
            outputStream = fs.create(new Path(file));
            outputStream.write(arg0, 0, arg0.length);
            outputStream.close();
            System.out.println("文件创建成功");
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }
    }
    //对文件重命名
    public static void rename(String oldName, String newName){
        boolean flag;
        try {
            flag = fs.rename(new Path(oldName), new Path(newName));
            if(flag){
                System.out.println("文件重命名成功");
            }
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }
    }


    //判断文件是否存在
    public static void  existFile(String file){
        boolean flag;
        try {
            flag = fs.exists(new Path(file));
            if(flag){
                System.out.println("存在");
            }else
            {
                System.out.println("不存在");
            }
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }
    }

    //删除文件
    public static void deleteFile(String file){
        boolean flag;
        try {
            flag = fs.deleteOnExit(new Path(file));
            if(flag){
                System.out.println("成功删除");
            }
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }
    }


    public static void close(){
        try {
            fs.close();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }
    }

    //上传文件
    public static void uploadFile(String src, String dst){
        try {
            fs.copyFromLocalFile(false, new Path(src), new Path(dst));
            System.out.println("文件上传成功");
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }
    }

    //下载文件
    public static void downFile(String src, String dst){
        try {
            fs.copyToLocalFile(new Path(src), new Path(dst));
            System.out.println("文件下载成功");
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }
    }

    //读取文件
    public static String readFile(String file){
        String str = "";
        FSDataInputStream  inputStream = null;
        byte[] arg0 = new byte[1024];
        try {
            inputStream = fs.open(new Path(file));
            inputStream.read(arg0);
            str = new String(arg0);
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }

        return str;
    }

    //返回指定文件夹的文件列表
    public static void listName(String dir){
        try {
            FileStatus files[] = fs.listStatus(new Path(dir));
            for(FileStatus file : files){
                System.out.println(file.getPath().getName() + "\t" + file.getLen());
            }
        } catch (FileNotFoundException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IllegalArgumentException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        } catch (IOException e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }finally{
            close();
        }


    }

    public static void main(String[] args) throws IOException {
        // TODO Auto-generated method stub
//      HDFSDemo2 hdfs = new HDFSDemo2();
        HDFSDemo2.mkdir("/test/t2");
//      HDFSDemo2.createFile("/test/3.txt", "fdsf23432432违法所得");
//      HDFSDemo2.createFile(args[0], args[1]);
//      HDFSDemo2.rename("/test/test2", "/test/test1");
//      HDFSDemo2.existFile("/test/3.txt");
//      HDFSDemo2.deleteFile("/test/5.txt");
//      HDFSDemo2.downFile(args[0], args[1]);
//      String str = HDFSDemo2.readFile("/test/3.txt");
//      System.out.println(str);
//      HDFSDemo2.listName("/test");

    }

}