HDFS文件系统操作JAVA-API

版权声明：本文为博主原创文章，未经博主允许不得转载。 https://blog.csdn.net/usher_ou/article/details/79178285

public class HDFSIO {

    /**
     * 创建文件夹
 * @param path
 * @throws IOException
 */

public static void mkdir(String path)throws IOException{

    //读取配置文件 
    Configuration configuration = new Configuration();
    //获取文件系统  
    FileSystem fSystem = FileSystem.get(URI.create("hdfs://192.168.32.128:9000/"), configuration);
    Path srcPath = new Path(path);
    boolean flag = fSystem.mkdirs(srcPath);
    if(flag){
        System.out.println("create dir ok!");
    }else {
        System.out.println("create dir failure");
    }
    fSystem.close();
}
/**
 * 删除文件或者文件目录  
 * @throws IOException 
 * **/  
public static void rmdir(String filePath) throws IOException {  
    //读取配置文件  
    Configuration conf = new Configuration();  
    //获取文件系统  
    FileSystem fs = FileSystem.get(URI.create("hdfs://192.168.32.128:9000"),conf);  
    Path path = new Path(filePath);  

    //调用deleteOnExit(）  
    boolean flag = fs.deleteOnExit(path);  
    //  fs.delete(path);  
    if(flag) {  
         System.out.println("delete ok!");  
    }else {  
         System.out.println("delete failure");  
    }  

    //关闭文件系统  
    fs.close();  
}  
/**
 *  创建文件
 * @param dsString
 * @param contents
 * @throws IOException
 */
@SuppressWarnings("unused")
private static void createFile(String dsString ,byte[] contents) throws IOException{

    // TODO Auto-generated method stub
    Configuration configuration = new Configuration();
    FileSystem fSystem = FileSystem.get(URI.create("hdfs://192.168.32.128:9000/"),configuration);
    Path dsPath = new Path(dsString);
    //打开一个输出流
    FSDataOutputStream outputStream = fSystem.create(dsPath);
    outputStream.write(contents);

    //close
    outputStream.close();
    fSystem.close();
    System.out.println("create success!");
}
/**
 * 列出目录下的文件
 * @param path
 * @throws IOException
 */
 public static void listFile(String path) throws IOException{  

        //读取配置文件  
        Configuration conf = new Configuration();  
        //获取文件系统  
        FileSystem fs = FileSystem.get(URI.create("hdfs://192.168.32.128:9000"),conf);  
        //获取文件或目录状态  
        FileStatus[] fileStatus = fs.listStatus(new Path(path));  
        //打印文件的路径  
        for (FileStatus file : fileStatus) {  
            System.out.println(file.getPath());  
        }  

        //关闭文件系统  
        fs.close();  
     }  
 /**
  * 上传文件
  * @param src
  * @param dst
  * @throws IOException
  */
 private static void uploadFile(String src,String dst) throws IOException{

    // TODO Auto-generated method stub
     Configuration configuration = new Configuration();
     FileSystem fsFileSystem = FileSystem.get(URI.create("hdfs:192.168.32.128:9000"),configuration);
     Path srcPath = new Path(src);
     Path dstPath = new Path(dst);
    //调用文件系统的文件复制函数,前面参数是指是否删除原文件，true为删除，默认为false  
     fsFileSystem.copyFromLocalFile(false, srcPath,dstPath);
     //打印文件路径  
     System.out.println("Upload to "+configuration.get("fs.default.name"));  
     System.out.println("------------list files------------"+"\n");  
     FileStatus [] fileStatus = fsFileSystem.listStatus(dstPath);
     for(FileStatus file: fileStatus){
         System.out.println(file.getPath());
     }
     fsFileSystem.close();
}
 /**
  * 重命名
  * @param oldName
  * @param newName
  * @throws IOException
  */
 private static void renameFile(String oldName,String newName)throws IOException {
    // TODO Auto-generated method stub
     Configuration configuration = new Configuration();
     FileSystem fs = FileSystem.get(URI.create("hdfs://192.168.32.128:9000"),configuration);
     Path oldPath = new Path(oldName);
     Path newPath = new Path(newName);

     boolean flag = fs.rename(oldPath, newPath);
     if(flag){
         System.out.println("rename ok!");
     }else {
        System.out.println("rename failure!");
    }
     fs.close();
}
 /**
  * 读取文件
  * @param uriString
  * @throws IOException
  */
 private static void readfile(String uriString) throws IOException {
    // TODO Auto-generated method stub
     Configuration configuration = new Configuration();
     FileSystem fSystem = FileSystem.get(URI.create("hdfs://192.168.32.128:9000"), configuration);

     InputStream inputStream = null;
     try {
        inputStream = fSystem.open(new Path(uriString));
        //复制到标准输出流  
        IOUtils.copyBytes(inputStream, System.out,4096,false );
    } catch (Exception e) {
        // TODO: handle exception
        e.printStackTrace();
    }finally{
        IOUtils.closeStream(inputStream);
    }
}
 /**
  * 添加到文件的末尾(src为本地地址，dst为hdfs文件地址)
  * 追加到文件末尾
  * @param srcString
  * @param dsString
  * @throws IOException
  */
 private static void appendFile(String srcString,String dsString)throws IOException {
    // TODO Auto-generated method stub
     Configuration configuration = new Configuration();
     FileSystem fsFileSystem = FileSystem.get(URI.create("hdfs://192.168.32.128:9000"),configuration);
     Path dstPath = new Path(dsString);
     //创建需要写入的文件流 
     InputStream inputStream = new BufferedInputStream(new FileInputStream(srcString));
    //文件输出流写入 
     FSDataOutputStream outputStream = fsFileSystem.append(dstPath);
     IOUtils.copyBytes(inputStream, outputStream,4096,true);
}

FileSystem的get（）方法有两个。
FileSystem fs = FileSystem.get(URI.create(“hdfs://localhost:9000”),conf); //默认在hdfs上读取文件
FileSystem fs = FileSystem.get(conf); //默认从本地上读取文件