API para el uso de las operaciones de HDFS (Java)

package com.zhengkw.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;


import static java.lang.System.out;

/**
 * @ClassName:HDFSclient
 * @author: zhengkw
 * @description: HDFS客户端
 * @date: 20/02/20上午 11:28
 * @version:1.0
 * @since: jdk 1.8
 */
public class HDFSclient {
    private FileSystem fileSystem;

    /**
     * @descrption:创建目录 优先级
     * @return: void
     * @date: 20/02/20 上午 11:43
     * @author: zhengkw
     */
    @Test
    public void testMkdirs() throws IOException, URISyntaxException, InterruptedException {
        //获取文件系统
        Configuration configuration = new Configuration();
        //配置在集群上运行(core-site)

       configuration.set("dfs.replication", "2");
        //获取文件系统对象
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), configuration, "zhengkw");
        //创建目录
        boolean mkdirFlag = fileSystem.mkdirs(new Path("/user/zhengkw/sanguo"));
        // 关闭资源
        fileSystem.close();

    }

    /**
     * @descrption:从本地上传
     * @return: void
     * @date: 20/02/20 下午 12:38
     * @author: zhengkw
     */
    @Test
    public void testCopyFromLocalFile() throws URISyntaxException, IOException, InterruptedException {
        //创建configuration对象
        Configuration configuration = new Configuration();
        //获取文件系统对象(url,config,username)
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), configuration, "zhengkw");

        //上传
        fileSystem.copyFromLocalFile(false, new Path("f:/123.txt"), new Path("/user/zhengkw/sanguo/345.txt"));
        //关闭资源
        fileSystem.close();
        out.println("finish");

    }

    /**
     * @descrption:初始化获取fs对象
     * @return: void
     * @date: 20/02/20 下午 2:28
     * @author: zhengkw
     */
    @Before
    public void init() throws URISyntaxException, IOException, InterruptedException {
        fileSystem = FileSystem.get(new URI("hdfs://hadoop103:9000"), new Configuration(), "zhengkw");
    }


    /***
     * @descrption:下载
     * @return: void
     * @date: 20/02/20 下午 1:17
     * @author: zhengkw
     */
    @Test
    public void testCopyToLocalFile() throws IOException {

        fileSystem.copyToLocalFile(false, new Path("/user/zhengkw/sanguo/345.txt"), new Path("F:/test/mimiga.txt"), true);
        out.println("finish!");
    }

    /**
     * @descrption:关闭资源
     * @return: void
     * @date: 20/02/20 下午 2:28
     * @author: zhengkw
     */
    @After
    public void close() throws IOException {
        fileSystem.close();
    }

    /**
     * @descrption:删除目录
     * @return: void
     * @date: 20/02/20 下午 2:31
     * @author: zhengkw
     */
    @Test
    public void testDelete() throws IOException {

        fileSystem.delete(new Path("/user"), true);//true 递归
    }

    /**
     * @descrption: 修改文件名
     * @return: void
     * @date: 20/02/20 下午 2:37
     * @author: zhengkw
     */
    @Test
    public void changeFileName() throws IOException {
        fileSystem.rename(new Path("/user/zhengkw/sanguo/345.txt"), new Path("/user/zhengkw/sanguo/新目标.txt"));
    }

    /**
     * @descrption:打印文件信息
     * @return: void
     * @date: 20/02/20 下午 3:15
     * @author: zhengkw
     */
    @Test
    public void listFiles() throws IOException {
        RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/"), true);

        while (listFiles.hasNext()) {
            LocatedFileStatus status = listFiles.next();
            out.println("这个是路径:" + status.getPath());
            out.println("长度:" + status.getLen());
            out.println("权限：" + status.getPermission());
            out.println("块大小:" + status.getBlockSize());
            out.println("分组：" + status.getGroup());


            // 获取存储的块信息
            BlockLocation[] blockLocations = status.getBlockLocations();
            for (BlockLocation blockLocation : blockLocations
            ) {
                // 获取块存储的主机节点
                String[] hosts = blockLocation.getHosts();
                for (String host : hosts
                ) {
                    out.println(host);
                }

            }
            System.out.println("---------------分割线-----------------");
        }
    }

    /**
     * @descrption:判断文件夹还是文件
     * @return: void
     * @date: 20/02/20 下午 3:41
     * @author: zhengkw
     */

    @Test
    public void judgeFileOrDirectory() throws IOException {
        //1获取文件状态（当前目录）
        FileStatus[] listStatus = fileSystem.listStatus(new Path("/user/zhengkw/sanguo"));
        //遍历文件状态
        for (FileStatus fs : listStatus
        ) {
            // 2判断是文件还是文件夹
            // 如果是文件输出这个是file，如果是目录输出这个是目录xxx
            if (fs.isFile()) {
                out.println("这个是文件：" + fs.getPath().getName());
            } else {
                out.println("这个是目录" + fs.getPath().getName());
            }

        }


    }

 


}

POM Maven

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.zhengkw</groupId>
    <artifactId>HDFSclient</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>RELEASE</version>
        </dependency>
        <dependency>
            <groupId>org.apache.logging.log4j</groupId>
            <artifactId>log4j-core</artifactId>
            <version>2.8.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-client</artifactId>
            <version>2.7.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-hdfs</artifactId>
            <version>2.7.2</version>
        </dependency>
        <!--     <dependency>
                   <groupId>jdk.tools</groupId>
                   <artifactId>jdk.tools</artifactId>
                   <version>1.8</version>
                   <scope>system</scope>
                   <systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
               </dependency>-->
    </dependencies>


</project>