hdfs operation of obtaining a list of files in the specified path

Add pom.xml dependence

        <dependency>
            <groupId>org.apache.hive</groupId>
            <artifactId>hive-jdbc</artifactId>
            <version>2.1.0</version>
            <exclusions>
                <exclusion>
                    <groupId>org.slf4j</groupId>
                    <artifactId>slf4j-log4j12</artifactId>
                </exclusion>
                <!--                <exclusion>-->
                <!--                    <groupId>log4j</groupId>-->
                <!--                    <artifactId>log4j</artifactId>-->
                <!--                </exclusion>-->
                <exclusion>
                    <groupId>org.apache.logging.log4j</groupId>
                    <artifactId>*</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.eclipse.jetty</groupId>
                    <artifactId>*</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.eclipse.jetty.orbit</groupId>
                    <artifactId>javax.servlet</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.mortbay.jetty</groupId>
                    <artifactId>servlet-api-2.5</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>javax.servlet</groupId>
                    <artifactId>servlet-api</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>org.apache.hive.shims</groupId>
                    <artifactId>hive-shims-common</artifactId>
                </exclusion>
            </exclusions>
        </dependency>
    </dependencies>

Achieve obtain a list of files in the specified path

package worktrain;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HdfsController {

    //获取指定路径下的文件列表
    public static void Hdfs() {
        try {
            Configuration conf = new Configuration();
            conf.set("fs.defaultFS", "hdfs://121.36.17.42:9000");
            FileSystem fs = null;
            //fs = FileSystem.get(new URI("hdfs://10.8.6.126:8020"),conf); //这两种方式都可以配置hdfs ip
            fs = FileSystem.get(conf);

            Path path = new Path("/");
            //通过fs的listStatus方法获取一个指定path的所有文件信息(status),因此我们需要传入一个hdfs的路径,返回的是一个filStatus数组
            FileStatus[] fileStatuses = fs.listStatus(path);
            for (FileStatus fileStatus : fileStatuses) {
                //判断当前迭代对象是否是目录
                boolean isDir = fileStatus.isDirectory();
                //获取当前文件的绝对路径
                String fullPath = fileStatus.getPath().toString();
                System.out.println("isDir:" + isDir + ",Path:" + fullPath);
            }
        }
        catch (Exception e){
            System.out.println(e.getStackTrace());
        }
    }

    public static void main(String[] args) {
        Hdfs();
    }
}

 

Published 111 original articles · won praise 57 · views 60000 +

Guess you like

Origin blog.csdn.net/qq_38358499/article/details/104642597