HDFS（12）--HDFS的javaAPI操作

创建maven工程并导入jar包

<id>cloudera</id>

<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>

</repository>

</repositories>

<groupId>org.apache.Hadoop</groupId>

<artifactId>Hadoop-client</artifactId>

</dependency>

<groupId>org.apache.Hadoop</groupId>

<artifactId>Hadoop-common</artifactId>

</dependency>

<groupId>org.apache.Hadoop</groupId>

<artifactId>Hadoop-hdfs</artifactId>

</dependency>

<groupId>org.apache.Hadoop</groupId>

<artifactId>Hadoop-mapreduce-client-core</artifactId>

</dependency>

<groupId>junit</groupId>

<artifactId>junit</artifactId>

</dependency>

<groupId>org.testng</groupId>

<artifactId>testng</artifactId>

<version>RELEASE</version>

</dependency>

</dependencies>

<build>

<groupId>org.apache.maven.plugins</groupId>

<artifactId>maven-compiler-plugin</artifactId>

</configuration>

</plugin>

<groupId>org.apache.maven.plugins</groupId>

<artifactId>maven-shade-plugin</artifactId>

<phase>package</phase>

<goals>

<goal>shade</goal>

</goals>

</configuration>

</execution>

</executions>

</plugin>

<!-- <plugin>

<artifactId>maven-assembly-plugin </artifactId>

<descriptorRef>jar-with-dependencies</descriptorRef>

</descriptorRefs>

<mainClass>cn.itcast.Hadoop.db.DBToHdfs2</mainClass>

</manifest>

</archive>

</configuration>

<id>make-assembly</id>

<phase>package</phase>

<goals>

<goal>single</goal>

</goals>

</execution>

</executions>

</plugin>-->

</plugins>

</build>

使用url的方式访问数据

@Test
public void demo1()throws Exception{
    //第一步：注册hdfs 的url，让java代码能够识别hdfs的url形式
    URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());

    InputStream inputStream = null;
    FileOutputStream outputStream =null;
    //定义文件访问的url地址
    String url = "hdfs://192.168.52.100:8020/test/input/install.log";
    //打开文件输入流
    try {
        inputStream = new URL(url).openStream();
        outputStream = new FileOutputStream(new File("c:\\hello.txt"));
        IOUtils.copy(inputStream, outputStream);
    } catch (IOException e) {
        e.printStackTrace();
    }finally {
        IOUtils.closeQuietly(inputStream);
        IOUtils.closeQuietly(outputStream);
    }
}

如果执行出现以下错误，可以参见资料如何解决，也可以不用理会，不会影响程序的执行。记得配置完成环境变量之后重启开发工具

在 java 中操作 HDFS，主要涉及以下 Class：

Configuration：该类的对象封转了客户端或者服务器的配置;

FileSystem：该类的对象是一个文件系统对象，可以用该对象的一些方法来对文件进行操作，通过 FileSystem 的静态方法 get 获得该对象。

FileSystem fs = FileSystem.get(conf)

get 方法从 conf 中的一个参数 fs.defaultFS 的配置值判断具体是什么类型的文件系统。如果我们的代码中没有指定 fs.defaultFS，并且工程 classpath下也没有给定相应的配置，conf中的默认值就来自于Hadoop的jar包中的core-default.xml ，默认值为： file:/// ，则获取的将不是一个DistributedFileSystem 的实例，而是一个本地文件系统的客户端对象

具体代码示例：

/**
 * @author kismet
 * @date 2019-11-07 13:26
 */
public class text02 {
    static Configuration conf=new Configuration();

    public static void main(String[] args) throws IOException, URISyntaxException {
//        listStatus();
//        rename();
//        gettimr();
//        delfile();
//        mkdir();
//        addfile();
//        put();
        check();
    }

//展示目录下使有文件
    public static void listStatus() throws URISyntaxException, IOException {
        FileSystem hdfs = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        FileStatus status[] = hdfs.listStatus(new Path("/abc"));
        for (int i = 0; i <status.length ; ++i) {
            System.out.println(status[i].getPath().toString());
            hdfs.close();
        }
    }
//修改目录下指定文件名
    public static void rename() throws URISyntaxException, IOException {
        FileSystem hdfs = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        Path path = new Path("/abc/a.txt");
        Path path1 = new Path("/abc/aa.txt");
        boolean rename = hdfs.rename(path, path1);
        System.out.println(rename);
    }
//获取文件日期
    public static void gettimr() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        FileStatus fileStatus = fileSystem.getFileStatus(new Path("/abc/aa.txt"));
        long modificationTime = fileStatus.getModificationTime();
        System.out.println(modificationTime);
    }
 //删除文件
    public static void delfile() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        boolean delete = fileSystem.delete(new Path("/abc/aa.txt"), true);
        System.out.println(delete);
    }
 //创建文件夹
    public static void mkdir() throws URISyntaxException, IOException{
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        boolean mkdirs = fileSystem.mkdirs(new Path("/aaa"));
        System.out.println(mkdirs);
    }
//创建数据
    public static void addfile() throws URISyntaxException, IOException{
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        byte[] bytes = "hellowold".getBytes();
        FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/aaa/a.txt"));
        fsDataOutputStream.write(bytes,0,bytes.length);
        fsDataOutputStream.close();
    }
//上传数据
    public static void put() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        Path path = new Path("F:/ttt.txt");
        Path path1 = new Path("/");
        fileSystem.copyFromLocalFile(path,path1);
    }
//检查目录是否存在
    public static void check() throws URISyntaxException, IOException {
        FileSystem fileSystem = FileSystem.get(new URI("hdfs://192.168.100.88:8020"), conf);
        Path path = new Path("/abc");
        boolean exists = fileSystem.exists(path);
        System.out.println(exists);
    }
}

kismetG

发布了80 篇原创文章 · 获赞 168 · 访问量 8万+

私信关注

HDFS（12）--HDFS的javaAPI操作

猜你喜欢