hadoop(二)–java操作hdfs文件系统

hadoop(二)–java操作hdfs文件系统

一、简介

这里介绍使用java客户端操作hdfs文件系统。

二、 步骤

2.1 添加maven依赖

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-common</artifactId>
    <version>3.0.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-hdfs</artifactId>
    <version>3.0.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-mapreduce-client-core</artifactId>
    <version>3.0.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>3.0.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-annotations</artifactId>
    <version>3.0.0</version>
</dependency>
<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
    <version>3.0.0</version>
</dependency>

2.2 hdfs操作

直接上代码，详情请查看代码注释

package com.dragon.study.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.IOException;
import java.util.Arrays;

public class HadoopPaperMain {
    public static void main(String[] args) throws Exception {
        //创建连接
        Configuration config = new Configuration();
        config.set("fs.defaultFS", "hdfs://192.168.0.1:9000");
        FileSystem fs = FileSystem.get(config);

        //创建文件夹
        Path newDirPath = new Path("/test1");
        if(!fs.exists(newDirPath)){
            fs.mkdirs(newDirPath);
        }

        //查看指定目录下的文件
        Path listDirPath = new Path("/test3");
        FileStatus[] fileStatuses = fs.listStatus(listDirPath);
        Arrays.stream(fileStatuses).forEach(fileStatus -> System.out.println(fileStatus.getPath().toString()));

        //写文件
        Path newFilePath = new Path("/test3/t1.txt");
        FSDataOutputStream fsout = fs.create(newFilePath);
        fsout.write("好好学习，天天向上".getBytes("utf8"));
//        fsout.writeChars("study hard ");
        fsout.flush();
        fsout.close();

        //读文件
        Path readFilePath = new Path("/test2/t2.txt");
        FSDataInputStream fsin = fs.open(readFilePath);
        IOUtils.copyBytes(fsin.getWrappedStream(), System.out,4096);

        //查看文件位置信息
        Path locationFilePath = new Path("/test3/t1.txt");
        FileStatus fileStatus = fs.getFileStatus(locationFilePath);
        BlockLocation[] locations = fs.getFileBlockLocations(locationFilePath, 0,fileStatus.getLen());
        Arrays.stream(locations).forEach(location -> {
            try {
                System.out.println(Arrays.toString(location.getHosts()));
            } catch (IOException e) {
                e.printStackTrace();
            }
        });

        //删除文件或文件夹
        Path delPath = new Path("/test1");
        fs.delete(delPath, true);

        //上传文件
        Path srcPath = new Path("E:/tmp/t2.txt");
        Path dstPath = new Path("/test2/");
        fs.copyFromLocalFile(srcPath,dstPath);

        fs.close();
    }
}