BigData<6>_HDFS Java API 操作

package com.bigdata.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;


/**
 * 使用Java API 操作HDFS文件系统
 */
public class HDFSApp {

    public static final String HDFS_PATH = "hdfs://hadoop000:8020";
    FileSystem fileSystem = null;
    Configuration configuration = null;


    @Before
    public void setUp() throws Exception {
        configuration = new Configuration();

        /**
         * 为了后面访问client访问datanNode（open，write等操作）
         * 这里把datanode访问方式设置为用hostname访问，client端上建立了hadoop000和阿里云服务器的外网映射
         * 否则，由于云服务器上搭建的伪分布式系统中，namenode和datanode是同一个机器，采用内网连接，
         * nameNode返回给Client dataNode的地址来写数据时候，此时返回的dataNode地址是内网地址，client无法连接
         */
        configuration.set("dfs.client.use.datanode.hostname","true");

        /**
         * 伪分布式，修改默认的副本系数
         */
        configuration.set("dfs.replication","1");

        /**
         * 构造一个访问指定HDFS系统的客户端对象
         * 第一个参数：HDFS的URI
         * 第二个参数：客户端指定的配置参数
         * 第三个参数：客户端的身份，用户名
         */
        fileSystem = FileSystem.get(new URI(HDFS_PATH), configuration, "guoqiang");
        System.out.println("--------setUp--------");
    }

    /**
     * 创建文件夹
     * 只涉及到nameNode的操作，不回连接到dataNode
     */
    @Test
    public void mkdir() throws Exception {
        fileSystem.mkdirs(new Path("/jUnitTest/testB"));
    }

    /**
     * 查看HDFS内容
     */
    @Test
    public void text() throws Exception {
        FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"));
        IOUtils.copyBytes(in, System.out, 1024);
    }

    /**
     * 创建文件
     */
    @Test
    public void create() throws Exception {
        FSDataOutputStream out = fileSystem.create(new Path("/jUnitTest/testB/a.txt"));
        out.writeUTF("Hello GQ");
        out.flush();
        out.close();
    }

    /**
     * 文件名更改
     */
    @Test
    public void rename() throws Exception {
        Path oldPath = new Path("/jUnitTest/testB/a.txt");
        Path newPath = new Path("/jUnitTest/testB/b.txt");
        boolean result = fileSystem.rename(oldPath,newPath);
        System.out.println(result);

    }

    /**
     * 拷贝本地文件到fileSystem
     */
    @Test
    public void copyFromLocalFile() throws Exception {
        Path src = new Path("/Users/mac126/code/javaProgram/InsertionSort.java");
        Path dst = new Path("/jUnitTest/InsertionSort.java");
        fileSystem.copyFromLocalFile(src, dst);
    }

    /**
     * 拷贝大文件，带进度条
     */
    @Test
    public void copyFromLocalBigFile() throws Exception {
        InputStream in = new BufferedInputStream(new FileInputStream(new File("/Users/mac126/Downloads/94011351-1-80.flv")));

        FSDataOutputStream out = fileSystem.create(new Path("/graduateVideo/zyz.flv"), new Progressable() {
            @Override
            public void progress() {
                System.out.print("#");
            }
        });
        IOUtils.copyBytes(in, out, 4096);
    }

    /**
     * 拷贝HDFS文件到本地，即下载
     */
    @Test
    public void copyToLocalFile() throws Exception {
        Path src = new Path("/jUnitTest/InsertionSort.java");
        Path dst = new Path("/Users/mac126/blog/javaProgram.java");
        fileSystem.copyToLocalFile(src, dst);
    }

    /**
     * 查看目标文件夹下的所有文件
     */
    @Test
    public void listFiles() throws Exception {
        FileStatus[] statuses = fileSystem.listStatus(new Path("/jUnitTest/"));
        for (FileStatus file : statuses) {
            String isDir = file.isDirectory() ? "文件夹":"文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();

            System.out.println(isDir + "\t" + permission + "\t" + permission + "\t" +
                    replication + "\t" + length + "\t" + path);
        }
    }

    /**
     * 递归查看目标文件夹下的所有文件
     */
    @Test
    public void listFilesRecursion() throws Exception {
        RemoteIterator<LocatedFileStatus> iterator = fileSystem.listFiles(new Path("/jUnitTest/"), true);
        while(iterator.hasNext()) {
            LocatedFileStatus file = iterator.next();
            String isDir = file.isDirectory() ? "文件夹":"文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();

            System.out.println(isDir + "\t" + permission + "\t" + permission + "\t" +
                    replication + "\t" + length + "\t" + path);
        }
    }

    /**
     * 查看文件块信息
     */
    @Test
    public void getFileBlockLocations() throws Exception {
        FileStatus status = fileSystem.getFileStatus(new Path("/graduateVideo/zyz.flv"));
        BlockLocation[] blockLocations = fileSystem.getFileBlockLocations(status, 0, status.getLen());
        for(BlockLocation blockLocation : blockLocations) {
            for(String name : blockLocation.getNames()) {
                System.out.println(name + " :" + blockLocation.getOffset() + " : " + blockLocation.getLength());
            }
        }
    }

    /**
     * 删除操作
     */
    @Test
    public void delete() throws Exception {
        boolean result = fileSystem.delete(new Path("/jUnitTest/InsertionSort.java"), false);
        System.out.println(result);
    }


    @After
    public void tearDown() {
        configuration = null;
        fileSystem = null;
        System.out.println("--------tearDown--------");
    }

//    public static void main(String[] args) throws Exception {
//        Configuration configuration = new Configuration();
//        FileSystem fileSystem = FileSystem.get(new URI("hdfs://hadoop000:8020"), configuration, "guoqiang");
//        boolean result = fileSystem.mkdirs(new Path("/hdfsApi/test"));
//        System.out.println(result);
//    }
}