实验环境
hadoop版本: 2.6.5
HDFS API
Java抽象类org.apache.hadoop.fs.FileSystem定义了hadoop的一个文件系统接口。该类是一个抽象类,通过以下两种静态工厂方法可以过去FileSystem实例:
public static FileSystem.get(Configuration conf) throws IOException
public static FileSystem.get(URI uri, Configuration conf) throws IOException
常见方法介绍:
public boolean mkdirs(Path f) throws IOException
一次性新建所有目录(包括父目录), f是完整的目录路径。
public FSOutputStream create(Path f) throws IOException
创建指定path对象的一个文件,返回一个用于写入数据的输出流
create()有多个重载版本,允许我们指定是否强制覆盖已有的文件、文件备份数量、写入文件缓冲区大小、文件块大小以及文件权限。
public boolean copyFromLocal(Path src, Path dst) throws IOException
将本地文件拷贝到HDFS文件系统
public boolean exists(Path f) throws IOException
检查文件或目录是否存在
public boolean delete(Path f, Boolean recursive)
永久性删除指定的文件或目录,如果f是一个空目录或者文件,那么recursive的值就会被忽略。只有recursive=true时,一个非空目录及其内容才会被删除。
FileStatus类封装了文件系统中文件和目录的元数据,包括文件长度、块大小、备份、修改时间、所有者以及权限信息。
例程
package com.tongfang.learn.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.*;
import java.net.URI;
public class HdfsTest {
private static final String HDFS_URL = "hdfs://192.168.1.160:9000";
FileSystem fileSystem = null;
Configuration configuration = null;
/**
* 创建hdfs目录
* @throws IOException
*/
@Test
public void testMkdir() throws IOException {
fileSystem.mkdirs(new Path("/hdfs/test"));
}
/**
* 创建文件
* @throws IOException
*/
@Test
public void testCreateFile() throws IOException {
FSDataOutputStream out = fileSystem.create(new Path("/hdfs/test/a.txt"));
out.write("hello, I am a file !!!".getBytes());
out.flush();
out.close();
}
/**
* 追加文件
* @throws IOException
*/
@Test
public void testAppendFile() throws IOException {
FSDataOutputStream out = fileSystem.append(new Path("/hdfs/test/a.txt"));
out.write("append to file".getBytes());
out.flush();
out.close();
}
/**
* 查看文件内容
* @throws IOException
*/
@Test
public void testCatFile() throws IOException {
FSDataInputStream in = fileSystem.open(new Path("/hdfs/test/a.txt"));
IOUtils.copyBytes(in, System.out, 1024);
in.close();
}
/**
* 重命名文件
* @throws IOException
*/
@Test
public void testRename() throws IOException {
Path oldPath = new Path("/hdfs/test/a.txt");
Path newPath = new Path("/hdfs/test/b.txt");
fileSystem.rename(oldPath, newPath);
}
/**
* 删除文件
*/
@Test
public void testDelete() throws IOException {
fileSystem.delete(new Path("/hdfs/test"), true);
}
/**
* 上传本地文件到HDFS
* @throws IOException
*/
@Test
public void testCopyFromLocal() throws IOException {
Path localPath = new Path("D:\\input.txt");
Path hdfsPath = new Path("/hdfs/input.txt");
fileSystem.copyFromLocalFile(localPath, hdfsPath);
}
@Test
public void testCopyFromLocalWithStream() throws IOException {
InputStream in = new BufferedInputStream(new FileInputStream("D:\\input.zip"));
FSDataOutputStream output = fileSystem.create(new Path("/hdfs/input.zip"), new Progressable() {
@Override
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, output, 4096);
}
/**
* 下载HDFS文件到本地
* @throws IOException
*/
@Test
public void testCopyToLocal() throws IOException {
Path localPath = new Path("D:\\local.txt");
Path hdfsPath = new Path("/hdfs/input.txt");
fileSystem.copyToLocalFile(false, hdfsPath, localPath, true);
}
/**
* 下载Hdfs文件到本地
*/
@Test
public void testCopyToLocalWithStream() throws IOException {
InputStream input = fileSystem.open(new Path("/hdfs/input.zip"));
OutputStream output = new FileOutputStream("D:\\input.zip");
IOUtils.copyBytes(input, output, 4096, true);
}
/**
* 查看某个目录下的所有文件
* @throws IOException
*/
@Test
public void testListFiles() throws IOException {
FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfs"));
for (FileStatus status : statuses) {
String isDir = status.isDirectory() ? "目录" : "文件";
short replication = status.getReplication();
long len = status.getLen();
String path = status.getPath().toString();
long modtime = status.getModificationTime();
System.out.println(isDir + " " + replication + " " + len + " " +
path + " " + modtime);
}
}
/**
* 获取某个文件各个块在集群主机的分布信息
* @throws IOException
*/
@Test
public void testFileBlockLocation() throws IOException {
FileStatus status = fileSystem.getFileStatus(new Path("/hdfs/input.zip"));
BlockLocation[] locations = fileSystem.getFileBlockLocations(status, 0, status.getLen());
for (BlockLocation location: locations) {
String[] hosts = location.getHosts();
String allHosts = "";
for (String host : hosts) {
allHosts += host;
allHosts += " ";
}
System.out.println("block: " + location + " hosts: " + allHosts);
}
}
@Before
public void setUp() throws Exception {
configuration = new Configuration();
//必须添加这两个配置,否则可能导致append文件失败
configuration.set("dfs.client.block.write.replace-datanode-on-failure.policy","NEVER");
configuration.set("dfs.client.block.write.replace-datanode-on-failure.enable", "true");
fileSystem = FileSystem.get(new URI(HDFS_URL), configuration, "hadoop");
System.out.println("testcase setup");
}
@After
public void tearDown() {
configuration = null;
fileSystem = null;
System.out.println("testcase teardown");
}
}
pom.xml:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.tongfang.learn</groupId>
<artifactId>learn</artifactId>
<version>1.0-SNAPSHOT</version>
<name>learn</name>
<!-- FIXME change it to the project's website -->
<url>http://www.example.com</url>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.7</maven.compiler.source>
<maven.compiler.target>1.7</maven.compiler.target>
<hadoop.version>2.6.5</hadoop.version>
</properties>
<dependencies>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
</dependencies>
<build>
<pluginManagement><!-- lock down plugins versions to avoid using Maven defaults (may be moved to parent pom) -->
<plugins>
<plugin>
<artifactId>maven-clean-plugin</artifactId>
<version>3.0.0</version>
</plugin>
<!-- see http://maven.apache.org/ref/current/maven-core/default-bindings.html#Plugin_bindings_for_jar_packaging -->
<plugin>
<artifactId>maven-resources-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.7.0</version>
</plugin>
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.20.1</version>
</plugin>
<plugin>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
</plugin>
<plugin>
<artifactId>maven-install-plugin</artifactId>
<version>2.5.2</version>
</plugin>
<plugin>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</pluginManagement>
</build>
</project>