Java implements 8 api operations of hdfs
-
- 1. Pretreatment preparation
- 2. Maven project dependencies
- 3. Java source code
- Fourth, the realization of api operation
-
- 1. Preparations before realization
- 2. Create a path on hdfs
- 3. Delete the path on hdfs
- 4. Create hdfs file and write data
- 5. Delete files on hdfs
- 6. Move the path and rename the file on hdfs
- 7. Read the file content on hdfs
- 8. Upload local files under windows to hdfs
- 9. Download files from hdfs to local
1. Pretreatment preparation
1. Configure the local hadoop3.1.3 directory file
First copy the hadoop decompression version (tar -zxvf decompression) on Linux, and then delete (rm -rf) its share subdirectory (otherwise the downloaded file is relatively large)
Use MobaXterm to download to the local (such as D:\othersSofts directory), change the name to hadoop3.1.3, and then delete the hadoop directory file for download on the linux virtual machine
Add the Windows dependencies it needs in its /bin directory, and replace it when it encounters a file with the same name
2. Configure environment variables
2. Maven project dependencies
Create a Maven project and add the following dependencies to pom.xml:
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.1.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.1.3</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.3</version>
</dependency>
3. Java source code
package org.igeek.hdfsapi;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
/**
* 使用hdfs的api进行操作
*/
public class HdfsApiTest {
//文件系统对象
FileSystem fs = null;
/**
* 初始化文件系统
*
* @throws URISyntaxException
* @throws IOException
*/
@Before
public void init() throws URISyntaxException, IOException, InterruptedException {
//创建一个配置对象
Configuration conf = new Configuration();
// hdfs中的NameNode地址
//方法一: ip地址:端口
// String hdfsPath ="hdfs://192.168.31.53:8020";
//方法二:hostname:端口
String hdfsPath = "hdfs://bigdata03:8020";
String userName = "root";
//获取文件系统对象
fs = FileSystem.get(new URI(hdfsPath), conf, userName);
System.out.println("hdfs文件系统对象初始化完成!");
}
/**
* 关闭文件系统
*
* @throws IOException
*/
@After
public void close() throws IOException {
fs.close();
System.out.println("hdfs文件系统已关闭!");
}
/**
* 创建hdfs上的路径
*
* @throws IOException
*/
@Test
public void createPath() throws IOException {
boolean createFlag = fs.mkdirs(new Path("/hdfs_api"));
if (createFlag) {
System.out.println("路径创建成功");
} else {
System.out.println("路径创建失败");
}
}
/**
* 删除hdfs上的路径
*
* @throws IOException
*/
@Test
public void deletePath() throws IOException {
Path path = new Path("/test");
//判断删除路径是否存在
if (fs.exists(path)) {
//使用递归删除
boolean deleteFlag = fs.delete(path, true);
if (deleteFlag) {
System.out.println("删除路径成功");
} else {
System.out.println("删除路径失败");
}
}
}
/**
* 创建hdfs文件并写入数据
* @throwsIOException
*/
@Test
public void createFile() throws IOException {
//创建文件路径
Path path = new Path("/hdfs_api/add.txt");
FSDataOutputStream fos = fs.create(path);
//写出的数据
String content = "我是通过api写入的数据";
//通过流进行数据写出
fos.write(content.getBytes());
// 流的刷新
fos.flush();
}
/**
*删除hdfs上的文件
*@throws IOException
*/
@Test
public void deleteFile() throws IOException{
boolean deleteFlag = fs.deleteOnExit(new Path( "/touchz.txt"));
if(deleteFlag){
System.out. println("删除文件成功");
} else {
System. out. println("删除文件失败");
}
}
/**
* hdfs上的文件移动路径并改名
*@throws IOException
*/
@Test
public void moveFile() throws IOException {
//文件的源路径
Path src = new Path("/put.txt");
//文件移动后的路径
Path dst = new Path("/hdfs_api/put_new.txt");
boolean renameFlag = fs.rename(src, dst);
if (renameFlag) {
System.out.println("文件移动成功");
} else {
System.out.println("文件移动失败");
}
}
/**
* 读取hdfs上的文件内容
* @throws IOException
*/
@Test
public void readFile() throws IOException {
FSDataInputStream fis = fs.open(new Path("/hdfs_api/add.txt"));
IOUtils.copyBytes(fis, System.out, 2048, false);
System.out.println("\n");
}
/**
* 上传windows下的本地文件到hdfs上
* @throws IOException
*/
@Test
public void uploadEile() throws IOException {
//要上传的hdfs路径
Path src = new Path("D:\\othersofts\\hadoop3.1.3\\etc\\hadoop\\core-site.xml");
//文件的本地路径
Path dst = new Path("/hdfs_api");
fs.copyFromLocalFile(true, src, dst);
System.out.println("文件从本地上传hdfs成功");
}
/**
* 从hdfs上下载文件到本地
* @throws IOException
*/
@Test
public void downloadFile() throws IOException {
// hdfs路径
Path src = new Path("/hdfs_api/add.txt");
//本地路径
Path dst = new Path("D:\\");
fs.copyToLocalFile(false, src, dst,false);
System.out.println("下载文件成功");
}
}
Fourth, the realization of api operation
Note: right-click to execute the corresponding method under each annotation @Test, such as:
1. Preparations before realization
Then complete the initialization of the file system (annotation using @Before) and closing the file system (annotation using @After) in the Maven project:
//文件系统对象
FileSystem fs = null;
/**
* 初始化文件系统
*
* @throws URISyntaxException
* @throws IOException
*/
@Before
public void init() throws URISyntaxException, IOException, InterruptedException {
//创建一个配置对象
Configuration conf = new Configuration();
// hdfs中的NameNode地址
//方法一: ip地址:端口
// String hdfsPath ="hdfs://192.168.31.53:8020";
//方法二:hostname:端口
String hdfsPath = "hdfs://bigdata03:8020";
String userName = "root";
//获取文件系统对象
fs = FileSystem.get(new URI(hdfsPath), conf, userName);
System.out.println("hdfs文件系统对象初始化完成!");
}
/**
* 关闭文件系统
*
* @throws IOException
*/
@After
public void close() throws IOException {
fs.close();
System.out.println("hdfs文件系统已关闭!");
}
2. Create a path on hdfs
/**
* 创建hdfs上的路径
*
* @throws IOException
*/
@Test
public void createPath() throws IOException {
boolean createFlag = fs.mkdirs(new Path("/hdfs_api"));
if (createFlag) {
System.out.println("路径创建成功");
} else {
System.out.println("路径创建失败");
}
}
Enter the hadoop command to view the result of creating /hdfs_api:
3. Delete the path on hdfs
/**
* 删除hdfs上的路径
*
* @throws IOException
*/
@Test
public void deletePath() throws IOException {
Path path = new Path("/test");
//判断删除路径是否存在
if (fs.exists(path)) {
//使用递归删除
boolean deleteFlag = fs.delete(path, true);
if (deleteFlag) {
System.out.println("删除路径成功");
} else {
System.out.println("删除路径失败");
}
}
}
Enter the hadoop command to view the result of deleting /test:
4. Create hdfs file and write data
/**
* 创建hdfs文件并写入数据
* @throwsIOException
*/
@Test
public void createFile() throws IOException {
//创建文件路径
Path path = new Path("/hdfs_api/add.txt");
FSDataOutputStream fos = fs.create(path);
//写出的数据
String content = "我是通过api写入的数据";
//通过流进行数据写出
fos.write(content.getBytes());
// 流的刷新
fos.flush();
}
Enter the hadoop command to view the written data
5. Delete files on hdfs
/**
*删除hdfs上的文件
*@throws IOException
*/
@Test
public void deleteFile() throws IOException{
boolean deleteFlag = fs.deleteOnExit(new Path( "/touchz.txt"));
if(deleteFlag){
System.out. println("删除文件成功");
} else {
System. out. println("删除文件失败");
}
}
After entering the hadoop command, it is found that the existing /touchz.txt is deleted:
6. Move the path and rename the file on hdfs
/**
* hdfs上的文件移动路径并改名
*@throws IOException
*/
@Test
public void moveFile() throws IOException {
//文件的源路径
Path src = new Path("/put.txt");
//文件移动后的路径
Path dst = new Path("/hdfs_api/put_new.txt");
boolean renameFlag = fs.rename(src, dst);
if (renameFlag) {
System.out.println("文件移动成功");
} else {
System.out.println("文件移动失败");
}
}
Enter the hadoop command to view:
7. Read the file content on hdfs
/**
* 读取hdfs上的文件内容
* @throws IOException
*/
@Test
public void readFile() throws IOException {
FSDataInputStream fis = fs.open(new Path("/hdfs_api/add.txt"));
IOUtils.copyBytes(fis, System.out, 2048, false);
System.out.println("\n");
}
8. Upload local files under windows to hdfs
/**
* 上传windows下的本地文件到hdfs上
* @throws IOException
*/
@Test
public void uploadEile() throws IOException {
//要上传的hdfs路径
Path src = new Path("D:\\othersofts\\hadoop3.1.3\\etc\\hadoop\\core-site.xml");
//文件的本地路径
Path dst = new Path("/hdfs_api");
fs.copyFromLocalFile(true, src, dst);
System.out.println("文件从本地上传hdfs成功");
}
9. Download files from hdfs to local
/**
* 从hdfs上下载文件到本地
* @throws IOException
*/
@Test
public void downloadFile() throws IOException {
// hdfs路径
Path src = new Path("/hdfs_api/add.txt");
//本地路径
Path dst = new Path("D:\\");
fs.copyToLocalFile(false, src, dst,false);
System.out.println("下载文件成功");
}
Download result:
