Hadoop_HDFS文件操作
-
新建hadoop_hdfs工程
-
pom.xml：
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>com.blu</groupId>
<artifactId>hadoop_hdfs</artifactId>
<version>0.0.1-SNAPSHOT</version>
<dependencies>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-hdfs -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>3.1.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-client -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.1.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-common -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.1.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<scope>test</scope>
</dependency>
</dependencies>
</project>
- HDFStest.java类
package com.blu;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IOUtils;
import org.apache.kerby.asn1.util.IOUtil;
import org.junit.Test;
import com.sun.jersey.core.spi.scanning.FilesScanner;
public class HDFStest {
@Test
public void uploadFile() {
Configuration conf = new Configuration();
//���ø����������ȼ�����xml�����ļ���jar���е�hdfs-default.xml
//conf.set("dfs.replication", "2");
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
fs.copyFromLocalFile(new Path("D:\\data\\testdata.txt"), new Path("/testdata2.txt"));
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Test
public void downloadFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
fs.copyToLocalFile(false,new Path("/hello.txt"), new Path("D:\\"),true);
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Test
public void createFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
FSDataOutputStream fos = fs.create(new Path("/input/a.txt"));
fos.write("Hello, a!".getBytes());
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Test
public void renameFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
fs.rename(new Path("/testdata11.txt"), new Path("/testdata1111.txt"));
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Test
public void listFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
// ����1���ļ�·��
// ����2���Ƿ�������ļ��У�trueΪ������falseΪ��������
RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
while(files.hasNext()) {
LocatedFileStatus fileStatus = files.next();
String filename = fileStatus.getPath().getName();
System.out.println("�ļ�����"+ filename);
System.out.println("��������"+ fileStatus.getReplication());
}
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Test
public void listStatusFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
FileStatus[] status = fs.listStatus(new Path("/"));
for(FileStatus sta : status) {
System.out.println("�ļ�����"+ sta.getPath().getName());
System.out.println("�Ƿ����ļ���"+ sta.isFile());
}
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Test
public void deleteFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
// ����1��Ҫɾ�����ļ�
// ����2���Ƿ���ɾ����������1���ļ�����дtrue
fs.delete(new Path("/testdata1111.txt"), true);
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
@Test
public void mkdir() {
Configuration conf = new Configuration();
FileSystem fs = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
fs.mkdirs(new Path("/input2/a"));
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
// ʹ��IOUtils���ļ�/�����ļ�
@Test
public void readFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
FSDataInputStream inputStream = null;
FileOutputStream fos = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
inputStream = fs.open(new Path("/hello.txt"));
fos = new FileOutputStream(new File("D:\\demo.txt"));
//IOUtils.copyBytes(inputStream, System.out, conf);
IOUtils.copyBytes(inputStream, fos, conf);
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
IOUtils.closeStream(inputStream);
IOUtils.closeStream(fos);
} catch (IOException e) {
e.printStackTrace();
}
}
}
// ʹ��IOUtils�ϴ��ļ�
@Test
public void upFile() {
Configuration conf = new Configuration();
FileSystem fs = null;
FileInputStream fis = null;
FSDataOutputStream fos = null;
try {
fs = FileSystem.get(new URI("hdfs://192.168.3.8:9000"), conf, "hadoop01");
fis = new FileInputStream(new File("D:\\demo.txt"));
fos = fs.create(new Path("/input/DEMO.txt"));
IOUtils.copyBytes(fis, fos, conf);
} catch (Exception e) {
e.printStackTrace();
}finally {
try {
fs.close();
IOUtils.closeStream(fis);
IOUtils.closeStream(fos);
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
- 在resource目录下创建hdfs-site.xml
<?xml version="1.0" encoding="UTF-8"?>
<!-- 设置副本数 -->
<property>
<name>dfs.replication</name>
<value>1</value>
<description>Default block replication</description>
</property>
