package com.hadoop.hdfs;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
public class HdfsUtil {
FileSystem fs = null;
@Before
public void init() throws IOException, InterruptedException, URISyntaxException{
//读取classpath下的**-site.xml配置文件,解析内容,封装到conf对象中
Configuration conf = new Configuration();
//直接在代码中对conf中的配置信息进行手动设置,会覆盖掉配置文件中的读取的值,所以不用copy**-site.xml文件,
conf.set("fs.defaultFS", "hdfs://192.168.21.115:9000/");
fs = FileSystem.get(new URI("hdfs://192.168.21.115:9000/"), conf, "admin");
}
/**
* 上传文件
* @throws IOException
*/
@Test
public void upload() throws IOException{
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://192.168.21.115:9000/");
FileSystem fs = FileSystem.get(conf);
Path path = new Path("hdfs://192.168.21.115:9000/aa/test1.txt");
FSDataOutputStream out = fs.create(path);
FileInputStream is = new FileInputStream("e:/test.txt");
IOUtils.copy(is, out);
}
/**
* 封装好的上传文件方法
* @throws IOException
* @throws IllegalArgumentException
*/
@Test
public void upload2() throws IllegalArgumentException, IOException{
fs.copyFromLocalFile(new Path("e:/test.txt"), new Path("hdfs://192.168.21.115:9000/bb/cc/text2.txt"));
}
/**
* 下载文件
* @throws Exception
* @throws IllegalArgumentException
* fail
*/
@Test
public void download() throws Exception{
fs.copyToLocalFile(new Path("hdfs://192.168.21.115:9000/bb/cc/text1.txt"), new Path("E:///test3333.txt"));
}
/**
* 下载1
* @throws Exception
*/
@Test
public void download1() throws Exception{
fs.copyToLocalFile(false,new Path("hdfs://192.168.21.115:9000/2.txt"), new Path("E:/test3333.txt"),true);
}
/**
* 下载文件2
* @throws IOException
*/
@Test
public void download2() throws IOException{
FSDataInputStream in = fs.open(new Path("hdfs://192.168.21.115:9000/2.txt"));
FileOutputStream out = new FileOutputStream("E:/test444.txt");
IOUtils.copy(in, out);
}
/**
* 查看文件信息
* @throws IOException
* @throws IllegalArgumentException
* @throws FileNotFoundException
*/
@Test
public void litFiles() throws FileNotFoundException, IllegalArgumentException, IOException{
//递归遍历
RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
while(files.hasNext()){
LocatedFileStatus file = files.next();
Path path = file.getPath();
String name = path.getName();
System.out.println("---------"+name);
}
System.out.println("=========================");
FileStatus[] list = fs.listStatus(new Path("/"));
for(FileStatus status : list){
String pathName = status.getPath().getName();
System.out.println(pathName+"---------"+(status.isDirectory()?"is dir":"is file"));
}
}
/**
* 创建文件夹
* @throws IOException
* @throws IllegalArgumentException
*/
@Test
public void mkdir() throws IllegalArgumentException, IOException{
fs.mkdirs(new Path("/test/aa/bb"));
}
/**
* 删除文件/文件夹
* @throws IOException
* @throws IllegalArgumentException
*/
@Test
public void delFile() throws IllegalArgumentException, IOException{
fs.delete(new Path("/test"),true);
}
}