Windows系统上 HDFS java API的使用

1.创建文件夹

package hdfsApi1;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
//创建文件夹
public class Api1 {
	public static void main(String[] args) {
		try {
//			读取配置文件
			Configuration conf  = new Configuration();
//			远程连接
			URI uri = new URI("hdfs://192.168.200.133:9000");
//			得到文件系统
			FileSystem fs = FileSystem.get(uri, conf, "wsm");
//			要创建的文件路径 包装为Path
			Path path = new Path("/spark/file");
			if (!fs.exists(path)) {
				if (fs.mkdirs(path)) {
					System.out.println("文件创建成功");
				}else {
					System.out.println("文件创建失败");
				}
			}else {
				System.out.println("文件已经存在");
			}
		} catch (URISyntaxException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
		
		
		
		
	}
}

2.列出文件状态

package hdfsApi1;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
//列出文件状态
public class API2 {
	public static void main(String[] args) {
		try {
//			读取配置文件
			Configuration conf = new Configuration();
//			远程URI
			URI uri = new URI("hdfs://192.168.200.132:9000");
//			获取文件系统
			FileSystem fs = FileSystem.get(uri,conf,"python");
//			需要列出哪个路径下的文件
			Path path = new Path("/wsm");
			FileStatus[] fss = fs.listStatus(path);
			for(FileStatus f1 : fss) {
				System.out.println(f1.getPath().getName());
			}
			
		} catch (URISyntaxException | IOException | InterruptedException e) {
			e.printStackTrace();
		}
	}
}

3.创建文件，并写入数据

package hdfsApi1;

import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//创建文件 并且写入数据
public class API3 {
	public static void main(String[] args) {
		try {
			Configuration conf = new Configuration();
//			设置文件的副本数 因为搭建的是伪分布式 所以副本数为 1
			conf.set("dfs.replication","1");
			URI uri = new URI("hdfs://192.168.200.132:9000");
			FileSystem fs = FileSystem.get(uri, conf,"python");
			String fileName = "/wsm/1.txt";
			Path path = new Path(fileName);
			if (!fs.exists(path)) {
//				通过 FSDataOutputStream 向文件写入数据 append()是追加
				FSDataOutputStream fos = fs.create(path);
				fos.writeBytes("this is 1.txt");
				fos.flush();
				fos.close();
			}else {
				System.out.println("文件已经存在");
			}
		} catch (URISyntaxException | IOException | InterruptedException e) {
			e.printStackTrace();
		}
	}
}

4.读取文件

public class API4 {
	public static void main(String[] args) {
		try {
			Configuration conf = new Configuration();
			URI uri = new URI("hdfs://192.168.200.132:9000");
			FileSystem fs = FileSystem.get(uri, conf, "python");
			Path f = new Path("/wsm/1.txt");
//			打开文件
			FSDataInputStream fis = fs.open(f);
			
			int len = -1;
			byte[] bt = new byte[1024];
			while ( (len=fis.read(bt)) >0 ) {
				System.out.println(new String(bt,0,len));
			}
		} catch (URISyntaxException | IOException | InterruptedException e) {
			e.printStackTrace();
		}
	}
}

5.删除文件

public class API5 {
	public static void main(String[] args) {
		try {
			
			Configuration conf  = new Configuration();
			URI uri = new URI("hdfs://192.168.200.132:9000");
			FileSystem fs = FileSystem.get(uri, conf, "python");
			
			Path path = new Path("/wsm/1.txt");
			if (fs.exists(path)) {
				//false 不递归删除
				if (fs.delete(path,false)) {
					System.out.println("文件删除成功");
				}else {
					System.out.println("文件删除失败");
				}
			}else {
				System.out.println("文件不存在");
			}
		} catch (URISyntaxException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
}

6.使用copyFromLocalFile上传文件

//文件上传
public class API7 {
	public static void main(String[] args) {
		try {
			Configuration conf = new Configuration();
			conf.set("dfs.replication","1");
			URI uri = new URI("hdfs://192.168.200.132:9000");
			FileSystem fs = FileSystem.get(uri, conf,"python");
			Path dst = new Path("/wsm/2_upload.txt");
			Path src = new Path("upload.txt");
			fs.copyFromLocalFile(src, dst);
			System.out.println("上传成功");
		} catch (URISyntaxException | IOException | InterruptedException e) {
			e.printStackTrace();
		}
	}
}

7.使用copyToLocalFile实现文件下载

//文件下载
public class API8 {
	public static void main(String[] args) {
		try {
			Configuration conf = new Configuration();
			conf.set("dfs.replication","1");
			URI uri = new URI("hdfs://192.168.200.132:9000");
			FileSystem fs = FileSystem.get(uri, conf,"python");
			Path src = new Path("/wsm/2_upload.txt");
			Path dst = new Path("download2.txt");
			fs.copyToLocalFile(src, dst);
			System.out.println("下载成功");
		} catch (URISyntaxException | IOException | InterruptedException e) {
			e.printStackTrace();
		}
	}
}

8.使用IOUtils读取文件

public class API9 {
	public static void main(String[] args) {
		try {
			Configuration conf = new Configuration();
			conf.set("dfs.replication","1");
			URI uri = new URI("hdfs://192.168.200.132:9000");
			FileSystem fs = FileSystem.get(uri, conf,"python");
			
			Path path = new Path("/wsm/2_upload.txt");
			FSDataInputStream fis = fs.open(path);
			
			IOUtils.copy(fis, System.out);
		} catch (URISyntaxException | IOException | InterruptedException e) {
			e.printStackTrace();
		}
	}
}

9.用IOUtils写入文件 然后用FileInputStream读取

public class API_B10 {
	public static void main(String[] args) {
		try {
			Configuration conf = new Configuration();
			conf.set("dfs.replication","1");
			URI uri = new URI("hdfs://192.168.200.132:9000");
			FileSystem fs = FileSystem.get(uri, conf,"python");
			Path path = new Path("/wsm/1.txt");
			FSDataOutputStream fos = fs.append(path);//文件系统追加其他文件的内容
			FileInputStream fis = new FileInputStream(new File("upload.txt"));
			IOUtils.copy(fis, fos);
			System.out.println("成功");
		} catch (URISyntaxException | IOException | InterruptedException e) {
			e.printStackTrace();
		}
	}
}