package operateFile;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
public class HDFS_File {
public static void main(String args[]) throws IOException{
HDFS_File fu = new HDFS_File();
// HadoopFileUtil hu = new HadoopFileUtil();
String src = "/hadoop/files02";
String dst = "hdfs://s21:9000/te";
dst = "hdfs://192.168.100.221:9000/user/ad";
String localSrc = "/hadoop/file02";
dst = "hdfs://localhost:9000/user/filess021";
Configuration conf = new Configuration();
Object obj = conf.get("fs.default.name");
System.out.println(obj);
String dst2 = "hdfs://localhost:9000/user/ad";
// fu.putFileFormLocal(conf, src, dst);
// hu.createFile(src, dst);
/* InputStream in = new BufferedInputStream(new FileInputStream(src));
fu.createFileByInputStream(conf,in,dst);*/
// fu.CreateFile(conf, dst);
fu.PutFile(conf, src, dst);
// fu.ReadFile(conf, dst);
// fu.GetFile(conf,dst, src);
// fu.ReNameFile(conf, dst, dst2);
// fu.DelFile(conf, dst2, false);
// fu.GetFileModTime(conf, src);
}
//read the file from HDFS
public void ReadFile(Configuration conf, String FileName){
try{
FileSystem hdfs = FileSystem.get(URI.create(FileName),conf);
FSDataInputStream dis = hdfs.open(new Path(FileName));
IOUtils.copyBytes(dis, System.out, 4096, false);
dis.close();
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
//copy the file from HDFS to local
public void GetFile(Configuration conf, String srcFile, String dstFile){
try {
FileSystem hdfs = FileSystem.get(URI.create(srcFile),conf);
Path srcPath = new Path(srcFile);
Path dstPath = new Path(dstFile);
hdfs.copyToLocalFile(false,srcPath, dstPath);
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
public void createFileByInputStream(Configuration conf, InputStream in , String dstFile) throws IOException{
FileSystem fs = FileSystem.get(URI.create(dstFile), conf);
// fs = FileSystem.get(conf);
/* FileStatus[] fst = fs.listStatus(new Path("/user"));
for(FileStatus f:fst){
System.out.println(f.getPath());
}*/
OutputStream out = fs.create(new Path(dstFile));
/* OutputStream out = fs.create(new Path(dstFile), new Progressable() {
public void progress() {
System.out.print(".");
}
});*/
IOUtils.copyBytes(in, out, 4096, true);
}
public void putFileFormLocal(Configuration conf, String srcFile, String dstFile) throws IOException{
InputStream in = new BufferedInputStream(new FileInputStream(srcFile));
/* FileSystem hdfs = FileSystem.get(conf);
boolean b = hdfs.exists(new Path("/hadoop/hadoop"));
System.out.println(b);*/
FileSystem fs = FileSystem.get(URI.create(dstFile), conf);
// fs = FileSystem.get(conf);
FileStatus[] fst = fs.listStatus(new Path("/user"));
for(FileStatus f:fst){
System.out.println(f.getPath());
}
OutputStream out = fs.create(new Path(dstFile), new Progressable() {
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, out, 4096, true);
}
//copy the local file to HDFS
public void PutFile(Configuration conf, String srcFile, String dstFile) throws IOException {
/* try {
FileSystem hdfs = FileSystem.get(conf);
Path srcPath = new Path(srcFile);
Path dstPath = new Path(dstFile);
hdfs.copyFromLocalFile(srcPath, dstPath);
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
*/
FileSystem fs = FileSystem.get(URI.create(dstFile),conf);
// Hadoop DFS deals with Path
Path outFile = new Path(dstFile);
// Check if input/output are valid
File inputfile = new File(srcFile);
if(!inputfile.exists()&&!inputfile.isFile())
{
// printAndExit("Input should be a file");
}
// if (fs.exists(outFile))
// printAndExit("Output already exists");
// Read from and write to new file
// FSDataInputStream in = fs.open(inFile);
FSDataOutputStream out = fs.create(outFile);
InputStream image = new FileInputStream(inputfile);
byte buffer[] = new byte[256];
try {
int bytesRead = 0;
while ((bytesRead = image.read(buffer)) > 0) {
out.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
System.out.println("Error while copying file");
} finally {
image.close();
out.close();
}
}
//create the new file
public FSDataOutputStream CreateFile(Configuration conf, String FileName){
try {
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(URI.create(FileName),config);
Path path = new Path(FileName);
FSDataOutputStream outputStream = hdfs.create(path);
return outputStream;
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
//rename the file name
public boolean ReNameFile(Configuration conf, String srcName, String dstName){
try {
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(URI.create(srcName),config);
Path fromPath = new Path(srcName);
Path toPath = new Path(dstName);
boolean isRenamed = hdfs.rename(fromPath, toPath);
return isRenamed;
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
}
//delete the file
// tyep = true, delete the directory
// type = false, delece the file
public boolean DelFile(Configuration conf, String FileName, boolean type){
try {
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(URI.create(FileName),config);
Path path = new Path(FileName);
boolean isDeleted = hdfs.delete(path, type);
return isDeleted;
}catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
}
//Get HDFS file last modification time
public long GetFileModTime(Configuration conf, String FileName){
try{
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(config);
Path path = new Path(FileName);
FileStatus fileStatus = hdfs.getFileStatus(path);
long modificationTime = fileStatus.getModificationTime();
System.out.println(modificationTime);
return modificationTime;
}catch(IOException e){
e.printStackTrace();
}
return 0;
}
//checke if a file exists in HDFS
public boolean CheckFileExist(Configuration conf, String FileName){
try{
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(config);
Path path = new Path(FileName);
boolean isExists = hdfs.exists(path);
return isExists;
}catch(IOException e){
e.printStackTrace();
}
return false;
}
//Get the locations of a file in the HDFS cluster
public List<String []> GetFileBolckHost(Configuration conf, String FileName){
try{
List<String []> list = new ArrayList<String []>();
Configuration config = new Configuration();
FileSystem hdfs = FileSystem.get(config);
Path path = new Path(FileName);
FileStatus fileStatus = hdfs.getFileStatus(path);
BlockLocation[] blkLocations = hdfs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
int blkCount = blkLocations.length;
for (int i=0; i < blkCount; i++) {
String[] hosts = blkLocations[i].getHosts();
list.add(hosts);
}
return list;
}catch(IOException e){
e.printStackTrace();
}
return null;
}
//Get a list of all the nodes host names in the HDFS cluster
public String[] GetAllNodeName(Configuration conf){
try{
Configuration config = new Configuration();
FileSystem fs = FileSystem.get(config);
DistributedFileSystem hdfs = (DistributedFileSystem) fs;
DatanodeInfo[] dataNodeStats = hdfs.getDataNodeStats();
String[] names = new String[dataNodeStats.length];
for (int i = 0; i < dataNodeStats.length; i++) {
names[i] = dataNodeStats[i].getHostName();
}
return names;
}catch(IOException e){
e.printStackTrace();
}
return null;
}
}
hdfs 操作类自己的
猜你喜欢
转载自zhrglchp.iteye.com/blog/1436637
今日推荐
周排行