JAVA access to HDFS (full of dry goods! Introduction to basic operations)

The following introductory source code all explain the functions, not much to say (don't forget to turn on Hadoop before running)

 

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

//读取指定文件，将文件内容输出到控制台
public class ReadData {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
        //定义文件系统路径
		String hdfs_address="hdfs://master:9000";
		//访问的具体文件路径
		String uri=hdfs_address+"/in/words";
		//填写配置信息(默认加载core-default.xml和core-site.xml)
		Configuration conf=new Configuration();
		FSDataInputStream in=null;//输入流字节
		try {
			//第一个参数必须以对象形式传递，不能是字符串
			//第二个参数是配置信息，即上面的Configuration对象
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), conf, "czh");
			in=fs.open(new Path(uri));
		    /*第二个参数是输出到控制台，第三个是缓冲区，
		           第四个是copy完了之后要不要关闭源数据和目的数据
		     */
			//把读到的数据输出到控制台
			IOUtils.copyBytes(in, System.out,4096,false);
		}catch(Exception e) {
			e.printStackTrace();
		}
		finally {
			IOUtils.closeStream(in);//关闭资源
		}
	}

}

 

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//创建目录
public class CreateDir {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		Configuration conf=new Configuration();
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), conf, "czh");
			//mkdirs函数会返回一个布尔值来代表是否创建成功
			boolean flag=fs.mkdirs(new Path(hdfs_address+"/my/tmp"));
			System.out.println("Directory create"+(flag?"successfully":"failed"));
		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

 

 

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//上传文件
public class PutFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    Path src=new Path("D:\\a.txt");//注意分割符与Windows下相反,此文件必须存在！
		    Path dest=new Path(hdfs_address+"/my");//上传目标路径
		    fs.copyFromLocalFile(src, dest);
		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

 

 

import java.io.FileOutputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

//下载文件,并写入到本地D:\\my\\路径下的b.txt文件中
public class DownFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    Path src=new Path(hdfs_address+"/my/a.txt");
		    FileOutputStream out=new FileOutputStream("D:\\my\\b.txt");
		    //open函数用于创建一个输入流，起点是src
		    FSDataInputStream in=fs.open(src);
		    IOUtils.copyBytes(in, out, 4096,false);
;		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

 

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//列出目录下的文件
public class ListFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    FileStatus files[]=fs.listStatus(new Path(hdfs_address+"/my"));
		    for(FileStatus file:files) {
		    	System.out.println(file.getPath());
		    }
;		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}

 

import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

//删除文件
public class DeleteFile {

	public static void main(String[] args) {
		// TODO Auto-generated method stub
		String hdfs_address="hdfs://master:9000";
		try {
			FileSystem fs=FileSystem.get(URI.create(hdfs_address), new Configuration(), "czh");
		    boolean flag=fs.delete(new Path(hdfs_address+"/my"),true);//第二个参数是是否递归删除
		    System.out.println("File deleted:"+flag);
;		}catch(Exception e) {
			e.printStackTrace();
		}

	}

}