hadoop底层API示例

hadoop底层API示例

包括创建目录，上传文件，从HDFS读取文件

package com.ihep.yzz.test;

import java.io.BufferedInputStream;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.junit.Before;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.After;
import org.junit.Test;

public class TestHdfs {

	Configuration conf = null;
	FileSystem fs = null;

	@Before
	public void conn() throws IOException {
		conf = new Configuration();
		conf.addResource(new File("E:\\eclipse-workspace\\hello\\src\\core-site.xml").toURI().toURL());
		conf.addResource(new File("E:\\eclipse-workspace\\hello\\src\\hdfs-site.xml").toURI().toURL());
		fs = FileSystem.get(conf);
	}

	@Test
	public void mkdir() throws IOException {
		Path path = new Path("/mytemp");
		if (fs.exists(path)) {
			fs.delete(path, true);
		}
		fs.mkdirs(path);
	}

	@Test
	public void uploadFile() throws IOException {
		// 文件上传
		Path path = new Path("/mytemp/hello.txt");
		FSDataOutputStream fdos = fs.create(path);

		// 拿到磁盘文件
		InputStream is = new BufferedInputStream(
				new FileInputStream("E:\\eclipse-workspace\\hello\\src\\hdfs-site.xml"));

		IOUtils.copyBytes(is, fdos, conf, true);
	}

	@Test
	public void readFile() throws IOException {
		Path path = new Path("/mytemp/hello.txt");
		FileStatus file = fs.getFileStatus(path);
		BlockLocation[] blks =  fs.getFileBlockLocations(file, 0, file.getLen());
		
		for (BlockLocation blockLocation : blks) {
			System.out.println(blockLocation);
		}
		
		FSDataInputStream fdis = fs.open(path);
//		在某个位置开始读
		fdis.seek(1214);
		System.out.println((char)fdis.readByte());
		System.out.println((char)fdis.readByte());
		System.out.println((char)fdis.readByte());
		System.out.println((char)fdis.readByte());
		System.out.println((char)fdis.readByte());
		System.out.println((char)fdis.readByte());
		
	}
	
	@After
	public void close() throws IOException {
		fs.close();
	}
}