Hadoop environment configuration on windows and HDFS API programming demonstration

1. Decompress the Hadoop compressed package and place it in the specified directory.

2. Configure the local environment of Hadoop.
Create a new HADOOP_HOME and

add path.

3.
Install maven to solve the dependency problem of java development.
You can download it directly from the official website
https://maven.apache.org/

<mirror> <id>alimaven</id> <name>aliyun maven</name> <url>http://maven.aliyun.com/nexus/content/groups/public/</url> <mirrorOf>central</mirrorOf> </mirror>

Modify the configuration file here. The original one is downloaded from the official website by default. If you don't overturn the wall, the connection to the Internet is slow and limited, so here we can connect to Ali's mirror. The location is probably between 150-160 rows.

    <dependency>
    		<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>2.7.3</version>
	</dependency>

		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>2.7.3</version>
		</dependency>

		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>2.7.3</version>
		</dependency>

Add this code to pom.xml

HDFS API programming

package Test.com.test;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;

public class UploadFile {
    
    

//	@SuppressWarnings("deprecation")
	public static void main(String[] args) {
    
    
		// 实例化conf
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://192.168.2.101:9000");
		System.setProperty("HADOOP_USER_NAME", "root");

		// 实例化文件系统
		try {
    
    
			FileSystem fs = FileSystem.get(conf);
			// 文件下载操作
			//fs.copyFromLocalFile(new Path("E:/student.txt"), new Path("/data"));
			// 文件夹删除操作
			//fs.delete(new Path("/banhua.txt"), true);
			//修改文件名称
			//fs.rename(new Path("/dashuju/jj.txt"), new Path("/data/banhua.txt"));
			//创建文件夹
			//fs.mkdirs(new Path("/add"));
			//在创建的文件夹下添加文件
			//fs.mkdirs(new Path("/add/wyg.txt"));
			
			
			// 2 获取文件详情
//			RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path("/"), true);
//				
//			while(listFiles.hasNext()){
    
    
//				LocatedFileStatus status = listFiles.next();
//					
//				// 输出详情
//				// 文件名称
//				System.out.println(status.getPath().getName());
//				// 长度
//				System.out.println(status.getLen());
//				// 权限
//				System.out.println(status.getPermission());
//				// 分组
//				System.out.println(status.getGroup());
//					
//				// 获取存储的块信息
//				BlockLocation[] blockLocations = status.getBlockLocations();
//					
//				for (BlockLocation blockLocation : blockLocations) {
    
    
//						
//					// 获取块存储的主机节点
//					String[] hosts = blockLocation.getHosts();
//						
//					for (String host : hosts) {
    
    
//						System.out.println(host);
//					}
//				}
//					
//				System.out.println("-----------已有的文件信息----------");
//			}

			
			//2 判断是文件还是文件夹
			FileStatus[] listStatus = fs.listStatus(new Path("/"));
				
			for (FileStatus fileStatus : listStatus) {
    
    
				
				// 如果是文件
				if (fileStatus.isFile()) {
    
    
						System.out.println("f:"+fileStatus.getPath().getName());
					}else {
    
    
						System.out.println("d:"+fileStatus.getPath().getName());
					}
				}
			
			fs.close();
		} catch (IOException e) {
    
    
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

}