package project.etl.core.util;
import java.io.FileNotFoundException;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
/ **
* HDFS operations tools
* @author Snow
* @date: 2019-10-13 10:01:59
* /
public class HDFSUtil {
/ **
* Gets the default file system
* @return
* @throws IOException
* @author Snow
* @date: 2019-10-13 10:16:18
* /
public static getFileSystem the FileSystem () throws IOException {
the Configuration new new = the conf the Configuration ();
the FileSystem FileSystem = FileSystem.get (the conf);
return FileSystem;
}
/ * *
* authentication is required to obtain the file system
* @param user username
* @param keytab verify the file path
* @return the fileSystem
* @throws IOException
* @author Snow
* @date: 2019-10-13 10:14:06
*/
public static FileSystem getFileSystem(String user,String keytab) throws IOException {
Configuration conf = new Configuration();
UserGroupInformation.loginUserFromKeytab(user, keytab);
FileSystem fileSystem = FileSystem.get(conf);
return fileSystem;
}
/**
* 向文件系统指定路径上传单个文件或文件夹
* @param fileSystem
* @param HDFSPath
* @param localPath
* @throws IOException
* @author snow
* @date: 2019-10-13 10:27:21
*/
public static void uploadFileToHDFS(FileSystem fileSystem,Path HDFSPath,Path localPath) throws IOException {
{IF (fileSystem.exists (HDFSPath)!)
boolean BOOL = fileSystem.mkdirs (HDFSPath);
IF (BOOL) {
System.out.println ( "Creating folders success!");
} the else {
System.out.println ( "create folder failed!");
}
}
fileSystem.copyFromLocalFile (localPath, HDFSPath);
}
/ **
* Specifies the path of a single file or download a file system folder
* @param the fileSystem
* @param HDFSPath
* @param localPath
* @ Snow author
* @throws IOException
* @date: 2019-10-13 11:02:54
* /
public static void loadHDFSFile (the FileSystem FileSystem, the Path HDFSPath, the Path the localPath) throws IOException {
fileSystem.copyToLocalFile (HDFSPath, the localPath);
}
/ **
* delete a file system files or folders
* @param FileSystem
* @param HDFSPath
* @throws IOException
* @author Snow
* @date: 2019-10-13 11:33: 22
* /
public static void delHDFSFile (the FileSystem the FileSystem, Path HDFSPath) throws IOException {
IF (fileSystem.exists (HDFSPath)) {
// prohibit recursively delete files
fileSystem.delete (HDFSPath, false);
}
}
/ **
* View files files and directories under the specified path system
* @param the FileSystem
* @param HDFSPath
* @throws FileNotFoundException
* @throws IOException
* @author Snow
* @date: 2019-10-13 12:03:12
*/
public static void checkHDFSFile(FileSystem fileSystem,Path HDFSPath) throws FileNotFoundException, IOException {
FileStatus[] fileStatuses = fileSystem.listStatus(HDFSPath);
for (FileStatus fileStatus : fileStatuses) {
String isDir = fileStatus.isDirectory() ? "文件夹：:" : "文件：";
short replication = fileStatus.getReplication();
String path = fileStatus.getPath().toString();
System.out.println(isDir + "\t" + replication + "\t" + path);
}
}
/**
* 关闭连接
* @param fileSystem
* @throws IOException
* @author snow
* @date: 2019-10-13 12:05:44
*/
public static void close(FileSystem fileSystem) throws IOException {
fileSystem.close();
}
}