hdfs文件上传下载--client模式--Kerberos认证

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;

public class FileSystemConnection {
    private static String HDFS_URI = "hdfs://10.46.0.213:8020";
    private static String KERBEROS_PRINC = "admin@TDH";
    private static String KERBEROS_PRINC_KEYTAB_PATH =
            "D:\\Repo\\codeRepo\\git\\data-service\\data-service-service-impl\\src\\main\\resources\\admin.keytab";
    public static final Logger logger = LoggerFactory.getLogger(FileSystemConnection.class);

    public static FileSystem getHdfsConnection() {
        try {
            Configuration conf = new Configuration();
            conf.set("fs.defaultFS", HDFS_URI);
            conf.set("hadoop.security.authentication", "kerberos");
            conf.set("hadoop.security.authorization", "true");
            conf.set("dfs.datanode.address","0.0.0.0:50010");
//            conf.set("dfs.datanode.ipc.address","0.0.0.0:50020");
            // 没开kerberos,注释下面两行
            logger.info("kerberos principal: {} ,keytab path: {}", KERBEROS_PRINC, KERBEROS_PRINC_KEYTAB_PATH);
            UserGroupInformation.setConfiguration(conf);
            UserGroupInformation.loginUserFromKeytab(KERBEROS_PRINC, KERBEROS_PRINC_KEYTAB_PATH);
            FileSystem fs = FileSystem.get(conf);
            return fs;
        } catch (Exception e) {
            throw new RuntimeException("get hdfs connection failure", e);
        }
    }

    private static void downLoadFile() throws IOException {
     FileSystem dfs=getHdfsConnection(); String src="hdfs://10.46.0.213:8020/tmp/tdt"; String dest="D:\\TestFs"; Path path = new Path(src); try { if (dfs.exists(path)) { File file = new File(dest + File.separator + path.getName()); file.createNewFile(); InputStream in = dfs.open(path); OutputStream out = new FileOutputStream(file); IOUtils.copyBytes(in, out, conf); in.close(); out.close(); } } catch (Exception e) { e.printStackTrace(); } } }

猜你喜欢

转载自www.cnblogs.com/jvStarBlog/p/11416096.html
今日推荐