import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.security.UserGroupInformation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; public class FileSystemConnection { private static String HDFS_URI = "hdfs://10.46.0.213:8020"; private static String KERBEROS_PRINC = "admin@TDH"; private static String KERBEROS_PRINC_KEYTAB_PATH = "D:\\Repo\\codeRepo\\git\\data-service\\data-service-service-impl\\src\\main\\resources\\admin.keytab"; public static final Logger logger = LoggerFactory.getLogger(FileSystemConnection.class); public static FileSystem getHdfsConnection() { try { Configuration conf = new Configuration(); conf.set("fs.defaultFS", HDFS_URI); conf.set("hadoop.security.authentication", "kerberos"); conf.set("hadoop.security.authorization", "true"); conf.set("dfs.datanode.address","0.0.0.0:50010"); // conf.set("dfs.datanode.ipc.address","0.0.0.0:50020"); // 没开kerberos,注释下面两行 logger.info("kerberos principal: {} ,keytab path: {}", KERBEROS_PRINC, KERBEROS_PRINC_KEYTAB_PATH); UserGroupInformation.setConfiguration(conf); UserGroupInformation.loginUserFromKeytab(KERBEROS_PRINC, KERBEROS_PRINC_KEYTAB_PATH); FileSystem fs = FileSystem.get(conf); return fs; } catch (Exception e) { throw new RuntimeException("get hdfs connection failure", e); } } private static void downLoadFile() throws IOException {
FileSystem dfs=getHdfsConnection(); String src="hdfs://10.46.0.213:8020/tmp/tdt"; String dest="D:\\TestFs"; Path path = new Path(src); try { if (dfs.exists(path)) { File file = new File(dest + File.separator + path.getName()); file.createNewFile(); InputStream in = dfs.open(path); OutputStream out = new FileOutputStream(file); IOUtils.copyBytes(in, out, conf); in.close(); out.close(); } } catch (Exception e) { e.printStackTrace(); } } }
hdfs文件上传下载--client模式--Kerberos认证
猜你喜欢
转载自www.cnblogs.com/jvStarBlog/p/11416096.html
今日推荐
周排行