1、创建Maven项目,导入以下依赖,必须:
<dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <version>2.6.0</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> <version>2.6.0</version> </dependency>
2、获取客户端配置文件hdfs-site.xml及core-site.xml
文件目录:
3、测试程序
public static void test1(String user, String keytab, String dir) throws Exception { Configuration conf = new Configuration(); conf.addResource(new Path("D:/cdh_10/hdfs-site.xml")); conf.addResource(new Path("D:/cdh_10/core-site.xml")); System.setProperty("java.security.krb5.conf", "D:/cdh_10/krb5.conf"); UserGroupInformation.setConfiguration(conf); UserGroupInformation.loginUserFromKeytab(user, keytab); listDir(conf, dir); } public static void listDir(Configuration conf, String dir) throws IOException { FileSystem fs = FileSystem.get(conf); FileStatus files[] = fs.listStatus(new Path(dir)); for (FileStatus file : files) { System.out.println(file.getPath()); } } public static void main(String[] args) { String user = "xxx"; String keytab = "D:/cdh_10/xxx.keytab"; String dir = "hdfs://ns1/data"; try { test1(user, keytab, dir); } catch (Exception e) { e.printStackTrace(); } }
可能遇到的异常:
1、缺少kerberos配置文件设置时,如下
System.setProperty("java.security.krb5.conf", "D:/cdh_10/krb5.conf");
抛出异常:
java.lang.IllegalArgumentException: Can't get Kerberos realm at org.apache.hadoop.security.HadoopKerberosName.setConfiguration(HadoopKerberosName.java:65) at org.apache.hadoop.security.UserGroupInformation.initialize(UserGroupInformation.java:263) at org.apache.hadoop.security.UserGroupInformation.setConfiguration(UserGroupInformation.java:299) at com.xxx.hdfs.CdhHdfsOpt.test1(CdhHdfsOpt.java:122) at com.xxx.hdfs.CdhHdfsOpt.main(CdhHdfsOpt.java:141) Caused by: java.lang.reflect.InvocationTargetException at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.hadoop.security.authentication.util.KerberosUtil.getDefaultRealm(KerberosUtil.java:84) at org.apache.hadoop.security.HadoopKerberosName.setConfiguration(HadoopKerberosName.java:63) ... 4 more Caused by: KrbException: Cannot locate default realm at sun.security.krb5.Config.getDefaultRealm(Config.java:1029) ... 10 more