Hadoop常用命令
- Archive
1.如何创建档案
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo/bar
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo/bar/a
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo/bar/a/b
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo/bar/a/b/c
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo/bar/e
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo/bar/e/f
[root@hadoop2 tmp]# hdfs dfs -mkdir /foo/bar/e/f/g
[root@hadoop2 tmp]# hdfs dfs -ls /foo/bar/
Found 2 items
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 /foo/bar/a
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 /foo/bar/e
hadoop archive -archiveName zoo.har -p /foo/bar -r 3 /outputdir
[root@hadoop2 tmp]# hdfs dfs -ls /outputdir
Found 1 items
drwxr-xr-x - root supergroup 0 2019-04-06 21:52 /outputdir/zoo.har
2.如何在档案中查找文件
[root@hadoop2 tmp]# hdfs dfs -ls har:///outputdir/zoo.har
Found 2 items
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/a
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/e
[root@hadoop2 tmp]# hdfs dfs -ls -R har:///outputdir/zoo.har
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/a
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/a/b
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/a/b/c
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/e
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/e/f
drwxr-xr-x - root supergroup 0 2019-04-06 21:50 har:///outputdir/zoo.har/e/f/g
[root@hadoop2 tmp]# hdfs dfs -ls /outputdir/zoo.har
Found 4 items
-rw-r--r-- 3 root supergroup 0 2019-04-06 21:52 /outputdir/zoo.har/_SUCCESS
-rw-r--r-- 3 root supergroup 371 2019-04-06 21:52 /outputdir/zoo.har/_index
-rw-r--r-- 3 root supergroup 23 2019-04-06 21:52 /outputdir/zoo.har/_masterindex
-rw-r--r-- 3 root supergroup 0 2019-04-06 21:52 /outputdir/zoo.har/part-0
- checknative
见我的另一篇文章：https://mp.csdn.net/mdeditor/89314448#
3.conftest
验证配置XML文件的有效性
[root@hadoop2 hadoop]# hadoop conftest -conffile /opt/software/hadoop-3.2.0/etc/hadoop/core-site.xml
/opt/software/hadoop-3.2.0/etc/hadoop/core-site.xml: valid
OK
4.Distcp方式
见我的另一篇文章：https://blog.csdn.net/weixin_42868638/article/details/89320418
5.jar
运行一个jar文件。使用yarn jar 来代替启动YARN应用程序。
hadoop jar <jar在linux的路径> <main方法所在的类的全类名> <参数>
hadoop jar /root/wc1.jar cn.itcast.d3.hadoop.mr.WordCount hdfs://itcast:9000/words /out2
6.jnipath
[root@hadoop2 hadoop-3.2.0]# hadoop jnipath
/opt/software/hadoop-3.2.0/lib/native
7.version
[root@hadoop2 hadoop-3.2.0]# hadoop version
Hadoop 3.2.0
Source code repository https://github.com/apache/hadoop.git -r e97acb3bd8f3befd27418996fa5d4b50bf2e17bf
Compiled by sunilg on 2019-01-08T06:08Z
Compiled with protoc 2.5.0
From source with checksum d3f0795ed0d9dc378e2c785d3668f39
This command was run using /opt/software/hadoop-3.2.0/share/hadoop/common/hadoop-common-3.2.0.jar
8.envvars
[root@hadoop2 hadoop-3.2.0]# hadoop envvars
JAVA_HOME='/usr/java/jdk1.8.0_92'
HADOOP_COMMON_HOME='/opt/software/hadoop-3.2.0'
HADOOP_COMMON_DIR='share/hadoop/common'
HADOOP_COMMON_LIB_JARS_DIR='share/hadoop/common/lib'
HADOOP_COMMON_LIB_NATIVE_DIR='lib/native'
HADOOP_CONF_DIR='/opt/software/hadoop-3.2.0/etc/hadoop'
HADOOP_TOOLS_HOME='/opt/software/hadoop-3.2.0'
HADOOP_TOOLS_DIR='share/hadoop/tools'
HADOOP_TOOLS_LIB_JARS_DIR='share/hadoop/tools/lib'
9.daemonlog
见https://blog.csdn.net/weixin_42868638/article/details/89328051