超详细大数据学习之Hadoop伪分布式安装

1、用rpm安装jdk

rpm -i jdk-7u67-linux-x64.rpm

[root@node01 ~]# ll
total 123904
-rw-------. 1 root root       900 Jun  2 05:38 anaconda-ks.cfg
-rw-r--r--. 1 root root      8815 Jun  2 05:38 install.log
-rw-r--r--. 1 root root      3384 Jun  2 05:37 install.log.syslog
-rw-r--r--  1 root root 126857158 Jun 17 07:18 jdk-7u67-linux-x64.rpm
[root@node01 ~]# rpm -i jdk-7u67-linux-x64.rpm
Unpacking JAR files...
	rt.jar...
	jsse.jar...
	charsets.jar...
	tools.jar...
	localedata.jar...
	jfxrt.jar...

whereis java

[root@node01 ~]# whereis java
java: /usr/bin/java

vi + /etc/profile

在最后加上

export JAVA_HOME=/usr/bin/java
export PATH=$PATH:$JAVA_HOME/bin
-- INSERT --

source /etc/profile

vi + /etc/profile

export JAVA_HOME=/usr/bin/java
export PATH=$PATH:/usr/java/jdk1.7.0_67/bin
-- INSERT --

source /etc/profile

ll -a

[root@node01 ~]# ll -a
total 123940
dr-xr-x---.  3 root root      4096 Jun 17 07:17 .
dr-xr-xr-x. 22 root root      4096 Jun 17 06:58 ..
-rw-------.  1 root root       900 Jun  2 05:38 anaconda-ks.cfg
-rw-------.  1 root root       850 Jun 17 06:56 .bash_history
-rw-r--r--.  1 root root        18 May 20  2009 .bash_logout
-rw-r--r--.  1 root root       176 May 20  2009 .bash_profile
-rw-r--r--.  1 root root       176 Sep 23  2004 .bashrc
-rw-r--r--.  1 root root       100 Sep 23  2004 .cshrc
-rw-r--r--.  1 root root      8815 Jun  2 05:38 install.log
-rw-r--r--.  1 root root      3384 Jun  2 05:37 install.log.syslog
-rw-r--r--   1 root root 126857158 Jun 17 07:18 jdk-7u67-linux-x64.rpm
drwx------   2 root root      4096 Jun 17 07:01 .ssh
-rw-r--r--.  1 root root       129 Dec  4  2004 .tcshrc

若没有

ssh localhost

[root@node01 ~]# ssh localhost
The authenticity of host 'localhost (::1)' can't be established.
RSA key fingerprint is ca:52:70:e9:6e:d9:7b:0c:dd:25:ac:01:ab:ba:dc:ec.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'localhost' (RSA) to the list of known hosts.
root@localhost's password: 
Last login: Wed Jun 17 07:08:19 2020 from 192.168.19.1
[root@node01 ~]# exit
logout
Connection to localhost closed.

ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa

[root@node01 ~]# ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
Generating public/private dsa key pair.
Your identification has been saved in /root/.ssh/id_dsa.
Your public key has been saved in /root/.ssh/id_dsa.pub.
The key fingerprint is:
06:eb:92:67:a4:3a:f5:11:82:33:9c:7f:84:cf:01:45 root@node01
The key's randomart image is:
+--[ DSA 1024]----+
|     oE          |
|    .            |
| . o o.          |
|  * o +o         |
|   + =ooS        |
|    o==.         |
|   .+o+.         |
|  .. +.          |
|  ..             |
+-----------------+

cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

[root@node01 ~]# cd .ssh
[root@node01 .ssh]# ll
total 16
-rw-r--r-- 1 root root 601 Jun 17 07:55 authorized_keys
-rw------- 1 root root 672 Jun 17 07:52 id_dsa
-rw-r--r-- 1 root root 601 Jun 17 07:52 id_dsa.pub
-rw-r--r-- 1 root root 793 Jun 17 07:36 known_hosts

[root@node01 .ssh]# ssh localhost
Last login: Wed Jun 17 07:36:41 2020 from localhost
[root@node01 ~]# exit
logout
Connection to localhost closed.

[root@node01 .ssh]# ssh node01
The authenticity of host 'node01 (192.168.19.31)' can't be established.
RSA key fingerprint is ca:52:70:e9:6e:d9:7b:0c:dd:25:ac:01:ab:ba:dc:ec.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'node01,192.168.19.31' (RSA) to the list of known hosts.
Last login: Wed Jun 17 07:56:56 2020 from localhost
[root@node01 ~]# exit
logout
Connection to node01 closed.

mkdir ll

[root@node01 ~]# ll
total 123908
-rw-------. 1 root root       900 Jun  2 05:38 anaconda-ks.cfg
-rw-r--r--. 1 root root      8815 Jun  2 05:38 install.log
-rw-r--r--. 1 root root      3384 Jun  2 05:37 install.log.syslog
-rw-r--r--  1 root root 126857158 Jun 17 07:18 jdk-7u67-linux-x64.rpm
drwxr-xr-x  2 root root      4096 Jun 17 08:04 ll

[root@node01 ~]# cd /opt/
[root@node01 opt]# ll
total 4
drwxr-xr-x 2 root root 4096 Jun 17 08:02 ll
[root@node01 opt]# cd ll/
[root@node01 ll]# tar xf hadoop-2.6.5.tar.gz -C /opt/ll
[root@node01 ll]# ll
total 179296
drwxr-xr-x 9 root root      4096 May 24  2017 hadoop-2.6.5
-rw-r--r-- 1 root root 183594876 Jun 17 08:02 hadoop-2.6.5.tar.gz

vi + /etc/profile

export JAVA_HOME=/usr/bin/java
export HADOOP_HOME=/opt/ll/hadoop-2.6.5
export PATH=$PATH:/usr/java/jdk1.7.0_67/bin

cd /opt/ll/hadoop-2.6.5/etc/hadoop
vi hadoop-env.sh

[root@node01 ~]# cd /opt/ll/hadoop-2.6.5/etc/hadoop
[root@node01 hadoop]# vi hadoop-env.sh
[root@node01 hadoop]# vi hadoop-env.sh
[root@node01 hadoop]# vi mapred-env.sh
[root@node01 hadoop]# vi yarn-env.sh

给这三个文件里的JAVA_HOME都改成绝对路径/usr/java/jdk1.7.0_67

修改hadoop配置文件信息

vi core-site.xml

 <configuration>

 	<property>
        <name>fs.defaultFS</name>
        <value>hdfs://node01:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/var/ll/hadoop/pseudo</value>
    </property>

</configuration>

vi hdfs-site.xml

<configuration>

    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>node01:50090</value>
    </property>

</configuration>

vi slaves 

修改为node01

node01

4.格式化hdfs
hdfs namenode -format (只能格式化一次，再次启动集群不要执行，否则clusterID变了)
之前/var/ldy/hadoop/pseudo这个文件不存在，格式化后就存在了，检查一下看存在没

hdfs namenode -format

出现以下界面

[root@node01 hadoop]# cd /var/ll/hadoop/pseudo
[root@node01 pseudo]# ll
total 4
drwxr-xr-x 3 root root 4096 Jun 17 17:36 dfs
[root@node01 pseudo]# cd dfs/
[root@node01 dfs]# ll
total 4
drwxr-xr-x 3 root root 4096 Jun 17 17:36 name
[root@node01 dfs]# cd name/
[root@node01 name]# ll
total 4
drwxr-xr-x 2 root root 4096 Jun 17 17:36 current
[root@node01 name]# cd current
[root@node01 current]# ll
total 16
-rw-r--r-- 1 root root 321 Jun 17 17:36 fsimage_0000000000000000000
-rw-r--r-- 1 root root  62 Jun 17 17:36 fsimage_0000000000000000000.md5
-rw-r--r-- 1 root root   2 Jun 17 17:36 seen_txid
-rw-r--r-- 1 root root 205 Jun 17 17:36 VERSION

5.启动集群
start-dfs.sh

start-dfs.sh

[root@node01 current]# cd
[root@node01 ~]# start-dfs.sh
Starting namenodes on [node01]
node01: starting namenode, logging to /opt/ll/hadoop-2.6.5/logs/hadoop-root-namenode-node01.out
node01: starting datanode, logging to /opt/ll/hadoop-2.6.5/logs/hadoop-root-datanode-node01.out
Starting secondary namenodes [node01]
node01: starting secondarynamenode, logging to /opt/ll/hadoop-2.6.5/logs/hadoop-root-secondarynamen
ode-node01.out

6. 在浏览器里打开node03:50070（别用360浏览器！）
   
   7.创建目录：hdfs dfs -mkdir -p /user/root

hdfs dfs -mkdir -p /user/root

8. 上传文件： hdfs dfs -put 500miles.txt /user/root

扫描二维码关注公众号，回复： 11312905 查看本文章

hdfs dfs -put  500miles.txt   /user/root

刷新界面，点击root

9. Hadoop的log在哪里查

[root@node01 ~]# hdfs dfs -mkdir -p /user/root
[root@node01 ~]# hdfs dfs -put 500miles.txt /user/root
[root@node01 ~]# cd /opt/ll/hadoop-2.6.5
[root@node01 hadoop-2.6.5]# ll
total 136
drwxr-xr-x 2 root root  4096 May 24  2017 bin
drwxr-xr-x 3 root root  4096 May 24  2017 etc
drwxr-xr-x 2 root root  4096 May 24  2017 include
drwxr-xr-x 3 root root  4096 May 24  2017 lib
drwxr-xr-x 2 root root  4096 May 24  2017 libexec
-rw-r--r-- 1 root root 84853 May 24  2017 LICENSE.txt
drwxr-xr-x 2 root root  4096 Jun 17 18:00 logs
-rw-r--r-- 1 root root 14978 May 24  2017 NOTICE.txt
-rw-r--r-- 1 root root  1366 May 24  2017 README.txt
drwxr-xr-x 2 root root  4096 May 24  2017 sbin
drwxr-xr-x 4 root root  4096 May 24  2017 share

10. 停止集群：stop-dfs.sh

stop-dfs.sh

用jps检查一下

[root@node01 hadoop-2.6.5]# stop-dfs.sh
Stopping namenodes on [node01]
node01: stopping namenode
node01: stopping datanode
Stopping secondary namenodes [node01]
node01: stopping secondarynamenode
[root@node01 hadoop-2.6.5]# jps
2048 Jps