安装CDH5.7.1集群

Hadoop版本选择

目前Hadoop比较流行的主要有2个版本，Apache和Cloudera版本。
Apache Hadoop：维护人员比较多，更新频率比较快，但是稳定性比较差。
Cloudera Hadoop（CDH）：CDH：Cloudera公司的发行版本，基于ApacheHadoop的二次开发，优化了组件兼容和交互接口、简化安装配置、增加Cloudera兼容特性。

准备工作

以下步骤要在所有节点上执行

1. 修改hostname

vi /etc/sysconfig/network

Hostname=masternode1

修改hosts文件

2. 把所有要添加到集群中的主机都要加入hosts中，格式如下：

扫描二维码关注公众号，回复： 1703789 查看本文章

[root@masternode1 centos]# vi /etc/hosts

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4

::1 localhost localhost.localdomain localhost6 localhost6.localdomain6

10.10.10.11 masternode2

10.10.10.12 slavenode1

10.10.10.13 masternode1

10.10.10.14 slavenode2

10.10.10.15 slavenode3

3. 关闭防火墙

sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config;

chkconfig iptables off ;service iptables stop;chkconfig iptables --list

4. Ssh免密码登陆配置

[root@masternode1 centos]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

[root@masternode1 centos]# for i in {11,12,13,14,15}; do ssh-copy-id

[email protected].$i; done

5. 修改时区

[hadoop@masternode1 hadoop]# cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime

cp: overwrite `/etc/localtime'? y

[hadoop@masternode1 hadoop]# vi /etc/sysconfig/clock

ZONE="Asia/Shanghai"

UTC=false

ARC=false

[root@masternode1 centos]# for i in {11,12,13,14,15}; do scp /etc/sysconfig/clock [email protected].$i:/etc/sysconfig/clock; done

6. NTP服务器时间同步

⑴ vi /etc/ntp.conf

restrict 127.0.0.1

restrict -6 ::1

restrict 10.10.10.13 mask 255.255.255.0 nomodify notrap

server 127.127.1.0 # local clock

fudge 127.127.1.0 stratum 10

⑵ 启动ntp服务
#/etc/init.d/ntpd start
添加到自启动
执行如下命令，让每次系统启动的时候自动启动ntp服务：
chkconfig ntpd on

⑶客户端设置

[root@masternode2 centos]#vi /etc/ntp.conf

driftfile /var/lib/ntp/drift

server 192.168.237.230

restrict 192.168.237.230 nomodify notrap noquery

server 127.127.1.0 # local clock

fudge 127.127.1.0 stratum 10

[root@masternode2 centos]# for i in {12,14,15};do scp /etc/ntp.conf [email protected].$i:/etc/ntp.conf; done

ntp.conf 100% 1911 1.9KB/s 00:00

7. 安装java

[root@masternode1 centos]# mkdir /usr/java

[root@masternode1 centos]# cd /opt/hadoop/

[root@masternode1 hadoop]# tar xvf jdk-7u79-linux-x64.tar.gz -C /usr/java

8. Java安装包拷贝到其他机器上

[root@masternode1 hadoop]# for i in {11,12,13,14,15}; do scp -r /usr/java/ [email protected].$i:/usr;done

9. Mysql安装

①　卸载系统自带的mysql相关安装包，仅卸载mysql开头的包

[root@masternode1 mysql-5.6.25]# rpm -qa|grep mysql

mysql-libs-5.1.73-3.el6_5.x86_64

[root@masternode1 mysql-5.6.25]# rpm -e --nodeps mysql-libs-5.1.73-3.el6_5.x86_64

②　安装依赖包

[root@masternode1 mysql-5.6.25]# yum install gcc gcc-c++ ncurses-devel -y

③　安装cmake

下载cmake安装包

[rootp@masternode1 cmake-2.8.12]$ wget http://www.cmake.org/files/v2.8/cmake-2.8.12.tar.gz

[root@masternode1 cmake-2.8.12]$ tar zxvf cmake-2.8.12.tar.gz

[root@masternode1 cmake-2.8.12]$ cd cmake-2.8.12 && ./bootstrap && make && make install

④　创建mysql用户的相应目录

[root@masternode1 ~]# groupadd mysql;useradd -g mysql mysql

[root@masternode1 ~]# mkdir -p /opt/mysql

[root@masternode1 ~]# mkdir -p /opt/mysql/data

[root@masternode1 ~]# mkdir -p /opt/mysql/log

⑤　获取MySQL安装包并安装

wget http://dev.mysql.com/get/downloads/mysql/mysql-5.6.25.tar.gz

[hadoop@masternode1 hadoop]]$ tar zxvf mysql-5.6.25.tar.gz

[hadoop@masternode1 hadoop]]$ cd mysql-5.6.25 && mkdir data

[hadoop@masternode1 mysql-5.6.25]$ cmake -DCMAKE_INSTALL_PREFIX=/opt/hadoop/mysql-5.6.25 -DMYSQL_UNIX_ADDR=/opt/hadoop/mysql-5.6.25/mysql.sock -DDEFAULT_CHARSET=utf8 -DDEFAULT_COLLATION=utf8_general_ci -DWITH_INNOBASE_STORAGE_ENGINE=1 -DWITH_ARCHIVE_STORAGE_ENGINE=1 -DWITH_BLACKHOLE_STORAGE_ENGINE=1 -DMYSQL_DATADIR=/opt/hadoop/mysql-5.6.25/data/ -DMYSQL_TCP_PORT=3306 -DENABLE_DOWNLOADS=1-DENABLE_DOWNLOADS=1

若提示cmake命名不能运行，之后执行如下即可

export PATH=/opt/hadoop/cmake-2.8.12/bin/:$PATH

[hadoop@masternode1e mysql-5.6.25]# make && make install

注意

DMYSQL_UNIX_ADDR=/opt/hadoop/mysql-5.6.25/mysql.sock 必须目录有写的权限，chmod 756 /opt/hadoop/mysql-5.6.25/

或者-DMYSQL_UNIX_ADDR=/tmp/mysql.sock 只需/tmp有写的权限即可 chmod 756 /tmp

Chown mysql:mysql /tmp 因此DMYSQL_UNIX_ADDR最好配置/tmp/mysql.scok

⑥　创建软连接

[hadoop@masternode1 ~]# ln -s /opt/hadoop/mysql-5.6.25/libmysql/libmysqlclient.so.18 /usr/lib/libmysqlclient.so.18

[hadoop@masternode1 ~]# chown hadoop:hadoop /usr/lib/libmysqlclient.so.18

⑦　修改配置文件

[hadoop@masternode1 mysql-5.6.25]# cp support-files/my-default.cnf /etc/my.cnf

[hadoop@masternode1 mysql-5.6.25]# chown hadoop:hadoop /etc/my.cnf

[hadoop@masternode1 mysql-5.6.25]$ vi /etc/my.cnf

datadir=/opt/hadoop/mysql-5.6.25/data

log-error=/opt/hadoop/mysql-5.6.25/mysql_error.log

pid-file=/opt/hadoop/mysql-5.6.25/data/mysql.pid

socket=/opt/hadoop/mysql-5.6.25/mysql.sock

user=mysql

⑧　创建启动文件

[hadoop@masternode1 ~]# cp /opt/hadoop/mysql-5.6.25/support-files/mysql.server /etc/init.d/mysqld

[hadoop@masternode1 ~]# chown hadoop:hadoop /etc/init.d/mysqld ;chmod 756 /etc/init.d/mysqld

[hadoop@slavenode1 mysql-5.6.25]# mysqld -uroot -h127.0.0.1 -p

bash: mysqld: command not found

解决方法如下：

ln -s /opt/hadoop/mysql-5.6.25/bin/mysql /usr/bin/mysql

⑨　初始化数据库

[hadoop@masternode1 ~]$ chmod 756 /opt/hadoop/mysql-5.6.25/scripts/mysql_install_db

[[[email protected]]$/opt/hadoop/mysql-5.6.25/scripts/mysql_install_db --user=mysql --basedir=/opt/hadoop/mysql-5.6.25 --datadir=/opt/hadoop/mysql-5.6.25/data/ --defaults-file=/etc/my.cnf

[root@masternode1 mysql-5.6.25]# chmod +x /etc/init.d/mysqld

[root@masternode1 mysql-5.6.25]# chkconfig mysqld on

⑩　启动MySQL服务

[hadoop@masternode1 mysql-5.6.25]# service mysqld start

Starting MySQL. SUCCESS!

chkconfig --level 2345 mysqld on

10. 创建所需的数据库

-hive数据库,为什么使用latin1，后面安装异常会做说明

create database hive DEFAULT CHARSET latin1;

--集群监控数据库

--hue数据库

mysql> create database hue DEFAULT CHARSET utf8;

Query OK, 1 row affected (0.00 sec)

mysql> create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

mysql> create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

Query OK, 1 row affected (0.00 sec)

--oozie数据库

mysql> create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

Query OK, 1 row affected (0.00 sec)

创建用户

mysql> CREATE USER 'hive' IDENTIFIED BY 'hive';