二十三 HDP搭建大数据环境

关闭Selinux
vi /etc/selinux/config
SELINUX=disabled

关闭防火墙
systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld
systemctl status firewalld
firewall-cmd --state

现在的epel源
[root@localhost ~]# yum install epel-release -y

[root@localhost ~]# cat /etc/hostname
master
[root@localhost ~]# cat /etc/hostname
master

[root@localhost ~]# yum install ntp -y
[root@localhost ~]# systemctl enable ntpd.service
[root@localhost ~]# systemctl start ntpd.service
[root@localhost ~]# ntpdate 0.centos.pool.ntp.org
[root@localhost ~]# yum install httpd -y
[root@localhost ~]# reboot

安装httpd
[root@master ~]# yum install httpd -y
[root@master ~]# service httpd restart
[root@master ~]# chkconfig httpd on

安装JDK
先卸载系统中自带的JDK
[root@master ~]# rpm -qa | grep openjdk
java-1.7.0-openjdk-1.7.0.171-2.6.13.2.el7.x86_64
java-1.8.0-openjdk-1.8.0.161-2.b14.el7.x86_64
java-1.7.0-openjdk-headless-1.7.0.171-2.6.13.2.el7.x86_64
java-1.8.0-openjdk-headless-1.8.0.161-2.b14.el7.x86_64
[root@master ~]# yum remove java-1.7.0-openjdk-headless-1.7.0.171-2.6.13.2.el7.x86_64
[root@master ~]# yum remove java-1.8.0-openjdk-headless-1.8.0.161-2.b14.el7.x86_64 -y
[root@master ~]# tar -zxvf jdk-8u171-linux-x64.tar.gz -C /usr/local/

vi /etc/profile
export JAVA_HOME=/usr/local/jdk1.8.0_171
export PATH=.:$JAVA_HOME/bin:$PATH
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
[root@master ~]# source /etc/profile
[root@master ~]# java -version


SSH无密钥登录
[root@master ~]# ssh-keygen
[root@master ~]# ssh-copy-id [email protected]
[root@master ~]# sudo sh -c "echo umask 0022 >> /etc/profile"

设置系统运行参数
设置swappiness
[root@master ~]# echo 10 > /proc/sys/vm/swappiness
禁用 透明大页面压缩
[root@master ~]# echo never > /sys/kernel/mm/transparent_hugepage/defrag
[root@master ~]# echo never > /sys/kernel/mm/transparent_hugepage/enabled
[root@master ~]#

全部主机
修改配置文件/etc/sysctl.conf
kernel.shmmax= 1000000000
kernel.shmmni= 4096
kernel.shmall= 4000000000
kernel.sem= 250 512000 100 2048
kernel.sysrq= 1
kernel.core_uses_pid= 1
kernel.msgmnb= 65536
kernel.msgmax= 65536
kernel.msgmni= 2048
net.ipv4.tcp_syncookies= 0
net.ipv4.ip_forward= 0
net.ipv4.conf.default.accept_source_route= 0
net.ipv4.tcp_tw_recycle= 1
net.ipv4.tcp_max_syn_backlog= 200000
net.ipv4.conf.all.arp_filter= 1
net.ipv4.ip_local_port_range= 1281 65535
net.core.netdev_max_backlog= 200000
#java程序多时，设成2会导致很多服务不能去启动！
#vm.overcommit_memory= 2
vm.overcommit_memory=0
fs.nr_open= 3000000
kernel.threads-max= 798720
kernel.pid_max= 798720
#increase network
net.core.rmem_max=2097152
net.core.wmem_max=2097152

[root@master ~]# sysctl -p


修改/etc/security/limits.conf设置打开文件数限制
vim /etc/security/limits.conf
* soft nofile 2900000
* hard nofile 2900000
* soft nproc 131072
* hard nproc 131072

重新登录以使配置生效，或者使用命令临时设置
ulimit -n 10240


解压HDP3.0软件包到/var/www/html目录下
[root@master html]# tar -zxvf /root/ambari-2.7.1.0-centos7.tar.gz -C .
[root@master html]# tar -zxvf /root/HDP-3.0.1.0-centos7-rpm.tar.gz -C ./hdp/
[root@master html]# tar -zxvf /root/HDP-GPL-3.0.1.0-centos7-gpl.tar.gz -C ./hdp/
[root@master html]# tar -zxvf /root/HDP-UTILS-1.1.0.22-centos7.tar.gz -C ./hdp/

/etc/yum.repos.d配置文件
[root@master yum.repos.d]# cat ambari.repo
#VERSION_NUMBER=2.7.1.0-169
[ambari-2.7.1.0]
#json.url = http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json
name=ambari Version - ambari-2.7.1.0
baseurl=http://192.168.229.130/ambari/centos7/2.7.1.0-169/
gpgcheck=1
gpgkey=http://192.168.229.130/ambari/centos7/2.7.1.0-169/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

[root@master yum.repos.d]# cat hdp.repo
#VERSION_NUMBER=3.0.1.0-187
[HDP-3.0.1.0]
name=HDP Version - HDP-3.0.1.0
baseurl=http://192.168.229.130/hdp/HDP/centos7/3.0.1.0-187
gpgcheck=1
gpgkey=http://192.168.229.130/hdp/HDP/centos7/3.0.1.0-187/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1


[HDP-UTILS-1.1.0.22]
name=HDP-UTILS Version - HDP-UTILS-1.1.0.22
baseurl=http://192.168.229.130/hdp/HDP-UTILS/centos7/1.1.0.22
gpgcheck=1
gpgkey=http://192.168.229.130/hdp/HDP/centos7/3.0.1.0-187/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

#VERSION_NUMBER=3.0.1.0-187
[HDP-GPL-3.0.1.0]
name=HDP-GPL Version - HDP-GPL-3.0.1.0
baseurl=http://192.168.229.130/hdp/HDP-GPL/centos7/3.0.1.0-187
gpgcheck=1
gpgkey=http://192.168.229.130/hdp/HDP-GPL/centos7/3.0.1.0-187/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins
enabled=1
priority=1

[root@master yum.repos.d]#
使用刚才配置好的本地仓库，直接yum命令安装
[root@master ~]# yum install ambari-server -y

安装mysql
wget http://dev.mysql.com/get/mysql57-community-release-el7-7.noarch.rpm
rpm -ivh ./mysql57-community-release-el7-7.noarch.rpm
安装连接器yum install mysql-connector-java -y


rpm -ivh ./mysql-community-common-5.7.24-1.el7.x86_64.rpm
rpm -ivh ./mysql-community-libs-5.7.24-1.el7.x86_64.rpm
rpm -ivh ./mysql-community-libs-compat-5.7.24-1.el7.x86_64.rpm
rpm -ivh ./mysql-community-client-5.7.24-1.el7.x86_64.rpm
rpm -ivh ./mysql-community-server-5.7.24-1.el7.x86_64.rpm
rpm -ivh mysql-connector-java-5.1.25-3.el7.noarch.rpm


启动mysql,设置开机启动
[root@master ~]# service mysqld start
[root@master ~]# vi /etc/rc.local
#添加service mysqld start

查看mysql密码
[root@master ~]# grep "temporary password" /var/log/mysqld.log
2019-01-05T07:07:14.924796Z 1 [Note] A temporary password is generated for root@localhost: ,hbjy*P7aB<O
[root@master ~]# mysql -uroot -p
Enter password:,hbjy*P7aB<O
mysql> set global validate_password_policy=0;
mysql> set global validate_password_length=3;  #设置密码长度最小为6个字符
mysql> ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';
设置mysql远程链接
mysql> grant all privileges on *.* to 'root'@'%' identified by '123456';

登录mysql ,执行下面的语句，创建ambari数据库，为了在ambari-server安装后启动，否则报错
mysql> create database ambari;
mysql> use ambari;
mysql> CREATE USER 'ambari'@'%' IDENTIFIED BY '123456';
mysql> GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%';
mysql> CREATE USER 'ambari'@'localhost' IDENTIFIED BY '123456';
mysql> GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost';
mysql> CREATE USER 'ambari'@'master' IDENTIFIED BY '123456';
mysql> GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'master';
mysql> flush privileges;
mysql> source /var/lib/ambari-server/resources/Ambari-DDL-MySQL-CREATE.sql



配置 ambari server
[root@master ~]# ambari-server setup
Customize user account for ambari-server daemon [y/n] (n)? n
Checking JDK...
[2] Custom JDK
==============================================================================
Enter choice (1): 2
Path to JAVA_HOME: /usr/local/jdk1.8.0_171
Enable Ambari Server to download and install GPL Licensed LZO packages [y/n] (n)? y
Enter advanced database configuration [y/n] (n)? y
==============================================================================
Choose one of the following options:
[1] - PostgreSQL (Embedded)
[2] - Oracle
[3] - MySQL / MariaDB
[4] - PostgreSQL
[5] - Microsoft SQL Server (Tech Preview)
[6] - SQL Anywhere
[7] - BDB
==============================================================================
Enter choice (1): 3
Hostname (localhost): master
Port (3306):
Database name (ambari): ambari
Username (ambari): root                        #采用root账号超级权限
Enter Database Password (bigdata): 123456      #采用root账号超级权限
Re-enter password: 123456                      #采用root账号超级权限
Should ambari use existing default jdbc /usr/share/java/mysql-connector-java.jar [y/n] (y)? y
Proceed with configuring remote database connection properties [y/n] (y)? y

Adjusting ambari - server permissions and ownership...
Ambari Server 'setup' completed successfully.
[root@master ~]#

命令行方式设置mysql数据库链接库，用于oozie和range链接mysql时使用（如果上述操作不起作用，则需要采用下面操作）
[root@master ~]# ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar
Using python  /usr/bin/python
Setup ambari-server
Copying /usr/share/java/mysql-connector-java.jar to /var/lib/ambari-server/resources/mysql-connector-java.jar
If you are updating existing jdbc driver jar for mysql with mysql-connector-java.jar. Please remove the old driver jar, from all hosts. Restarting services that need the driver, will automatically copy the new jar to the hosts.
JDBC driver was successfully initialized.
Ambari Server 'setup' completed successfully.
[root@master ~]#

配置ambari-server自启动，启动ambari-server
[root@master ~]# systemctl enable ambari-server
[root@master ~]# systemctl start ambari-server


浏览器访问ambari-server服务
http://192.168.229.130:8080     账号admin密码admin
登录之后不要动其它的，先安装ambari-agent，否则后续会出现问题
所有主机节点，安装ambari-agent,并配置自动启动
[root@master ~]# yum install ambari-agent -y
[root@master ~]# systemctl enable ambari-agent
[root@master ~]# systemctl restart ambari-agent && systemctl status ambari-agent


如果出现问题，应该是hive和oozie数据库没有创建，则继续安装
创建oozie库
mysql> create database oozie  DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
mysql> create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

参考文档

https://www.cnblogs.com/dajianshi/p/9473650.html