环境信息
如果是单节点的，建议将虚拟机的内存最小设置为8G，硬盘大小为100G
| 类别 | 版本 | 备注 |
|---|---|---|
| CDH | 6.2.0 | 下载网址 |
| 数据库 | mysql 5.7 | |
| 系统 | CentOS7 | |
| jdk | 1.8 | cdh 修改过的 |
| parcels | 下载网址 |
CDH6.2.0中的软件版本信息
| Component | Component Version |
|---|---|
| Apache Avro | 1.8.2 |
| Apache Flume | 1.9.0 |
| Apache Hadoop | 3.0.0 |
| Apache HBase | 2.1.2 |
| HBase Indexer | 1.5 |
| Apache Hive | 2.1.1 |
| Hue | 4.3.0 |
| Apache Impala | 3.2.0 |
| Apache Kafka | 2.1.0 |
| Kite SDK | 1.0.0 |
| Apache Kudu | 1.9.0 |
| Apache Solr | 7.4.0 |
| Apache Oozie | 5.1.0 |
| Apache Parquet | 1.9.0 |
| Parquet-format | 2.3.1 |
| Apache Pig | 0.17.0 |
| Apache Sentry | 2.1.0 |
| Apache Spark | 2.4.0 |
| Apache Sqoop | 1.4.7 |
| Apache ZooKeeper | 3.4.5 |
基础环境
修改 HOST 文件
192.168.200.129 ifan scm bogon
关闭防火墙
# 临时关闭防火墙
systemctl stop firewalld.service
# 开机不启动
systemctl disable firewalld.service
ssh 免登录
ssh-keygen -t rsa
ssh-copy-id localhost
关闭 SELinux
什么是 SELinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config
setenforce 0
时钟同步
yum install ntp
crontab -e
# 添加定时任务
* * * * * /usr/sbin/ntpdate time1.aliyun.com 2>&1
禁用透明大页面压缩
CDH 配置需要
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
优化交换分区
echo "vm.swappiness = 10" >> /etc/sysctl.conf
sysctl -p
安装CDH
基础包准备
从官方网站下载这些软件包
cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm
cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm
cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm
oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
RPM-GPG-KEY-cloudera
cloudera-manager.repo
安装 mysql
安装
创建用户和数据库
# 创建 scm 数据库 用户名和密码为 scm/scm
CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY 'scm';
# 创建 amon 数据库 并设置用户名密码为 amon/amon
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY 'amon';
# 创建 rman 数据库 并设置用户名密码为 rman/rman
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY 'rman';
# 创建 hive 数据库 并设置用户名密码为 hive/hive
create database hive default charset utf8 collate utf8_general_ci;
create user hive@'%' identified by 'hive';
grant all privileges on hive.* to hive@'%';
# 创建 activity_monitor 数据库 并设置用户名密码为 activity_monitor
create database activity_monitor default charset utf8 collate utf8_general_ci;
create user activity_monitor@'%' identified by 'activity_monitor';
grant all privileges on activity_monitor.* to activity_monitor@'%';
# 创建 reports_manager 并设置用户名密码为 reports_manager/reports_manager
create database reports_manager default charset utf8 collate utf8_general_ci;
create user reports_manager@'%' identified by 'reports_manager';
grant all privileges on reports_manager.* to reports_manager@'%';
# 创建 oozie 数据库 并设置用户名密码为 oozie/oozie
create database oozie default charset utf8 collate utf8_general_ci;
create user oozie@'%' identified by 'oozie';
grant all privileges on oozie.* to oozie@'%';
# 创建 hue 数据库 并设置用户名密码为 hue/hue
create database hue default charset utf8 collate utf8_general_ci;
create user hue@'%' identified by 'hue';
grant all privileges on hue.* to hue@'%';
# 刷新权限
FLUSH PRIVILEGES;
配置 Cloudera Manager 仓库 (需要所有节点)
mv cloudera-manager.repo /etc/yum.repos.d/
rpm --import RPM-GPG-KEY-cloudera
安装JDK
rpm -ivh oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm
# 添加环境变量
vim /etc/profile
export JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera
export PATH=$JAVA_HOME/bin:$PATH
source /etc/profile
安装 CM Server 和 Agent
# 仅在master上安装
yum install cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm cloudera-manager-server-6.2.0-968826.el7.x86_64.rpm
# 在所有的节点上安装
yum install cloudera-manager-daemons-6.2.0-968826.el7.x86_64.rpm cloudera-manager-agent-6.2.0-968826.el7.x86_64.rpm
添加连接mysql的jar包
mkdir /usr/share/java/
cd /usr/share/java/
wget http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.47/mysql-connector-java-5.1.47.jar
# 必须将其改名
mv mysql-connector-java-5.1.47.jar mysql-connector-java.jar
初始化信息 连接数据库
cd /opt/cloudera/cm/schema
# mysql 和 CM Server 在同一台机器上
./scm_prepare_database.sh mysql scm scm
# mysql 和 CM Server 不在同一台机器上
./scm_prepare_database.sh mysql -h <mysql-host-ip> --scm-host <cm-server-ip> scm scm
启动 Cloudera Manager Server 服务
# 启动服务
systemctl start cloudera-scm-server
# 监控日志信息
tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log
当出现 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server. 则启动成功