Hadoop-CDH集群安装教程

其他 2021-12-13 13:47:53 阅读次数: 0

CDH 安装教程

集群准备

IP Hostname role
192.168.159.10 node1 master agent
192.168.159.11 node2 agent
192.168.159.12 node3 agent

修改hosts域名映射

vim /etc/hosts

添加如下内容

192.168.159.10  node1
192.168.159.11  node2
192.168.159.12  node3

配置免密登陆

root用户

ssh-keygen
ssh-copy-id -i ~/.ssh/id_rsa.pub root@node1
ssh-copy-id -i ~/.ssh/id_rsa.pub root@node2
ssh-copy-id -i ~/.ssh/id_rsa.pub root@node3

配置防火墙

关闭防火墙

# 查看该防火墙状态
systemctl status firewalld
# 关闭防火墙
systemctl stop firewalld
# 关闭防火墙，避免重启后恢复
systemctl disable firewalld

或在防火墙中开放端口 遇到防护墙不能关闭的情况，可使用该步骤！ 具体需要开放的端口参加（主机规划部分），以5432为例： 使用这些命令来永久打开一个新端口（如TCP/5432）。

sudo firewall-cmd --zone=public --add-port=5432/tcp --permanent
sudo firewall-cmd --reload

关闭SELinux

查看SELinux状态，执行：

/usr/sbin/sestatus -v

如果SELinux status参数为enabled即为开启状态

临时关闭：

setenforce 0

永久关闭SELinux，执行：

vim /etc/selinux/config

将 SELINUX=enforcing 改为 SELINUX=disabled 重启后生效

安装一些依赖以及删除部分依赖

yum ‐y install gcc python‐devel cyrus‐sasl* chrony iotop yum‐utils yum‐plugin-downloadonly psmisc ntp ntpdate httpd createrepo wget git rsync mod_ssl cyrus‐sasl‐plain cyrus‐sasl‐devel cyrus‐sasl‐gssapi libxslt krb5‐devel cyrus‐sasl‐gssapi cyrus‐sasl‐devel libxml2‐devel libxslt‐devel openldap‐devel python‐devel python‐simplejson sqlite‐devel

yum ‐y remove mariadb‐libs

systemctl start ntpd
systemctl enable ntpd

修改一些参数

sysctl vm.swappiness=10

echo 'vm.swappiness=10'>> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/ rc.local

s安装依赖

yum -y install gcc python‐devel cyrus‐sasl* chrony iotop yum‐utils yum‐plugin-downloadonly psmisc ntp ntpdate httpd createrepo wget git rsync mod_ssl cyrus‐sasl‐plain cyrus‐sasl‐devel cyrus‐sasl‐gssapi libxslt krb5‐devel cyrus‐sasl‐gssapi cyrus‐sasl‐devel libxml2‐devel libxslt‐devel openldap‐devel python‐devel python‐simplejson sqlite‐devel
# 删除mariadb
yum -y remove mariadb-libs

systemctl start ntpd
systemctl enable ntpd

修改一些参数

sysctl vm.swappiness=10

echo 'vm.swappiness=10'>> /etc/sysctl.conf
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.local

安装jdk

全部节点都需要安装

vim	 /etc/profile

在文件末尾添加

export JAVA_HOME=/usr/local/jdk/jdk1.8.0_202
export PATH=$PATH:$JAVA_HOME/bin

使修改生效

 source /etc/profile

查看java是否安装完成

 javac -version 
 jps

安装MySQL

选择node1

#下载这6个MySQL的rpm包
https://downloads.mysql.com/archives/get/p/23/file/mysql‐community‐common‐5.7.17‐1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql‐community‐libs‐ 5.7.17‐1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql‐community‐libs‐c ompat‐5.7.17‐1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql‐community‐devel‐ 5.7.17‐1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql‐community‐client ‐5.7.17‐1.el7.x86_64.rpm
https://downloads.mysql.com/archives/get/p/23/file/mysql‐community‐server ‐5.7.17‐1.el7.x86_64.rpm

#安装
yum -y localinstall mysql-community-common-5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql‐community‐libs‐5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql‐community‐libs‐compat‐5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql‐community‐devel‐5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql‐community‐client‐5.7.19-1.el7.x86_64.rpm 
yum -y localinstall mysql‐community‐server‐5.7.19-1.el7.x86_64.rpm

#启动
systemctl start mysqld

#开机自启动
systemctl enable mysqld

#查看临时密码
cat /var/log/mysqld.log | grep password

mysql ‐u root ‐p 
#首次登录设置密码
SET PASSWORD = PASSWORD('Zykj@123456');
quit;
#登陆设置允许外网访问当前的MySQL
mysql -u root -p 'Zykj@123456'
use mysql;
update user set host='%' where host='localhost';
delete from user where host !='%';
flush privileges;
quit;
mysql -u root -p 'Zykj@123456'
#创建ClouderaManager需要用到的几个数据库
create database cmf default character set='utf8';
create database amon default character set='utf8';
create database hive default character set='latin1';
create database hue default character set='utf8';
create database oozie default character set='utf8';
quit;

#整理mysql的驱动包,全部节点
mv mysql‐connector‐java‐5.1.27.jar mysql‐connector‐java.jar
mkdir /usr/share/java
mv mysql‐connector‐java.jar /usr/share/java/

下载CM组件

https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera‐m anager‐daemons‐6.2.1‐1426065.el7.x86_64.rpm
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera‐m anager‐agent‐6.2.1‐1426065.el7.x86_64.rpm
https://archive.cloudera.com/cm6/6.2.1/redhat7/yum/RPMS/x86_64/cloudera‐m anager‐server‐6.2.1‐1426065.el7.x86_64.rpm

#全部节点都安装daemons和agent
yum -y localinstall cloudera‐manager‐daemons‐6.2.1‐1426065.el7.x86_64.rpm
yum ‐y localinstall cloudera‐manager‐agent‐6.2.1‐1426065.el7.x86_64.rpm

#cdh001上安装server
yum ‐y localinstall cloudera‐manager‐server‐6.2.1‐1426065.el7.x86_64.rpm 
#到此为止，我们的CM服务安装完毕

下载CDH的parcel包

https://archive.cloudera.com/cdh6/6.2.1/parcels/CDH‐6.2.1‐1.cdh6.2.1.p0.1 425774‐el7.parcel
https://archive.cloudera.com/cdh6/6.2.1/parcels/CDH‐6.2.1‐1.cdh6.2.1.p0.1 425774‐el7.parcel.sha1
https://archive.cloudera.com/cdh6/6.2.1/parcels/manifest.json
#修改server和agent的配置文件
https://archive.cloudera.com/phoenix/6.2.0/parcels/PHOENIX‐5.0.0‐cdh6.2. 0.p0.1308267‐el7.parcel.sha
https://archive.cloudera.com/phoenix/6.2.0/parcels/PHOENIX‐5.0.0‐cdh6.2. 0.p0.1308267‐el7.parcel
https://archive.cloudera.com/phoenix/6.2.0/csd/PHOENIX‐1.0.jar

#然后把PHOENIX‐1.0.jar放到/opt/cloudera/csd目录 
[root@cdh001csd]#pwd
/opt/cloudera/csd
[root@cdh001csd]#ll
total8 
‐rw‐r‐‐r‐‐1 root root 5306 May 6 13:41 PHOENIX‐1.0.jar
#把凡是带sha1的都改成sha 
#把上面下载的包都放在/opt/cloudera/parcel‐repo下
[root@cdh001parcel‐repo]#pwd
/opt/cloudera/parcel‐repo
[root@cdh001parcel‐repo]#ll
total 2522808
‐rw‐r‐‐r‐‐1 root root 2093332003May213:39CDH‐6.2.1‐1.cdh6.2.1.p0.14 25774‐el7.parcel
‐rw‐r‐‐r‐‐1 root root 40May613:39CDH‐6.2.1‐1.cdh6.2.1.p0.1425774‐el 7.parcel.sha
‐rw‐r‐‐r‐‐1 root root 33875May613:39manifest.json
‐rw‐r‐‐r‐‐1 root root 402216960May613:42PHOENIX‐5.0.0‐cdh6.2.0.p0.1308267‐el7.parcel
‐rw‐r‐‐r‐‐1 root root 41May613:41PHOENIX‐5.0.0‐cdh6.2.0.p0.1308267‐ el7.parcel.sha

修改server和agent的配置文件

#node1上修改server
vim /etc/cloudera‐scm‐server/db.properties
#12行去掉注释把localhost改成cdh005
#15行去掉注释
#18行去掉注释，把cmf改成root
#21行去掉注释，密码那里填写Zykj@123456

# 全部节点修改agent
vim /etc/cloudera‐scm‐agent/config.ini
#15行把localhost修改为node1

启动CM，开始安装CDH集群

# 在node1上启动server
 systemctl start cloudera-scm-server
# 另开一个窗口，查看相关日志。有异常就解决异常
tail -200f /var/log/cloudera-scm-server/cloudera-scm-server.log 
# 这个异常可以忽略 ERROR ParcelUpdateService:com.cloudera.parcel.component s.ParcelDownloaderImpl: Unable to retrieve remote parcel repository manifes t

# 在全部节点上启动agent
 systemctl start cloudera-scm-agent

# 当在 node1上 netstat ‐tunlp | grep 7180 有内容时，说明我们可以访问web页 了

#重启命令
systemctl restart cloudera-scm-server
systemctl restart cloudera-scm-agent

测试部分

1、Kafka测试 2 创建一个topic
kafka‐topics ‐‐zookeeper cdh003:2181,cdh004:2181,cdh005:2181 ‐‐create‐‐r eplication‐factor 3 ‐‐partitions 10 ‐‐topic t1

开启kafka控制台消费者
kafka‐console‐consumer ‐‐bootstrap‐server cdh003:9092,cdh004:9092,cdh005: 9092 ‐‐topic t1

去别的节点开启一个Kafka控制台生产者
kafka‐console‐producer ‐‐broker‐list cdh003:9092,cdh004:9092,cdh005:9092 ‐‐topic t1
一边消费数据一边写入数据 

2、Hive测试
通过Hive建立一张表
create table if not exists t1(idint,namestring,scoredouble)row format delimited fields terminated by ',' stored as parquet; 
写个带有聚合函数的让它跑一下MapReduce
select count(*) from t1; 
3、Phoenix测试
create table if not exists t1(
id Integer,
name varchar,
clazz varchar,
score double
constraint pk primary key(id)
);
select * from t1;
upsert into t1 values(1001,'zhangsan','606',87.5);
select count(*) from t1;
select * from t1;

4、Impala测试
create table t2
(
name string,
dt string,
primary key(name,dt)
)
partition by hash partitions 16
stored as kudu tblproperties(
 'kudu.master_addresses'='cdh001:7051'
 );

select * from t2;
insert into t2 values('zhangsan','2020‐05‐06');
select count(*) from t2;
select * from t2;

问题总结

CDH安装失败，重装操作

删除Agent节点的UUID

rm -rf /var/lib/cloudera-scm-agent/cm_guid

清空主节点CM数据库

--进入主节点的mysql数据库，然后删除数据库
drop database cmf;

删除Agent节点namenode和datanode节点信息

rm -rf /data1/dfs/nn/*
rm -rf /data1/dfs/fn/*
rm -rf /data1/dfs/dn/*

在主节点上重新初始化CM数据库

重新创建数据库 cmf

create database cmf default character set='utf8';

cd /var/lib/cloudera-scm-agent
./scm_prepare_database.sh -hnode1 -P3306 --scm-host node1 --force mysql cmf root Zykj@123456

执行启动脚本

# 主节点
systemctl start cloudera-scm-server
# Agent节点
systemctl start cloudera-scm-agent

通过主节点的7180端口重新进行CDH的安装 http://主节点IP:7180/cmf/login

主机上未检测到 Java JDK

CDH添加新主机 - 主机上未检测到 Java JDK

在cloudera manager管理界面上操作往CDH集群添加新主机时，跳过JDK安装的步骤，因为服务器一般都会安装有JDK

全部步骤完成后，到主机配置也发现由如下提示：“主机上未检测到 Java JDK。”

如果不解决这个问题，角色将会无法启动。

在配置页面搜索java，配置对应路径，保存配置即可

image-20210221224139068

猜你喜欢

转载自blog.csdn.net/qq_36213352/article/details/117857445
今日推荐
面壁智能发布 Eurux-8x22B 开源大模型 —— 堪称「理科状元」
开源日报 | 谷歌扶持鸿蒙上位；开源Rabbit R1；Docker加持的安卓手机；微软的焦虑和野心；海尔电器把开放平台关了
中国码农的“35岁魔咒”
蘭雅 CorelDRAW 插件 2024.5.1 国际劳动节版，免费下载
Arc Browser for Windows 1.0 正式 GA
90后程序员开发视频搬运软件、不到一年获利超 700 万，结局很刑！
周排行
OOP第二次作业
java web 乱码问题
android 禁止scrollview 因控件变化自动滚动到底的方法
mysql服务解压版的安装(5.7)
centos7 nginx+tomcat配置https 安装免费SSL Let’s Encrypt
使用Mosquitto遗嘱机制实现感知客户端上下线功能的方法
面向对象之------多态与多态性
开发Teams Tabs应用程序
C# 希尔排序
第2章 Jupyter Notebooks
每日归档
更多
2024-05-06(40)
2024-05-05(0)
2024-05-04(7)
2024-05-03(19)
2024-05-02(0)
2024-05-01(4)
2024-04-30(1)
2024-04-29(40)
2024-04-28(0)
2024-04-27(56)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022