openEuler Linux 部署 HadoopHA

物联网 2023-04-08 09:12:32 阅读次数: 0

openEuler Linux 部署 HadoopHA

升级操作系统和软件

yum -y update

升级后建议重启

安装常用软件

yum -y install gcc gcc-c++ autoconf automake cmake make rsync vim man zip unzip net-tools zlib zlib-devel openssl openssl-devel pcre-devel tcpdump lrzsz tar wget

修改主机名

hostnamectl set-hostname hadoop
或者
vim /etc/hostname 
spark01

reboot

修改IP地址

vim /etc/sysconfig/network-scripts/ifcfg-ens160

网卡 配置文件示例

TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
NAME=ens160
UUID=943779e9-249c-44bb-b272-d49ea5831ed4
DEVICE=ens160
ONBOOT=yes
IPADDR=192.168.28.11
PREFIX=24
GATEWAY=192.168.28.2
DNS1=192.168.28.2

保存后
nmcli con up ens160
重启网络服务

关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

vim /etc/selinux/config

# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
# SELINUX=enforcing
# SELINUXTYPE= can take one of three two values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected. 
#     mls - Multi Level Security protection.
# SELINUXTYPE=targeted 

SELINUX=disabled

执行下面命令

setenforce 0

或者

sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
setenforce 0

创建软件安装目录并上传软件，配置环境变量

mkdir -p /opt/soft
cd /opt/soft
# 上传jdk zookeeper
tar -zxvf jdk-8u361-linux-x64.tar.gz
mv jdk1.8.0_361 jdk8
tar -zxvf hadoop-3.3.5.tar.gz
mv hadoop-3.3.5 hadoop3

vim /etc/profile
	
export JAVA_HOME=/opt/soft/jdk8
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin

export HADOOP_HOME=/opt/soft/hadoop3

export HADOOP_INSTALL=${HADOOP_HOME}
export HADOOP_MAPRED_HOME=${HADOOP_HOME}
export HADOOP_COMMON_HOME=${HADOOP_HOME}
export HADOOP_HDFS_HOME=${HADOOP_HOME}
export YARN_HOME=${HADOOP_HOME}
export PATH=${
    
    PATH}:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin
export HADOOP_CONF_DIR=${HADOOP_HOME}/etc/hadoop

export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

编辑完成后使用source命令使文件~/.bash_profile生效执行以下命令
source /etc/profile
检查环境变量
printenv

修改域名映射

vim /etc/hosts
192.168.28.11 spark01
192.168.28.12 spark02
192.168.28.13 spark03

修改后建议重启

修改Hadoop配置文件 在hadoop解压后的目录找到 etc/hadoop目录

cd /opt/soft/hadoop3

修改如下配置文件

  • hadoop-env.sh
  • core-site.xml
  • hdfs-site.xml
  • workers
  • mapred-site.xml
  • yarn-site.xml

hadoop-env.sh 文件末尾追加

export JAVA_HOME=/opt/soft/jdk8
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_ZKFC_USER=root
export HDFS_JOURNALNODE_USER=root

export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root

core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://puegg</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/hadoop_data</value>
    </property>
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>root</value>
    </property>
    <!-- HDFS连接zookeeper集群的地址和端口 -->
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>spark01:2181,spark02:2181,spark03:2181</value>
    </property>
    <property>
        <name>dfs.permissions.enabled</name>
        <value>false</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.root.groups</name>
        <value>*</value>
    </property>
</configuration>

hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <!-- 定义hdfs集群ID号 -->
    <property>
        <name>dfs.nameservices</name>
        <value>puegg</value>
    </property>
    <!-- 定义hdfs集群中namenode的ID号 -->
    <property>
        <name>dfs.ha.namenodes.puegg</name>
        <value>nn1,nn2</value>
    </property>
    <!-- 定义namenode的主机名和RPC协议的端口 -->
    <property>
        <name>dfs.namenode.rpc-address.puegg.nn1</name>
        <value>spark01:8020</value>
    </property>
    <property>
        <name>dfs.namenode.rpc-address.puegg.nn2</name>
        <value>spark02:8020</value>
    </property>
    <!-- 定义namenode的主机名和HTTP协议的端口 -->
    <property>
        <name>dfs.namenode.http-address.puegg.nn1</name>
        <value>spark01:9870</value>
    </property>
    <property>
        <name>dfs.namenode.http-address.puegg.nn2</name>
        <value>spark02:9870</value>
    </property>
    <!-- 定义共享edits的URL -->
    <property>
        <name>dfs.namenode.shared.edits.dir</name>
        <value>qjournal://spark01:8485;spark02:8485;spark03:8485/puegg</value>
    </property>
    <!-- 定义HDFS的客户端连接HDFS集群时返回active namenode地址 -->
    <property>
        <name>dfs.client.failover.proxy.provider.puegg</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <!-- HDFS集群中两个namenode切换状态时的隔离方法 -->
    <property>
        <name>dfs.ha.fencing.methods</name>
        <value>sshfence</value>
    </property>
    <!-- HDFS集群中两个namenode切换状态时的隔离方法的密钥 -->
    <property>
        <name>dfs.ha.fencing.ssh.private-key-files</name>
        <value>/root/.ssh/id_rsa</value>
    </property>

    <!-- journalnode集群中用于保存edits文件的目录 -->
    <property>
        <name>dfs.journalnode.edits.dir</name>
        <value>/opt/journalnode/data</value>
    </property>

    <!-- HA的HDFS集群自动切换namenode的开关-->
    <property>
        <name>dfs.ha.automatic-failover.enabled</name>
        <value>true</value>
    </property>
    <property>
        <name>dfs.safemode.threshold.pct</name>
        <value>1</value>
        <description>
            Specifies the percentage of blocks that should satisfy
            the minimal replication requirement defined by dfs.replication.min.
            Values less than or equal to 0 mean not to wait for any particular
            percentage of blocks before exiting safemode.
            Values greater than 1 will make safe mode permanent.
        </description>
    </property>:q!
</configuration>

workers

spark01
spark02
spark03

mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>
</configuration>

yarn-site.xml

<?xml version="1.0"?>
<configuration>
<!-- Site specific YARN configuration properties -->
<property>
  <name>yarn.resourcemanager.ha.enabled</name>
  <value>true</value>
</property>
<property>
  <name>yarn.resourcemanager.cluster-id</name>
  <value>cluster1</value>
</property>
<property>
  <name>yarn.resourcemanager.ha.rm-ids</name>
<property>
  <name>yarn.resourcemanager.hostname.rm1</name>
  <value>spark01</value>
</property>
<property>
  <name>yarn.resourcemanager.hostname.rm2</name>
  <value>spark02</value>
</property>
<property>
  <name>yarn.resourcemanager.webapp.address.rm1</name>
  <value>spark01:8088</value>
</property>
<property>
  <name>yarn.resourcemanager.webapp.address.rm2</name>
  <value>spark02:8088</value>
</property>
<property>
  <name>yarn.resourcemanager.zk-address</name>
  <value>spark01:2181,spark02:2181,spark03:2181</value>
</property>
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>

<property>
    <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>
<property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_MAPRED_HOME</value>
</property>
</configuration>

配置ssh免密钥登录

创建本地秘钥并将公共秘钥写入认证文件

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
# 或者
ssh-copy-id spark01
ssh-copy-id spark02
ssh-copy-id spark03
scp -rv  ~/.ssh root@spark02:~/
scp -rv  ~/.ssh root@spark03:~/

# 远程登录自己
ssh spark01
# Are you sure you want to continue connecting (yes/no)? 此处输入yes
# 登录成功后exit或者logout返回
exit

拷贝配置文件分发到其它服务器或者使用脚本分发

scp -v /etc/profile root@spark02:/etc
scp -v /etc/profile root@spark03:/etc
scp -rv /opt/soft/hadoop3/etc/hadoop/* root@spark02:/opt/soft/hadoop3/etc/hadoop/
scp -rv /opt/soft/hadoop3/etc/hadoop/* root@spark03:/opt/soft/hadoop3/etc/hadoop/

在各服务器上使环境变量生效

source /etc/profile

Hadoop初始化

# 创建数据目录
mkdir -p /home/hadoop_data
1.	启动三个zookeeper：zkServer.sh start
2.	启动三个JournalNode：hadoop-daemon.sh start journalnode
7.	在其中一个namenode上格式化：hdfs namenode -format
8.	把刚刚格式化之后的元数据拷贝到另外一个namenode上
    a)	启动刚刚格式化的namenode :hadoop-daemon.sh start namenode
    b)	在没有格式化的namenode上执行：hdfs namenode -bootstrapStandby
    c)	启动第二个namenode: hadoop-daemon.sh start namenode
9.	在其中一个namenode上初始化hdfs zkfc -formatZK
10.	停止上面节点：stop-dfs.sh
11.	全面启动：start-dfs.sh
12. 启动resourcemanager节点 yarn-daemon.sh start resourcemanager
yarn --daemon start

http://dl.bintray.com/sequenceiq/sequenceiq-bin/hadoop-native-64-2.5.0.tar


13、安全模式

hdfs dfsadmin -safemode enter  
hdfs dfsadmin -safemode leave


14、查看哪些节点是namenodes并获取其状态
hdfs getconf -namenodes
hdfs haadmin -getServiceState spark01

15、强制切换状态
hdfs haadmin -transitionToActive --forcemanual spark01

重点提示：

# 关机之前 依关闭服务
stop-yarn.sh
stop-dfs.sh
# 开机后 依次开启服务
start-dfs.sh
start-yarn.sh

或者

# 关机之前关闭服务
stop-all.sh
# 开机后开启服务
start-all.sh

#jps 检查进程正常后开启胡哦关闭在再做其它操作

猜你喜欢

转载自blog.csdn.net/qq_24330181/article/details/130027366
今日推荐
火速冲上 GitHub 热榜 —— 开源编程语言、框架哪有这么可爱？
北京人形机器人创新中心发布全球首个纯电驱拟人奔跑的全尺寸人形机器人“天工”
LFOSSA 源来如此公开课 | 掌握云原生未来：CNCF 认证全面攻略与备考秘籍
国产云输入法——仅华为无云端数据上传安全问题
开源日报 | 工业开源项目OGG 1.0；姐姐，你要和我一起配置火狐吗；苹果AI遥遥落后？Fedora 40
开放签电子签章：停止新增，优化体验，前进更进（五一假期前工作）
开源日报 | 中学生开源前端动画引擎；全球首个Llama3 8B中文版开源模型；联想电脑恐出局；Linus讽刺AI炒作
周排行
浏览器对同一域名进行请求的最大并发连接数
React Hook之自定义Hook
【转】MyBatis缓存机制
-Java-泛型
自动化测试常用脚本-发送邮件
LeetCode#859: Buddy Strings
java、Python处理字符串
第二篇の博客
Hadoop伪分布式环境安装
SQL Server进阶（十一）临时表、表变量
每日归档
更多
2024-04-27(56)
2024-04-26(39)
2024-04-25(22)
2024-04-24(36)
2024-04-23(26)
2024-04-22(39)
2024-04-21(0)
2024-04-20(6)
2024-04-19(5)
2024-04-18(0)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022