hadoop高可用部署（zookeeper + hadoop）部署

hadoop高可用部署（zookeeper + hadoop）部署

主要软件版本

software	version
java	jdk-8u171-linux-x64.rpm
hadoop	hadoop-2.7.6.tar.gz
zookeeper	zookeeper-3.4.12.tar.gz

集群规划

hostname	os	ip	software	deamon
master	centos7	192.168.3.100	jdk, hadoop	NameNode, ResourceManager, DFSZKFailoverController(zkfc)
master1	centos7	192.168.3.101	jdk, hadoop	NameNode, ResourceManager, DFSZKFailoverController(zkfc)
slave1	centos7	192.168.3.102	jdk, hadoop, zookeeper	DateNode, NodeManager, JournalNode, QuorumPeerMain
slave2	centos7	192.168.3.103	jdk, hadoop, zookeeper	DateNode, NodeManager, JournalNode, QuorumPeerMain
slave3	centos7	192.168.3.104	jdk, hadoop, zookeeper	DateNode, NodeManager, JournalNode, QuorumPeerMain

基础环境配置

基础环境配置参考基础环境配置章节

zookeeper安装

zookeeper安装参考zookeeper分布式安装

hadoop集群配置

1. 修改core-site.xml

<configuration>
    <!-- 指定hdfs的nameservice为ns1 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://master</value>
    </property>
    <!-- 指定hadoop临时目录 -->
    <property>  
        <name>hadoop.tmp.dir</name>
        <value>/usr/local/hadoop/tmp</value>
        <description>Abase for other temporary directories.</description>
    </property>             
    <!-- 指定zookeeper地址 -->
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>slave1:2181,slave1:2181,slave1:2181</value>
    </property>
</configuration>

此处 /usr/local/hadoop/tmp如果写成file:/usr/local/hadoop/tmp会报java.lang.IllegalArgumentException: URI has an authority component错误

2. 修改hdfs-site.xml

<configuration>
    <!--指定hdfs的nameservice为master，需要和core-site.xml中的保持一致 -->
    <property>
        <name>dfs.nameservices</name>
        <value>master</value>
    </property>
    <!-- ns1下面有两个NameNode，分别是nn1，nn2 -->
    <property>
        <name>dfs.ha.namenodes.master</name>
        <value>nn1,nn2</value>
    </property>
    <!-- nn1的RPC通信地址 -->
    <property>
        <name>dfs.namenode.rpc-address.master.nn1</name>
        <value>master:9000</value>
    </property>
    <!-- nn1的http通信地址 -->
    <property>
        <name>dfs.namenode.http-address.master.nn1</name>
        <value>master:50070</value>
    </property>
    <!-- nn2的RPC通信地址 -->
    <property>
        <name>dfs.namenode.rpc-address.master.nn2</name>
        <value>master1:9000</value>
    </property>
    <!-- nn2的http通信地址 -->
    <property>
        <name>dfs.namenode.http-address.ns1.nn2</name>
        <value>master1:50070</value>
    </property>
    <!-- 指定NameNode的元数据在JournalNode上的存放位置 -->
    <property>
        <name>dfs.namenode.shared.edits.dir</name>
        <value>qjournal://slave1:8485;slave2:8485;slave3:8485/ns1</value>
    </property>
    <!-- 指定JournalNode在本地磁盘存放数据的位置 -->
    <property>
        <name>dfs.journalnode.edits.dir</name>
        <value>/usr/local/hadoop/journaldata</value>
    </property>
    <!-- 开启NameNode失败自动切换 -->
    <property>
        <name>dfs.ha.automatic-failover.enabled</name>
        <value>true</value>
    </property>
    <!-- 配置失败自动切换实现方式 -->
    <property>
        <name>dfs.client.failover.proxy.provider.master</name>
        <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
    </property>
    <!-- 配置隔离机制方法，多个机制用换行分割，即每个机制暂用一行-->
    <property>
        <name>dfs.ha.fencing.methods</name>
        <value>
        sshfence
        shell(/bin/true)
        </value>
    </property>
    <!-- 使用sshfence隔离机制时需要ssh免登陆 -->
    <property>
        <name>dfs.ha.fencing.ssh.private-key-files</name>
        <value>/root/.ssh/id_rsa</value>
    </property>
    <!-- 配置sshfence隔离机制超时时间 -->
    <property>
        <name>dfs.ha.fencing.ssh.connect-timeout</name>
        <value>30000</value>
    </property>
</configuration>

3. 修改mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>

4. 修改yarn-site.xml

<configuration>
    <!-- 开启RM高可用 -->
    <property>
        <name>yarn.resourcemanager.ha.enabled</name>
        <value>true</value>
    </property>
    <!-- 指定RM的cluster id -->
    <property>
        <name>yarn.resourcemanager.cluster-id</name>
        <value>yrc</value>
    </property>
    <!-- 指定RM的名字 -->
    <property>
        <name>yarn.resourcemanager.ha.rm-ids</name>
        <value>rm1,rm2</value>
    </property>
    <!-- 分别指定RM的地址 -->
    <property>
        <name>yarn.resourcemanager.hostname.rm1</name>
        <value>master</value>
    </property>
    <property>
        <name>yarn.resourcemanager.hostname.rm2</name>
        <value>master1</value>
    </property>
    <!-- 指定zk集群地址 -->
    <property>
        <name>yarn.resourcemanager.zk-address</name>
        <value>slave1:2181,slave1:2181,slave1:2181</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>

4. 修改slaves文件

slave1
slave2
slave3

5. 修改hadoop-env.sh配置JAVA_HOME

export JAVA_HOME=/usr/java/default

复制文件至其他slave

scp -r /usr/local/hadoop slave1:/usr/local/
scp -r /usr/local/hadoop slave2:/usr/local/
scp -r /usr/local/hadoop slave3:/usr/local/
scp -r /usr/local/hadoop master1:/usr/local/

启动zookeeper集群

在slave1, slave2, slave3上分别启动zookeeper
zkServer.sh start  # 启动
zkServer.sh status # 查看状态
zkServer.sh stop   # 停止

启动journalnode

在slave1, slave2, slave3上分别启动journalnode
hadoop-daemon.sh start journalnode
jps # 查看JournalNode进程

在master节点格式化HDFS

hdfs namenode -format

格式化后会在根据core-site.xml中的hadoop.tmp.dir配置生成个文件，/usr/local/hadoop/tmp拷贝到master1的/usr/local/hadoop

scp -r /usr/local/hadoop/tmp master1:/usr/local/hadoop

在master节点格式化ZKFC

hdfs zkfc -formatZK

在master节点启动HDFS

start-dfs.sh

启动YARN

start-yarn.sh

web Interface

NameNode            http://192.168.3.100:50070        Default HTTP port is 50070
ResourceManager     http://192.168.3.100:8088         Default HTTP port is 8088