Install Jdk

Jdk:1.8 （jdk-8u144-linux-x64.tar.gz）

java -version

Install Hadoop

Hadoop:2.8.5 (hadoop-2.8.5.tar.gz)

cd /mydata/software

wget https://mirror.bit.edu.cn/apache/hadoop/common/hadoop-2.8.5/hadoop-2.8.5.tar.gz

tar -xvf hadoop-2.8.5.tar.gz

vim /etc/profile

#Java Config
export JAVA_HOME=/usr/java/jdk1.8.0_45
export JRE_HOME=/usr/java/jdk1.8.0_45/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib

#Hadoop Config
export HADOOP_HOME=/mydata/software/hadoop-2.8.5
export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native
export HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib"

#Path
export PATH=.:${JAVA_HOME}/bin:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:$PATH

source /etc/profile

server configuration

Modify hostname to ali1:

hostname

vim /etc/sysconfig/network

NETWORKING_IPV6=no
PEERNTP=no
NETWORKING=yes
HOSTNAME=ali1

vim /etc/hosts

172.17.163.85   ali1    ali1

Note: This is the intranet IP

Hadoop configuration

Before modifying the configuration, create a new folder:

mkdir  /root/hadoop
mkdir  /root/hadoop/tmp
mkdir  /root/hadoop/var
mkdir  /root/hadoop/dfs
mkdir  /root/hadoop/dfs/name
mkdir  /root/hadoop/dfs/data

First switch to the cd ${HADOOP_HOME}/etc/hadoop directory:

vim core-site.xml

<configuration>
   <property>
        <name>hadoop.tmp.dir</name>
        <value>/root/hadoop/tmp</value>
        <description>Abase for other temporary directories.</description>
   </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://ali1:9000</value>
    </property>
   <property>
        <name>fs.default.name</name>
        <value>hdfs://ali1:9000</value>
   </property>
</configuration>

vim hadoop-env.sh

export JAVA_HOME=/usr/java/jdk1.8.0_45

export HADOOP_OPTS="-Djava.library.path=${HADOOP_HOME}/lib/native"

vim hdfs-site.xml

<configuration>
<property>
   <name>dfs.name.dir</name>
   <value>/root/hadoop/dfs/name</value>
   <description>Path on the local filesystem where theNameNode stores the namespace and transactions logs persistently.</description>
</property>
<property>
   <name>dfs.data.dir</name>
   <value>/root/hadoop/dfs/data</value>
   <description>Comma separated list of paths on the localfilesystem of a DataNode where it should store its blocks.</description>
</property>
<property>
   <name>dfs.replication</name>
   <value>1</value>
</property>
<property>
      <name>dfs.permissions</name>
      <value>false</value>
      <description>need not permissions</description>
</property>
</configuration>

vim yarn-site.xml

<configuration>

<!-- Site specific YARN configuration properties -->

<property>
<!-- 指定resourceManager地址 -->
  <name>yarn.resourcemanager.hostname</name>
  <value>ali1</value>
</property>

<!-- reduce获取数据方式 -->
<property>
  <name>yarn.nodemanager.aux-services</name>
  <value>mapreduce_shuffle</value>
</property>

</configuration>

mv mapred-site.xml.template mapred-site.xml

<configuration>
<property>
    <name>mapred.job.tracker</name>
    <value>172.17.163.85:9001</value>
</property>
<property>
      <name>mapred.local.dir</name>
       <value>/root/hadoop/var</value>
</property>
<property>
       <name>mapreduce.framework.name</name>
       <value>yarn</value>
</property>
<property>
       <name>dfs.namenode.http-address</name>
       <value>172.17.163.85:50070</value>
</property>
</configuration>

Hadoop startup

Switch to ${HADOOP_HOME}/bin:

hadoop purpose -format

Switch to ${HADOOP_HOME}/sbin:

start-dfs.sh

start-yarn.sh

Browser access

http://39.102.77.195:8088/cluster (yarn management interface)

http://39.102.77.195:50070 (hdfs management interface)

Firewall settings

firewall-cmd --list-ports
firewall-cmd --zone=public --add-port=8088/tcp --permanent
firewall-cmd --zone=public --add-port=50070/tcp --permanent
firewall-cmd --reload

No password

ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

chattr -ia authorized_keys

cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

chattr +ia authorized_keys

Big data-Alibaba Cloud server installs stand-alone Hadoop