1. Mount the system ISO, obtain the linux system files and installation packages
[root@tanj2 home]# chkconfig --level 2345 iptables off
[root@tanj2 home]# mkdir /mnt/cdrom
[root@tanj2 cdrom]# mount -o loop /home/CentOS-6.7-x86_64-bin-DVD1.iso /mnt/cdrom
[root@tanj2 mnt]# cp -r cdrom/ /home/os
2. Configure the local yum library
[root @ tanj2 os] # cd /etc/yum.repos.d/
[root@tanj2 yum.repos.d]# vim os.repo
[os.repo]
name=os.repo
baseurl = file: /// home/os
enabled=1
gpgcheck=0
3. Install httpd service
[root@tanj2 yum.repos.d]# yum install httpd
[root@tanj2 yum.repos.d]# service iptables stop
[root@tanj2 yum.repos.d]# service httpd start
[root@tanj2 yum.repos.d]# chkconfig --level 2345 httpd on
[root@tanj2 home]# cp -r os/ /var/www/html/
4. Data hard disk partition and mount
[root@tanj2 home]# yum install parted
DEVICE_LIST="/dev/sda /dev/sdl /dev/sdg /dev/sdc /dev/sda /dev/sdd /dev/sdf /dev/sde /dev/sdj /dev/sdi /dev/sdh"
for DEVICE in $DEVICE_LIST
do
echo "+++++create partition for $DEVICE......"
parted -s $DEVICE mklabel gpt mkpart primary ext4 0% 100%
PARTITION="$DEVICE""1"
echo "+++++formatting $PARTITION......"
mkfs.ext4 -T largefile $PARTITION
#mkfs -t ext4 -j -m 1 -O extent,dir_index,spare_super $PARTITION
Done
[root@tanj2 home]# mkdir /data1
[root@tanj2 home]# vim /etc/fstab
/dev/sda1 /data1 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdb1 /data2 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdc1 /data3 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdd1 /data4 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sde1 /data5 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdf1 /data6 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdg1 /data7 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdh1 /data8 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdi1 /data9 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdj1 /data10 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
/dev/sdl1 /data11 ext4 defaults,noatime,nodiratime,barrier=0,data=writeback,commit=100 0 0
5. Install and configure mariadb
[root@tanj8 yum.repos.d]# yum install MariaDB-server MariaDB-client
[root@tanj2 repodata]# service mysql start
[root@tanj2 ~]# vim /etc/my.cnf.d/server.cnf
[mysqld]
lower_case_table_names=1
character_set_server = utf8
[root@tanj2 ~]# mysql –uroot
GRANT ALL PRIVILEGES ON *.* TO 'root'@'%'IDENTIFIED BY '123456' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'root'@'localhost'IDENTIFIED BY '123456' WITH GRANT OPTION;
flush privileges;
--hive database, why use latin1
create database hive DEFAULT CHARSET latin1;
--Cluster monitoring database
create database amon DEFAULT CHARSET utf8;
--hue database
create database hue DEFAULT CHARSET utf8;
--oozie database
create database oozie default charset utf8;
--authorization
GRANT ALL PRIVILEGES ON *.* TO 'cmf'@'%'IDENTIFIED BY '123456' WITH GRANT OPTION;
flush privileges;
6. Turn off selinux
[root @ tanj8 ~] # sestatus –v
[root@tanj8 yum.repos.d]# vim /etc/selinux/config
SELINUX=disabled
# SELINUXTYPE= can take one of these two values:
# targeted - Targeted processes are protected,
# mls - Multi Level Security protection.
SELINUXTYPE=targeted
7. Enable ntpd timing service
[root@tanj8 ~]# yum install ntp
[root@tanj8 ~]# vim /etc/ntp.conf
server 10.0.61.4 prefer iburst
server 127.127.1.0
[root@tanj8 ~]# chkconfig --level 2345 ntpd on
[root@tanj8 ~]# service ntpd start
8. System optimization
[root@tanj1 ~]# vim /etc/rc.local
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
[root@tanj2 etc]# echo never > /sys/kernel/mm/transparent_hugepage/defrag
[root@tanj2 etc]# echo never > /sys/kernel/mm/transparent_hugepage/enabled
[root@tanj2 etc]# vim /etc/sysctl.conf
vm.swappiness=10
[root@2 etc]# sysctl -p
9. ssh password-free login (append the public key generated by machine A to the ~/.ssh/authorized_keys file of machine B, then A can log in to machine B without password)
[root@tanj2 .ssh]# ssh-keygen -t rsa
[root@tanj2 .ssh]# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
[root@tanj3 .ssh]# scp authorized_keys root@B:/root/.ssh/
10. Set the hostname (based on the actual ip mapping)
[root@tanj3 .ssh]# vim /etc/hosts
127.0.0.1 tanj1
127.0.0.1 tanj2
127.0.0.1 tanj3
11. Uninstall jdk8
[root@tanj2 ~]# rpm -qa|grep jdk
[root@tanj2 ~]# rpm -e --nodeps java-1.8.0-openjdk-1.8.0.45-35.b13.el6.x86_64
[root@tanj2 ~]# rpm -e --nodeps java-1.8.0-openjdk-headless-1.8.0.45-35.b13.el6.x86_64
12. Install Oracle JDK and Cloudera Manager and Cloudera Manager Agent
[root@tanj2 cm511]# yum install oracle-j2sdk1.7
#Install manager-server manager-daemons on the master node on the cluster
[root@tanj2 local]# yum install cloudera-manager-daemons cloudera-manager-server
#All nodes on the cluster including the master node need to install CM Agent
[root@tanj2 local]#yum install cloudera-manager-agent cloudera-manager-daemons
[root@tanj2 local]# yum list installed
[root@tanj2 local]# yum list installed|grep oracle
[root@tanj2 local]# rpm -ql oracle-j2sdk1.7.x86_64
#Set the two properties server-host and server-port in the /etc/cloudera-scm-agent/config.ini file on all nodes of the cluster, which are the server IP and port where the CM Server process connected to the CM Agent runs
[root@tanj2 local]# vim /etc/cloudera-scm-agent/config.ini
[General]
# Hostname of the CM server.
server_host=cre2
# Port that the CM server is listening on.
server_port=7182
[root@tanj2 local]# chkconfig --level 2345 cloudera-scm-agent on
[root@tanj2 local]# chkconfig --level 2345 cloudera-scm-server on
[root @ tanj2 local] # sestatus –v
[root@tanj2 java]# ln -s mysql-connector-java.jar /usr/share/java/mysql-connector-java-5.1.42-bin.jar
[root@tanj2 local]# service cloudera-scm-agent start
[root@tanj2 schema]# /usr/share/cmf/schema/scm_prepare_database.sh mysql -uroot -p123456 --scm-host localhost cmf cmf 123456
[root@tanj2 cloudera-scm-server]# vim /etc/cloudera-scm-server/db.properties
com.cloudera.cmf.db.type=mysql
com.cloudera.cmf.db.host=localhost
com.cloudera.cmf.db.name=cmf
com.cloudera.cmf.db.user=cmf
com.cloudera.cmf.db.setupType=EXTERNAL
com.cloudera.cmf.db.password=123456
[root@tanj2 java]# service cloudera-scm-server start
[root@tanj2 java]# less /var/log/cloudera-scm-server/cloudera-scm-server.log
http://127.0.0.1:7180
13. cdh install
14. Install hdfs optimization
[root@tanj2 ~]# sudo -uhdfs hdfs dfsadmin -safemode leave
15. Install hive
[root@tanj2 ~]# sudo -uhdfs hdfs dfsadmin -safemode leave