Install CDH

1. Turn off the firewall

(1) Turn off the firewall

systemctl stop firewalld

(2) Prohibit the firewall from starting up

systemctl disable firewalld

(3) Check the firewall status

firewall-cmd --state

Second, configure the node name and IP address mapping

(1) Configure the host name

vi /etc/hostname

IP Address	HostName
192.168.138.118	master
192.168.138.117	slave01
192.168.138.116	slave02
192.168.138.115	slave03

(2) Modify the hosts of each node

# 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
# ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.138.118 master
192.168.138.117 slave01
192.168.138.116 slave02
192.168.138.115 slave03

Three, configure ssh password-free login

(1) Verify whether the node is installed ssh

ssh localhost

(2) Generate key

ssh-keygen -t rsa -P ''

(3) Copy the public key to other nodes

ssh-copy-id slave01
ssh-copy-id slave02
ssh-copy-id slave03

(4) Verify whether the password-free login node

ssh slave01
ssh slave02
ssh slave03

(5) Log out

exit

Fourth, configure SELinux

(1) Check the status of SELinux

getenforce

(2) Modify the / etc / selinux / config file (in some systems, the / etc / sysconfig / selinux file)

# 修改前
SELINUX=enforcing
# 修改后
SELINUX=disabled

5. Start the NTP service

(1) Verify whether the node is installed with NTP

rpm -qa | grep ntp

(2) Reinstall NTP

yum -y install ntp

(3) Modify the /etc/ntp.conf file

master node

# For more information about this file, see the man pages
# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).

driftfile /var/lib/ntp/drift

# Permit time synchronization with our time source, but do not
# permit the source to query or modify the service on this system.
restrict default nomodify notrap nopeer noquery

# Permit all access over the loopback interface.  This could
# be tightened as well, but to do so would effect some of
# the administrative functions.
# restrict <ip address> nomodify notrap nopeer noquery
restrict 192.168.138.118 nomodify notrap nopeer noquery
restrict 127.0.0.1 
restrict ::1

# Hosts on local network are less restricted.
# restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap
# restrict <gateway> mask <prefix> nomodify notrap
restrict 192.168.138.2 mask 255.255.255.0 nomodify notrap

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
# server 0.centos.pool.ntp.org iburst
# server 1.centos.pool.ntp.org iburst
# server 2.centos.pool.ntp.org iburst
# server 3.centos.pool.ntp.org iburst
server 127.127.1.0
Fudge 127.127.1.0 stratum 10

#broadcast 192.168.1.255 autokey	# broadcast server
#broadcastclient			# broadcast client
#broadcast 224.0.1.1 autokey		# multicast server
#multicastclient 224.0.1.1		# multicast client
#manycastserver 239.255.254.254		# manycast server
#manycastclient 239.255.254.254 autokey # manycast client

# Enable public key cryptography.
#crypto

includefile /etc/ntp/crypto/pw

# Key file containing the keys and key identifiers used when operating
# with symmetric key cryptography. 
keys /etc/ntp/keys

# Specify the key identifiers which are trusted.
#trustedkey 4 8 42

# Specify the key identifier to use with the ntpdc utility.
#requestkey 8

# Specify the key identifier to use with the ntpq utility.
#controlkey 8

# Enable writing of statistics records.
#statistics clockstats cryptostats loopstats peerstats

# Disable the monitoring facility to prevent amplification attacks using ntpdc
# monlist command when default restrict does not include the noquery flag. See
# CVE-2013-5211 for more details.
# Note: Monitoring will not be disabled with the limited restriction flag.
disable monitor

slave01 node

# For more information about this file, see the man pages
# ntp.conf(5), ntp_acc(5), ntp_auth(5), ntp_clock(5), ntp_misc(5), ntp_mon(5).

driftfile /var/lib/ntp/drift

# Permit time synchronization with our time source, but do not
# permit the source to query or modify the service on this system.
restrict default nomodify notrap nopeer noquery

# Permit all access over the loopback interface.  This could
# be tightened as well, but to do so would effect some of
# the administrative functions.
# restrict <ip address> nomodify notrap nopeer noquery
restrict 192.168.138.117 nomodify notrap nopeer noquery
restrict 127.0.0.1 
restrict ::1

# Hosts on local network are less restricted.
# restrict 192.168.1.0 mask 255.255.255.0 nomodify notrap
# restrict <gateway> mask <prefix> nomodify notrap
restrict 192.168.138.2 mask 255.255.255.0 nomodify notrap

# Use public servers from the pool.ntp.org project.
# Please consider joining the pool (http://www.pool.ntp.org/join.html).
# server 0.centos.pool.ntp.org iburst
# server 1.centos.pool.ntp.org iburst
# server 2.centos.pool.ntp.org iburst
# server 3.centos.pool.ntp.org iburst
server 192.168.138.118
Fudge 192.168.138.118 stratum 10

#broadcast 192.168.1.255 autokey	# broadcast server
#broadcastclient			# broadcast client
#broadcast 224.0.1.1 autokey		# multicast server
#multicastclient 224.0.1.1		# multicast client
#manycastserver 239.255.254.254		# manycast server
#manycastclient 239.255.254.254 autokey # manycast client

# Enable public key cryptography.
#crypto

includefile /etc/ntp/crypto/pw

# Key file containing the keys and key identifiers used when operating
# with symmetric key cryptography. 
keys /etc/ntp/keys

# Specify the key identifiers which are trusted.
#trustedkey 4 8 42

# Specify the key identifier to use with the ntpdc utility.
#requestkey 8

# Specify the key identifier to use with the ntpq utility.
#controlkey 8

# Enable writing of statistics records.
#statistics clockstats cryptostats loopstats peerstats

# Disable the monitoring facility to prevent amplification attacks using ntpdc
# monlist command when default restrict does not include the noquery flag. See
# CVE-2013-5211 for more details.
# Note: Monitoring will not be disabled with the limited restriction flag.
disable monitor

Other nodes have the same configuration as the Slave01 node
(4) Start the NTP service

systemctl start ntpd

(5) Start the NTP service

systemctl enable ntpd

(6) View the NTP service status

systemctl status ntpd

Master node

NTP service status of the master node

Slave01 node

NTP service status of Slave01 node

Six, install jdk1.8

(1) Download the jdk-8u11-linux-x64.tar.gz compressed package
(2) Unzip the jdk-8u11-linux-x64.tar.gz compressed package

tar -zvxf jdk-8u11-linux-x64.tar.gz

(3) Modify the file name of jdk1.8.0_11

mv jdk1.8.0_11 jdk1.8

(4) Modify the / etc / profile file and set the java environment variable

export JAVA_HOME=/usr/java/jdk1.8
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tool.jar
export PATH=$PATH:$JAVA_HOME/bin

Seven, configure httpd distribution server

(1) Install httpd

yum -y install httpd

(2) Modify the /etc/httpd/conf/httpd.conf configuration file

<IfModule mime_module>
    #
    # TypesConfig points to the file containing the list of mappings from
    # filename extension to MIME-type.
    #
    TypesConfig /etc/mime.types

    #
    # AddType allows you to add to or override the MIME configuration
    # file specified in TypesConfig for specific file types.
    #
    #AddType application/x-gzip .tgz
    #
    # AddEncoding allows you to have certain browsers uncompress
    # information on the fly. Note: Not all browsers support this.
    #
    #AddEncoding x-compress .Z
    #AddEncoding x-gzip .gz .tgz
    #
    # If the AddEncoding directives above are commented-out, then you
    # probably should define those extensions to indicate media types:
    #
    AddType application/x-compress .Z
    AddType application/x-gzip .gz .tgz .parcel

    #
    # AddHandler allows you to map certain file extensions to "handlers":
    # actions unrelated to filetype. These can be either built into the server
    # or added with the Action directive (see below)
    #
    # To use CGI scripts outside of ScriptAliased directories:
    # (You will also need to add "ExecCGI" to the "Options" directive.)
    #
    #AddHandler cgi-script .cgi

    # For type maps (negotiated resources):
    #AddHandler type-map var

    #
    # Filters allow you to process content before it is sent to the client.
    #
    # To parse .shtml files for server-side includes (SSI):
    # (You will also need to add "Includes" to the "Options" directive.)
    #
    AddType text/html .shtml
    AddOutputFilter INCLUDES .shtml
</IfModule>

(3) Make sure httpd is started

systemctl enable httpd

(4) Restart the httpd service

systemctl restart httpd

Eight, install the Cloudera Manager server

(1) Download the rpm package of Cloudera Manager and CDH in advance

wget https://archive.cloudera.com/cm6/6.1.0/redhat7/yum/RPMS/x86_64/
wget https://archive.cloudera.com/cdh6/6.1.0/redhat7/yum/RPMS/x86_64/

cloudera manager rpm package
cdh rpm package
(2) Create warehouse ·

createrepo .

(3) Move the cm6.1 and cdh6.1 folders to the / var / www / html / folder ·

mv cm6.1 /var/www/html/
mv cdh6.1 /var/www/html/

(4) Each machine creates the Centos7.repo file under /etc/yum.repos.d

[centos7]
name=centos7
baseurl=http://192.168.138.118/cm6.1
enable=true
gpgcheck=false

(5) Each machine establishes cloud data cache

yum makecache

(6) Install the Cloudera Manager package on the Cloudera Manager server host (Master node)

sudo yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server

(7) Install the Cloudera Manager package on the slave node

sudo yum install cloudera-manager-daemons cloudera-manager-agent

(8) Modify the /etc/cloudera-scm-agent/config.ini file

# Hostname of the CM server.
server_host=Master

For more Agent configuration, please refer to the Agent Configuration File
(9) to ensure that the server and agent services of the master node are started

sudo systemctl enable cloudera-scm-agent
sudo systemctl enable cloudera-scm-server

(10) Ensure that the agent service of the slave node is started

sudo systemctl enable cloudera-scm-agent

Nine, install MySQL

(1) Download and install MySQL warehouse

wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm

sudo rpm -ivh mysql-community-release-el7-5.noarch.rpm

sudo yum install mysql-server

sudo systemctl start mysqld

(2) Stop MySQL service

If you want to make changes to an existing database, make sure to stop all services that use the database before continuing

sudo systemctl stop mysqld

(3) Modify the /etc/my.cnf file

[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
# To prevent deadlocks
transaction-isolation = READ-COMMITTED

# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0

key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

# Set the max_connections property according to the size of your cluster
max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MySQL, if you enable the binary log and do not set
#a server_id, MySQL will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
# Cloudera Manager will not start if its tables are configured with the MyISAM engine
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
# Cloudera Management Service roles need high write throughput because they might insert many records in the database
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

sql_mode=STRICT_ALL_TABLES

(4) Make sure the MySQL service is started

sudo systemctl enable mysqld

(5) Open the MySQL service

sudo systemctl start mysqld

(6) Set MySQL administrator password and security related settings

sudo /usr/bin/mysql_secure_installation

[...]
Enter current password for root (enter for none):
OK, successfully used password, moving on...
[...]
Set root password? [Y/n] Y
New password:
Re-enter new password:
Remove anonymous users? [Y/n] Y
[...]
Disallow root login remotely? [Y/n] N
[...]
Remove test database and access to it [Y/n] Y
[...]
Reload privilege tables now? [Y/n] Y
All done!

(7) Download MySQL JDBC driver

wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz

(8) Unzip the mysql-connector-java-5.1.46.tar.gz compressed package

tar zxvf mysql-connector-java-5.1.46.tar.gz

(9) Copy the JDBC driver to the / usr / share / java / directory

cd mysql-connector-java-5.1.46
sudo cp mysql-connector-java-5.1.46-bin.jar /opt/cloudera/cm/schema/../lib/mysql-connector-java.jar

(10) Log in to MySQL

mysql -uroot -proot

(11) Create a database for Cloudera software

Databases for Cloudera Software
Service	Database	User
Cloudera Manager Server	scm	scm
Activity Monitor	amon	amon
Reports Manager	rman	rman
Hue	hue	hue
Hive Metastore Server	metastore	hive
Sentry Server	sentry	sentry
Cloudera Navigator Audit Server	are not	are not
Cloudera Navigator Metadata Server	navms	navms
Oozie	oozie	oozie

CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

(12) Set permissions

GRANT ALL ON scm.* TO 'scm'@'%' IDENTIFIED BY '123456';
GRANT ALL ON amon.* TO 'amon'@'%' IDENTIFIED BY '123456';
GRANT ALL ON rman.* TO 'rman'@'%' IDENTIFIED BY '123456';
GRANT ALL ON hue.* TO 'hue'@'%' IDENTIFIED BY '123456';
GRANT ALL ON metastore.* TO 'metastore'@'%' IDENTIFIED BY '123456';
GRANT ALL ON sentry.* TO 'sentry'@'%' IDENTIFIED BY '123456';
GRANT ALL ON nav.* TO 'nav'@'%' IDENTIFIED BY '123456';
GRANT ALL ON navms.* TO 'navms'@'%' IDENTIFIED BY '123456';
GRANT ALL ON oozie.* TO 'oozie'@'%' IDENTIFIED BY '123456';

(13) Refresh permissions

FLUSH PRIVILEGES;

(14) Check the created database

SHOW DATABASES;

(15) Check the user's permissions

SHOW GRANTS FOR '<user>'@'%';

Ten, configure the Cloudera Manager database

sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm
sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h master --scm-host master scm scm

11. Install CDH

(1) Start the Cloudera Manager service

sudo systemctl start cloudera-scm-server

(2) View Cloudera Manager server logs

tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log

JP-Destiny

Published 131 original articles · won 12 · 60,000 views +

Private letter concerns

Big Data-Install CDH