大数据原生包搭建脚本

其他 2019-02-25 13:27:04 阅读次数: 0
版权声明：最终解释权归汐银所有，转贴请私聊 https://blog.csdn.net/qq_39562468/article/details/84951790

提示：本脚本是运行在centos7上的，节点为master、slave1、slave2。搭建平台主要是基础环境（java环境配置、ntp、主机互通）zookeeper、hadoop、hbase、hive

java+zookeeper：

master：

#1.安装虚拟机，关闭防火墙、selinux(每个节点都要运行)
systemctl stop firewalld 
setenforce 0
#同步时间
yum install -y ntp
echo -e 'server 127.127.1.0 
fudge 127.127.1.0 stratum 10' >>/etc/ntp.conf
/bin/systemctl restart ntpd.service 
#----------------------------
#2.ssh免密码，编辑hosts文件(以下操作重复都在master,slave1,slave2上运行)
 ssh-keygen -t rsa
 ssh-copy-id master
 ssh-copy-id slave1
 ssh-copy-id slave2

#----------------------------
#3.安装JDK
mkdir -p /usr/java
tar -zxvf /opt/soft/jdk-8u171-linux-x64.tar.gz  -C  /usr/java
scp -r /usr/java/ root@slave1:/usr/java
scp -r /usr/java/ root@slave2:/usr/java
echo '开始添加java环境'
echo -e 'export JAVA_HOME=/usr/java/jdk1.8.0_171 
export CLASSPATH=$JAVA_HOME/lib/ 
export PATH=$PATH:$JAVA_HOME/bin 
export PATH JAVA_HOME CLASSPATH

export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10 
PATH=$PATH:$ZOOKEEPER_HOME/bin

export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3 
export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib
export PATH=$PATH:$HADOOP_HOME/bin

export HBASE_HOME=/usr/hbase/hbase-1.2.4 
export PATH=$PATH:$HBASE_HOME/bin

export HIVE_HOME=/usr/hive/apache-hive-2.1.1-bin 
export PATH=$PATH:$HIVE_HOME/bin' >> /etc/profile
echo 'java环境配置完成'

#3.安装zookeeper
mkdir -p /usr/zookeeper
#解压
tar -zxvf /opt/soft/zookeeper-3.4.10.tar.gz -C /usr/zookeeper/
sed '/dataDir=/d' /usr/zookeeper/zookeeper-3.4.10/conf/zoo_sample.cfg > /usr/zookeeper/zookeeper-3.4.10/conf/zoo.cfg
echo '开始修改zookeeper配置文件'
echo -e 'dataDir=/usr/zookeeper/zookeeper-3.4.10/zkdata 
dataLogDir=/usr/zookeeper/zookeeper-3.4.10/zkdatalog 
server.1=master:2888:3888 
server.2=slave1:2888:3888 
server.3=slave2:2888:3888'>>/usr/zookeeper/zookeeper-3.4.10/conf/zoo.cfg 
#开始创建配置文件夹
mkdir -p /usr/zookeeper/zookeeper-3.4.10/zkdata
mkdir -p /usr/zookeeper/zookeeper-3.4.10/zkdatalog
#开始分发
scp -r /usr/zookeeper slave1:/usr/
scp -r /usr/zookeeper slave2:/usr/
echo -e '1'>>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh start
echo 'output:zookeeper'
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh status
source /etc/profile
if [ $? == 0 ]
then
 echo "变量刷新成功"
else
   [ $? != 0 ]
   echo "变量刷新失败！"
exit 6
fi

slave1

#1.安装虚拟机，关闭防火墙、selinux(每个节点都要运行)
systemctl stop firewalld 
setenforce 0
#时间同步
yum install -y ntp
ntpdate master

#----------------------------
#2.ssh免密码，编辑hosts文件(以下操作重复都在master,slave1,slave2上运行)
ssh-keygen -t rsa
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2

#----------------------------
#3.安装JDK
#mkdir -p /usr/java
#tar -zxvf /opt/soft/jdk-8u171-linux-x64.tar.gz  -C  /usr/java
#scp -r /usr/java/jdk1.8.0_171/ root@slave1:/usr/java
#scp -r /usr/java/jdk1.8.0_171/ root@slave2:/usr/java
echo '开始添加java环境'
echo -e 'export JAVA_HOME=/usr/java/jdk1.8.0_171 
export CLASSPATH=$JAVA_HOME/lib/ 
export PATH=$PATH:$JAVA_HOME/bin 
export PATH JAVA_HOME CLASSPATH

export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10 
PATH=$PATH:$ZOOKEEPER_HOME/bin

export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3 
export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib
export PATH=$PATH:$HADOOP_HOME/bin

export HBASE_HOME=/usr/hbase/hbase-1.2.4 
export PATH=$PATH:$HBASE_HOME/bin

export HIVE_HOME=/usr/hive/apache-hive-2.1.1-bin 
export PATH=$PATH:$HIVE_HOME/bin' >> /etc/profile
echo 'java环境配置完成'

echo '开始安装zookeeper'

#3.安装zookeeper
#scp -r /usr/zookeeper/ slave1:/usr/
#scp -r /usr/zookeeper/ slave2:/usr/
echo -e '2'>>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh start
echo 'output:zookeeper'
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh status
echo 'zookeeper配置完成'
if [ $? == 0 ]
then
 echo "变量刷新成功"
else
   [ $? != 0 ]
   echo "变量刷新失败！"
exit 6
fi

slave2

#1.安装虚拟机，关闭防火墙、selinux(每个节点都要运行)
systemctl stop firewalld 
setenforce 0
#时间同步
yum install -y ntp
ntpdate master

#----------------------------
#2.ssh免密码，编辑hosts文件(以下操作重复都在master,slave1,slave2上运行)
ssh-keygen -t rsa
ssh-copy-id master
ssh-copy-id slave1
ssh-copy-id slave2

#----------------------------
#3.安装JDK
#mkdir -p /usr/java
#tar -zxvf /opt/soft/jdk-8u171-linux-x64.tar.gz  -C  /usr/java
#scp -r /usr/java/jdk1.8.0_171/ root@slave1:/usr/java
#scp -r /usr/java/jdk1.8.0_171/ root@slave2:/usr/java
echo '开始添加java环境'
echo -e 'export JAVA_HOME=/usr/java/jdk1.8.0_171 
export CLASSPATH=$JAVA_HOME/lib/ 
export PATH=$PATH:$JAVA_HOME/bin 
export PATH JAVA_HOME CLASSPATH

export ZOOKEEPER_HOME=/usr/zookeeper/zookeeper-3.4.10 
PATH=$PATH:$ZOOKEEPER_HOME/bin

export HADOOP_HOME=/usr/hadoop/hadoop-2.7.3 
export CLASSPATH=$CLASSPATH:$HADOOP_HOME/lib
export PATH=$PATH:$HADOOP_HOME/bin

export HBASE_HOME=/usr/hbase/hbase-1.2.4 
export PATH=$PATH:$HBASE_HOME/bin

export HIVE_HOME=/usr/hive/apache-hive-2.1.1-bin 
export PATH=$PATH:$HIVE_HOME/bin' >> /etc/profile
echo 'java环境配置完成'

echo '开始安装zookeeper'

#3.安装zookeeper
#scp -r /usr/zookeeper/ slave1:/usr/
#scp -r /usr/zookeeper/ slave2:/usr/
echo -e '3'>>/usr/zookeeper/zookeeper-3.4.10/zkdata/myid
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh start
echo 'output:zookeeper'
/usr/zookeeper/zookeeper-3.4.10/bin/zkServer.sh status
echo 'zookeeper配置完成'
if [ $? == 0 ]
then
 echo "变量刷新成功"
else
   [ $? != 0 ]
   echo "变量刷新失败！"
exit 6
fi

hadoop

master

#创建文档
mkdir /usr/hadoop
#解压组件
tar -zxvf  /opt/soft/hadoop-2.7.3.tar.gz  -C  /usr/hadoop/
#修改配置文件
sed -i 's/export JAVA_HOME=${JAVA_HOME}/ /g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/hadoop-env.sh
echo -e 'export JAVA_HOME=/usr/java/jdk1.8.0_171' >> /usr/hadoop/hadoop-2.7.3/etc/hadoop/hadoop-env.sh
echo 'hadoop-env.sh配置完成'

#删除配置文件多于信息
sed -i 's/<configuration>/ /g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/core-site.xml
sed -i 's#</configuration># #g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/core-site.xml
echo -e  '<configuration>
        <property>
                <name>fs.default.name</name>
                <value>hdfs://master:9000</value>
        </property>
        <property>
                <name>hadoop.tmp.dir</name>
                <value>/usr/hadoop/hadoop-2.7.3/hdfs/tmp</value>
        <description>A base for other temporary directories.</description>
        </property>
        <property>
                <name>io.file.buffer.size</name>
                <value>131072</value>
        </property>
        <property>
                <name>fs.checkpoint.period</name>
                <value>60</value>
        </property>
        <property>
                <name>fs.checkpoint.size</name>
                <value>67108864</value>
        </property>
</configuration>' >> /usr/hadoop/hadoop-2.7.3/etc/hadoop/core-site.xml
echo 'core-site.xml配置完成'

#删除配置文件多于信息
sed -i 's/<configuration>/ /g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/yarn-site.xml
sed -i 's#</configuration># #g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/yarn-site.xml
echo -e  '<configuration>
        <property>
                <name>yarn.resourcemanager.address</name>
                <value>master:18040</value>
        </property>
        <property>
                <name>yarn.resourcemanager.scheduler.address</name>
                <value>master:18030</value>
        </property>
        <property>
                <name>yarn.resourcemanager.webapp.address</name>
                <value>master:18088</value>
        </property>
        <property>
                <name>yarn.resourcemanager.resource-tracker.address</name>
                <value>master:18025</value>
        </property>
        <property>
                <name>yarn.resourcemanager.admin.address</name>
                <value>master:18141</value>
        </property>
        <property>
                <name>yarn.nodemanager.aux-services</name>
                <value>mapreduce_shuffle</value>
        </property>
        <property>
                <name>yarn.nodemanager.auxservices.mapreduce.shuffle.class</name>
                <value>org.apache.hadoop.mapred.ShuffleHandler</value>
        </property>
</configuration>' >>/usr/hadoop/hadoop-2.7.3/etc/hadoop/yarn-site.xml

echo ' yarn-site.xml 配置完成'
rm -rf  /usr/hadoop/hadoop-2.7.3/etc/hadoop/slaves
echo -e 'slave1
slave2' >>/usr/hadoop/hadoop-2.7.3/etc/hadoop/slaves
echo 'slaves配置完成'
echo -e 'master' >>/usr/hadoop/hadoop-2.7.3/etc/hadoop/master
echo 'master配置完成'

#删除配置文件多于信息
sed -i 's/<configuration>/ /g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/hdfs-site.xml
sed -i 's#</configuration># #g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/hdfs-site.xml
echo -e  '<configuration>
        <property>
                <name>dfs.replication</name>
                <value>2</value>
        </property>
        <property>
                <name>dfs.namenode.name.dir</name>
                <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/name</value>
                <final>true</final>
        </property>
        <property>
                <name>dfs.datanode.data.dir</name>
                <value>file:/usr/hadoop/hadoop-2.7.3/hdfs/data</value>
                <final>true</final>
        </property>
        <property>
                <name>dfs.namenode.secondary.http-address</name>
                <value>master:9001</value>
        </property>
        <property>
                <name>dfs.webhdfs.enabled</name>
                <value>true</value>
        </property>
        <property>
                <name>dfs.permissions</name>
                <value>false</value>
        </property>
</configuration>' >> /usr/hadoop/hadoop-2.7.3/etc/hadoop/hdfs-site.xml
echo 'hdfs-site.xml配置成功'
#删除配置文件多于信息
sed  's/<configuration>/ /g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml.template >/usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml
sed -i 's#</configuration># #g' /usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml
echo -e '<configuration>
        <property>
                <name>mapreduce.framework.name</name>
                <value>yarn</value>
        </property>
</configuration>'>> /usr/hadoop/hadoop-2.7.3/etc/hadoop/mapred-site.xml
echo ' mapred-site.xml 配置成功'
#分发Hadoop
scp -r /usr/hadoop root@slave1:/usr/
scp -r /usr/hadoop root@slave2:/usr/
#格式化Hadoop
hadoop namenode -format

hbase 

master

#创建文档
mkdir -p /usr/hbase
#解压
tar -zxvf /opt/soft/hbase-1.2.4-bin.tar.gz -C /usr/hbase
#修改hbase-env.sh配置文件
echo -e 'export HBASE_MANAGES_ZK=false
export JAVA_HOME=/usr/java/jdk1.8.0_171
export HBASE_CLASSPATH=/usr/hadoop/hadoop-2.7.3/etc/hadoop/'>> /usr/hbase/hbase-1.2.4/conf/hbase-env.sh

#修改hbase-site.xml
sed -i 's/<configuration>/ /g' /usr/hbase/hbase-1.2.4/conf/hbase-site.xml
sed -i 's#</configuration># #g' /usr/hbase/hbase-1.2.4/conf/hbase-site.xml

echo -e  '<configuration>
        <property>
                <name>hbase.rootdir</name>
                <value>hdfs://master:9000/hbase</value>
        </property>
        <property>
                <name>hbase.cluster.distributed</name>
                <value>true</value>
        </property>
        <property>
                <name>hbase.master</name>
                <value>hdfs://master:6000</value>
        </property>
        <property>
                <name>hbase.zookeeper.quorum</name>
                <value>master,slave1,slave2</value>
        </property>
        <property>
                <name>hbase.zookeeper.property.dataDir</name>
                <value>/usr/zookeeper/zookeeper-3.4.10</value>
        </property>
</configuration>' >>/usr/hbase/hbase-1.2.4/conf/hbase-site.xml

#删除regionservers配置文件
rm -rf /usr/hbase/hbase-1.2.4/conf/regionservers #修改regionservers配置文件
#修改regionservers配置文件
echo -e  'slave1
slave2' >>/usr/hbase/hbase-1.2.4/conf/regionservers
#拷贝文件到hdfs
cp /usr/hadoop/hadoop-2.7.3/etc/hadoop/hdfs-site.xml /usr/hbase/hbase-1.2.4/conf/
cp /usr/hadoop/hadoop-2.7.3/etc/hadoop/core-site.xml /usr/hbase/hbase-1.2.4/conf/
#分发hbase
scp -r /usr/hbase/ slave1:/usr/
scp -r /usr/hbase/ slave2:/usr/
#启动服务
/usr/hbase/hbase-1.2.4/bin/start-hbase.sh

 mysql

slave2（先安装数据库我太弱了有些功能没写）

#mysqlִ#mysql
rpm -ivh /opt/soft/mysql57-community-release-el7-8.noarch.rpm
yum -y install mysql-community-server
systemctl daemon-reload
systemctl start mysqld
systemctl enable mysqld
scp /lib/mysql-connector-java-5.1.5-bin.jar root@slave1:/usr/hive/apache-hive-2.1.1-bin/lib
echo '----Please copy the content below----'
echo 'set global validate_password_policy= 0;'
echo 'set global validate_password_length=4;'
echo "alter user 'root'@'localhost' identified by '123456';"
echo 'exit'
echo 'mysql -uroot -p123456'
echo "create user 'root'@'%' identified by '123456';"
echo "grant all privileges on *.* to 'root'@'%' with grant option;"
echo 'flush privileges;'
echo 'exit'
echo 'scp /lib/mysql-connector-java-5.1.5-bin.jar root@slave1:/usr/hive/apache-hive-2.1.1-bin/lib'
echo '----Input completed----'
PASSWORD=`cat /var/log/mysqld.log | grep  password | sed -n 1p | awk '{print $11}'`
mysql -uroot -p$PASSWORD
#sql

hive

master

#master节点运行的hive
#创建文件夹
mkdir -p /usr/hive
#解压文件夹
tar -zxvf /opt/soft/apache-hive-2.1.1-bin.tar.gz -C /usr/hive/
#开始分发数据
scp -r /usr/hive/ root@slave1:/usr/hive
#修改hive-env.sh配置文件
cp -rvf /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh.template /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh 
echo -e 'HADOOP_HOME=/usr/hadoop/hadoop-2.7.3'>> /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh
echo 'hive-env.sh配置文件ok'
#配置hive-site.xml
echo -e '<configuration>
        <property>
                <name>hive.metastore.warehouse.dir</name>
                <value>/user/hive_remote/warehouse</value> 
        </property> 
        <property>
                <name>hive.metastore.local</name> 
                <value>false</value> 
        </property>
        <property>
                <name>hive.metastore.uris</name> 
                <value>thrift://slave1:9083</value> 
        </property>
</configuration>'>>/usr/hive/apache-hive-2.1.1-bin/conf/hive-site.xml
echo 'hive-site.xml配置文件ok'

echo 'Hive Completed!!!'
echo 'please enter:'
echo '###/usr/hive/apache-hive-2.1.1-bin/bin/hive'

slave1

#slave1节点运行的hive  
#修改hive-env.sh配置文件
cp -rvf /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh.template /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh 
echo -e 'HADOOP_HOME=/usr/hadoop/hadoop-2.7.3'>> /usr/hive/apache-hive-2.1.1-bin/conf/hive-env.sh
echo 'hive-env.sh配置文件ok'
#配置hive-site.xml
echo -e '<configuration>
        <property>
                <name>hive.metastore.warehouse.dir</name>
                <value>/user/hive_remote/warehouse</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionURL</name>                 
                <value>jdbc:mysql://slave2:3306/hive?createDatabaseIfNotExist=true</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionDriverName</name>
                <value>com.mysql.jdbc.Driver</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionUserName</name>
                <value>root</value>
        </property>
        <property>
                <name>javax.jdo.option.ConnectionPassword</name>
                <value>123456</value>
        </property>
        <property>
                <name>hive.metastore.schema.verification</name>
                <value>false</value>
        </property>
        <property>
                <name>datanucleus.schema.autoCreateAll</name>
                <value>true</value>
        </property>
</configuration>'>>/usr/hive/apache-hive-2.1.1-bin/conf/hive-site.xml
echo 'hive-site.xml配置文件ok'
echo 'Hive Completed!!!'
echo 'please enter:'
echo '###/usr/hive/apache-hive-2.1.1-bin/bin/hive --service metastore'

猜你喜欢

转载自blog.csdn.net/qq_39562468/article/details/84951790
今日推荐
周排行
回表和覆盖索引
设计模式-template method
GLES3.0中文API-glDrawElements
Java中类之间的关系
iPhone应用提交流程：如何将App程序发布到App Store
关于c内联函数不能有循环递归
移动端点击事件、滑动不可用的坑~~
16，SSH远程登录服务
SqlDataReader C#数据库查询结果数据
痴情研究java内存中的对象
每日归档
更多
2024-06-17(0)
2024-06-16(0)
2024-06-15(0)
2024-06-14(0)
2024-06-13(0)
2024-06-12(0)
2024-06-11(0)
2024-06-10(0)
2024-06-09(0)
2024-06-08(0)

代码天地 © 2018 codetd.com

境外服务器   最新电视剧2022