系统结构实践第四次作业

1.使用Docker-compose实现Tomcat+Nginx负载均衡

负载均衡原理

nginx反向代理原理：

  反向代理（Reverse Proxy）方式是指以代理服务器来接受客户端的连接请求，然后将请求转发给网络上的web服务器（可能是apache，nginx，tomcat，iis等）并将web服务器上得到的结果返回给请求连接的客户端，此时代理服务器对外就表现为一个服务器。

nginx代理tomcat集群，代理2个以上tomcat

文件创建如下：

各文件内容如下：
default.conf：

upstream tomcats {
    server 145_tomcat1:8080; # 主机名:端口号
    server 145_tomcat2:8080; # tomcat默认端口号8080
    server 145_tomcat3:8080; # 默认使用轮询策略
}

server {
    listen 80;
    server_name localhost;

    location / {
        proxy_pass http://tomcats; # 请求转向tomcats
    }
}

docker-compose.yml:

version: "3"
services:
    nginx:
        image: nginx
        container_name: 145_nginx
        ports:
            - "80:80"
        volumes:
            - ./nginx/default.conf:/etc/nginx/conf.d/default.conf # 挂载配置文件
        depends_on:
            - tomcat01
            - tomcat02
            - tomcat03

    tomcat01:
        image: tomcat
        container_name: 145_tomcat1
        volumes:
            - ./tomcat1:/usr/local/tomcat/webapps/ROOT # 挂载web目录

    tomcat02:
        image: tomcat
        container_name: 145_tomcat2
        volumes:
            - ./tomcat2:/usr/local/tomcat/webapps/ROOT

    tomcat03:
        image: tomcat
        container_name: 145_tomcat3
        volumes:
            - ./tomcat3:/usr/local/tomcat/webapps/ROOT

运行docker-compose

docker-compose up -d

查看容器

查看web端

了解nginx的负载均衡策略，并至少实现nginx的2种负载均衡策略

负载均衡策略1：轮询策略

import requests

url="http://127.0.0.1"

for i in range(0,10):
	reponse=requests.get(url)
	print(reponse.text)

负载均衡策略2：权重策略

修改Default.conf如下

test2.py:

import requests

url="http://127.0.0.1"
count={}
for i in range(0,2000):
    response=requests.get(url)
    if response.text in count:
        count[response.text]+=1;
    else:
        count[response.text]=1
for a in count:
    print(a, count[a])

2.使用Docker-compose部署javaweb运行环境

直接使用老师给的项目

负载均衡

修改dcoker-compose.yml

version: '2'
services:
  tomcat01:
    image: tomcat
    hostname: 145_javaweb
    container_name: tomcat4
    ports:
     - "5050:8080"
    volumes:
     - "$PWD/webapps:/usr/local/tomcat/webapps"
    networks:
      webnet:
        ipv4_address: 15.22.0.15
  tomcat02:
    image: tomcat
    container_name: tomcat5
    ports:
     - "5051:8080"
    volumes:
     - "$PWD/webapps:/usr/local/tomcat/webapps"
    networks:
      webnet:
        ipv4_address: 15.22.0.16
  mymysql:
    build: .
    image: mymysql:test
    container_name: mymysql
    ports:
      - "3306:3306"
    command: [
            '--character-set-server=utf8mb4',
            '--collation-server=utf8mb4_unicode_ci'
    ]
    environment:
      MYSQL_ROOT_PASSWORD: "123456"
    networks:
      webnet:
        ipv4_address: 15.22.0.6
  nginx:
     image: nginx
     ports:
         - "8080:8080"
     volumes:
         - ./default.conf:/etc/nginx/conf.d/default.conf # 挂载配置文件
networks:
 webnet:
   driver: bridge
   ipam:
     config:
       - subnet: 15.22.0.0/24
         gateway: 15.22.0.2

修改default.conf

upstream tomcats {
    server tomcat4:8080 weight=1; 
    server tomcat5:8080 weight=2; 
    
}

server {
    listen 8080;
    server_name localhost;

    location / {
        proxy_pass http://tomcats; # 请求转向tomcats
        
    }
}

重新启动容器即可

3.使用Docker搭建大数据集群环境

先拉取镜像

搭建hadoop环境

创建build文件 运行容器

sudo docker run -it -v /home/ivan145/build:/root/build --name ubuntu ubuntu

进入容器换源

cat<<EOF>/etc/apt/sources.list
deb http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-backports main restricted universe multiverse
deb http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
deb-src http://mirrors.aliyun.com/ubuntu/ bionic-proposed main restricted universe multiverse
EOF

安装软件以及配置ssh

apt-get update
apt-get install vim       
apt-get install ssh       
/etc/init.d/ssh start  

cd ~/.ssh
ssh-keygen -t rsa # 一直按回车即可
cat id_rsa.pub >> authorized_keys 

JDK的安装

apt install openjdk-8-jdk
vim ~/.bashrc       # 在文件末尾添加以下两行，配置Java环境变量：
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/
export PATH=$PATH:$JAVA_HOME/bin
source ~/.bashrc 
java -version #查看是否安装成功

安装hadoop

docker cp ./build/hadoop-3.1.3.tar.gz 容器ID：/root/build
cd /root/build
tar -zxvf hadoop-3.1.3.tar.gz -C /usr/local
vim ~/.bashrc  
export HADOOP_HOME=/usr/local/hadoop-3.1.3
export CLASSPATH=.:$JAVA_HOME/lib:$JRE_HOME/lib
export PATH=$PATH:$HADOOP_HOME/sbin:$HADOOP_HOME/bin:$JAVA_HOME/bin
source ~/.bashrc # 使.bashrc生效
hadoop version

配置hadoop集群

修改hadoop-env.sh

export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64

#core-site.xml
<configuration>
          <property> 
                  <name>hadoop.tmp.dir</name>
                  <value>file:/usr/local/hadoop-3.1.3/tmp</value>
                  <description>Abase for other temporary directories.</description>
          </property>
          <property>
                  <name>fs.defaultFS</name>
                  <value>hdfs://master:9000</value>
          </property>
</configuration>
#hdfs-site.xml
<configuration>
        <property>
                <name>dfs.replication</name>
                <value>1</value>
        </property>
        <property>
                <name>dfs.namenode.name.dir</name>
		        <value>file:/usr/local/hadoop-3.1.3/tmp/dfs/name</value>
	</property>
	<property>
                <name>dfs.datanode.data.dir</name>
                <value>file:/usr/local/hadoop-3.1.3/tmp/dfs/data</value>
	</property>
	<property>
                <name>dfs.permissions.enabled</name>
                <value>false</value>
        </property>
</configuration>
#mapred-site.xml
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=/usr/local/hadoop-3.1.3</value>
    </property>
    <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=/usr/local/hadoop-3.1.3</value>
    </property>
    <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=/usr/local/hadoop-3.1.3</value>
    </property>
</configuration>
#yarn-site.xml
<?xml version="1.0" ?>
<configuration>
<!-- Site specific YARN configuration properties -->
        <property>
               <name>yarn.nodemanager.aux-services</name>
               <value>mapreduce_shuffle</value>
        </property>
        <property>
               <name>yarn.resourcemanager.hostname</name>
               <value>Master</value>
        </property>
        <property>
               <name>yarn.nodemanager.vmem-pmem-ratio</name>
               <value>2.5</value>
        </property>
</configuration>

对于start-dfs.sh和stop-dfs.sh文件，添加下列参数：

HDFS_DATANODE_USER=root
HADOOP_SECURE_DN_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_SECONDARYNAMENODE_USER=root

对于start-yarn.sh和stop-yarn.sh，添加下列参数：

YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root

运行Hadoop集群

在三个终端上开启三个容器运行ubuntu/hadoop镜像，分别表示Hadoop集群中的master,slave01和slave02；

# 第一个终端
docker run -it -h master --name master ubuntu/hadoop
# 第二个终端
docker run -it -h slave01 --name slave01 ubuntu/hadoop
# 第三个终端
docker run -it -h slave02 --name slave02 ubuntu/hadoop

修改/etc/hosts

vim /etc/hosts
#修改为
172.17.0.2      master
172.17.0.3      slave01
172.17.0.4      slave02

测试ssh

ssh slave01
ssh slave02

修改workers

vim /usr/local/hadoop-3.1.3/etc/hadoop/workers
# 将localhost替换成两个slave的主机名
slave01
slave02

启动集群

cd /usr/local/hadoop-3.2.1
bin/hdfs namenode -format # 格式化文件系统
sbin/start-dfs.sh # 开启NameNode和DataNode服务
bin/hdfs dfs -mkdir /user # 建立HDFS文件夹，也可以放到下面示例程序中进行
bin/hdfs dfs -mkdir /user/root
bin/hdfs dfs -mkdir input
bin/hdfs dfs -put etc/hadoop/*.xml input # 将xml复制到input下，作为示例程序输入
sbin/start-yarn.sh # 开启ResourceManager和NodeManager服务
jps # 查看服务状态

运行Hadoop示例程序

bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.1.3.jar grep input output 'dfs[a-z.]+' # 运行示例
bin/hdfs dfs -get output output # 获取输出结果
cat output/* # 查看输出结果
sbin/stop-all.sh # 停止所有服务

用时：

大致花了两个下午，从搭建环境开始，问题比较多的地方就是第二个实验，按着教程一步步走却还是出错了，一直在调试，最后也没太搞懂这个给东西。第三个实验相当于是把大数据的实验拿过来了，就感觉比较轻松，不过途中有错的地方花了很多时间，看了很多同学的博客，有很多出错的地方都能很好的解决了