1.1 Fullnat模式与NAT模式的区别
1.1.1 NAT模式下报文变化
发送 接收
cip ---> vip
cip ---> rip（DNAT）
rip ---> cip
vip ---> cip（SNAT）
1.1.1 fullnat模式下报文变化
发送 接收
cip ------> vip (通过DNAT把vip转换为lip)
lip -------> rip （SNAT + DNAT）
rip -------->lip
vip ------> cip（SNAT + DNAT）
注释：
1. CIP为客户端的地址
2. VIP为虚拟地址
3. rip为真实的服务器
4. lip为本地地址
5. SNAT为来源地址转换
6. DNAT为目的地址转换
1.1 FULLnat模式搭建
1.1.1 环境准备
lvs服务器(3台) |
后端web(nginx:80) |
10.0.0.31 lvs-node01 |
|
10.0.0.32 lvs-node02 |
|
10.0.0.33 lvs-node03 |
|
10.0.0.129(vip) |
1.1.2 Hosts解析修改
[root@lvs-node02 ~]# vim /etc/hosts 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 10.0.0.31 lvs-node01 10.0.0.32 lvs-node02 10.0.0.33 lvs-node03
1.1.3 DNS解析修改
[root@lvs-node02 ~]# vim /etc/resolv.conf nameserver 223.5.5.5 nameserver 223.6.6.6
1.1.4 关闭selinux与iptables
[root@lvs-node02 ~]# getenforce Disabled [root@lvs-node02 ~]# /etc/init.d/iptables status iptables: Firewall is not running.
1.1.5 配置yum源
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-6.repo wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-6.repo
1.1.6 内核优化
net.ipv4.conf.all.arp_ignore = 1 net.ipv4.conf.all.arp_announce = 2 net.core.netdev_max_backlog = 500000 net.ipv4.ip_forward = 1
1.1.7 下载安装包
全部放在/usr/local/src目录中 wget http://kb.linuxvirtualserver.org/images/a/a5/Lvs-fullnat-synproxy.tar.gz wget ftp://ftp.redhat.com/pub/redhat/linux/enterprise/6Server/en/os/SRPMS/kernel-2.6.32-220.el6.src.rpm mv kernel-2.6.32-220.el6.src.rpm Linux-2.6.32-220.23.1.el6.x86_64.lvs.src.gz Lvs-fullnat-synproxy.tar.gz /usr/local/src/
1.1.8 安装依赖包
yum install -y xmlto asciidoc elfutils-libelf-devel zlib-devel binutils-devel newt-devel python-devel hmaccalc perl\(ExtUtils::Embed\) rng-tools lrzsz openssl-devel popt-devel
1.2 编译、定制支持fullnat的内核(三台服务器操作)
参考文档: https://blog.csdn.net/dengyuelin/article/details/54628774 https://blog.csdn.net/sinat_36888624/article/details/79144364 http://blog.51cto.com/shanks/1536539
1.2.1 安装kernel-2.6.32-220.el6.src.rpm
[root@lvs-node01 ~]# cd /usr/local/src/ [root@lvs-node01 src]# rpm -ivh kernel-2.6.32-220.el6.src.rpm
1.2.2 生成内核源码
默认的，你会在root家目录下看到rpmbuild目
cd ~/rpmbuild/SPECS rpmbuild -bp kernel.spec
##说明：rng-tools用于在执行rpmbuild -bb --target=`uname -m` kernel.spec的时候生成随机数，不然会卡在那里，但是根据卡的地方倒退回去会看到提示就执行rngd -r /dev/hwrandom，不行的话执行 rngd -r /dev/urandom，因此需要安装此工具
1.2.3 对生成的内核源码打patch默认的
#在/usr/local/src下解压Lvs-fullnat-synproxy.tar.gz
[root@lvs-node01 src]# cd /usr/local/src [root@lvs-node01 src]# tar xf Lvs-fullnat-synproxy.tar.gz [root@lvs-node01 src]# cd ~/rpmbuild/BUILD/kernel-2.6.32-220.el6/linux-2.6.32-220.el6.x86_64/ [root@lvs-node01 linux-2.6.32-220.el6.x86_64]# cp /usr/local/src/lvs-fullnat-synproxy/lvs-2.6.32-220.23.1.el6.patch ./ [root@lvs-node01 linux-2.6.32-220.el6.x86_64]# patch -p1 < lvs-2.6.32-220.23.1.el6.patch
#淘宝将IP_VS改成了22，测试时遇到些麻烦，因此改为20了。 #vim .config CONFIG_IP_VS_TAB_BITS=20 #你可以修改Makefile把内核的名称做下标记（line：4） EXTRAVERSION = .FNAT.shanks.e27.x86_64
1.2.4 Make
make -j16 make modules_install make install
1.2.5 配置grub.conf
#vim /boot/grub/grub.conf default=0
1.2.6 重启服务器reboot
reboot之后uname -r
[root@lvs-node01 ~]# uname -r 2.6.32 [root@lvs-node01 ~]# ipvsadm -L IP Virtual Server version 1.2.1 (size=4194304)(编译之后变成了这些 编译之前为4096) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn
1.3 基于fullnat配置keepalived，分为两种模式：主备模式、集群模式（OSPF）
文章中介绍了如何配置fullnat的keepalived，分为两种模式。
主备模式：active/standby只有一台fullant跑业务，另外一台热备状态
集群模式：active/active...多台fullnat一起跑业务。
http://shanks.blog.51cto.com/3899909/1387469
1.3.1 安装keepalived
[root@lvs-node01 src]# cd /usr/local/src/lvs-fullnat-synproxy/ [root@lvs-node01 lvs-fullnat-synproxy]# tar xf lvs-tools.tar.gz [root@lvs-node01 lvs-fullnat-synproxy]# cd tools/keepalived/ #安装依赖包： yum install -y openssl-devel popt-devel #编译keepalived ./configure --with-kernel-dir="/lib/modules/`uname -r`/build" make make install #复制基本配置 mkdir /etc/keepalived/ -p cp -a bin/genhash /usr/local/bin/ cp -a bin/keepalived /sbin/ cp -a keepalived/etc/init.d/keepalived.init /etc/init.d/keepalived cp -a keepalived/etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf cp -a keepalived/etc/init.d/keepalived.sysconfig /etc/sysconfig/keepalived
1.3.2 安装ipvsadm
[root@lvs-node01 ipvsadm]# cd /usr/local/src/lvs-fullnat-synproxy/tools/ipvsadm/ [root@lvs-node01 ipvsadm]# make && make install
1.3.3 系统自身参数配置
打开irqbalance # service irqbalance start # chkconfig --level 2345 irqbalance on vim /etc/sysctl.conf # configure for lvs net.ipv4.conf.all.arp_ignore = 1 net.ipv4.conf.all.arp_announce = 2 net.core.netdev_max_backlog = 500000 sysctl -p
1.3.4 keepalived三台服务器配置文件(主备模式)
1.3.4.1 lvs-node01配置文件
[root@lvs-node01 ipvsadm]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id lvs-node01
}
local_address_group laddr_g1 {
10.0.0.31
}
virtual_server_group shanks1 {
10.0.0.129 80
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 150
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.129 dev eth0 label eth0:1
}
}
virtual_server 10.0.0.129 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1
real_server 10.0.0.33 80 {
weight 10
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_prot 80
}
}
virtual_server 10.0.0.129 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1
real_server 10.0.0.32 80 {
weight 10
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_prot 80
}
}
}
1.3.4.2 lvs-node02配置文件
[root@lvs-node02 ~]# cat /etc/keepalived/keepalived.conf
global_defs {
router_id lvs-node02
}
local_address_group laddr_g1 {
10.0.0.32
}
virtual_server_group shanks1 {
10.0.0.129 80
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.129 dev eth0 label eth0:1
}
}
virtual_server 10.0.0.129 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1
real_server 10.0.0.33 80 {
weight 10
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_prot 80
}
}
virtual_server 10.0.0.129 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1
real_server 10.0.0.32 80 {
weight 10
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_prot 80
}
}
}
1.3.4.3 lvs-node03配置文件
[root@lvs-node03 ~]# cat /etc/keepalived/keepalived.conf
global_defs {
router_id lvs-node03
}
local_address_group laddr_g1 {
10.0.0.33
}
virtual_server_group shanks1 {
10.0.0.129 80
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.0.129 dev eth0 label eth0:1
}
}
virtual_server 10.0.0.129 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1
real_server 10.0.0.33 80 {
weight 10
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_prot 80
}
}
virtual_server 10.0.0.129 80 {
delay_loop 6
lb_algo rr
lb_kind FNAT
protocol TCP
syn_proxy
laddr_group_name laddr_g1
real_server 10.0.0.32 80 {
weight 10
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_prot 80
}
}
}
1.3.5 配置文件配置完成启动keepalived
[root@lvs-node03 ~]# /etc/init.d/keepalived start Starting keepalived: [ OK ] [root@lvs-node03 ~]# chkconfig keepalived on [root@lvs-node03 ~]# chkconfig --list keepalived keepalived 0:off 1:off 2:on 3:on 4:on 5:on 6:off
1.3.6 检查fullnat模式效果
[root@lvs-node01 ipvsadm]# ipvsadm -L IP Virtual Server version 1.2.1 (size=4194304) Prot LocalAddress:Port Scheduler Flags -> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP 10.0.0.129:http rr synproxy -> lvs-node02:http FullNat 10 0 0 -> lvs-node03:http FullNat 10 0 0
浏览器访问测试
http://10.0.0.129
