1. 前言
本文承接上文[docker 网络] docker跨主机网络 ovs gre 测试1, 将对ovs使用gre实现docker跨主机容器访问继续进行测试. [docker 网络] docker跨主机网络 ovs gre 测试1中容器的网络在同一个子网中, 本文将会测试容器在不同子网中如何实现的.
需要对docker网络类型有基本了解, 可以参考[mydocker]---docker的四种网络模型与原理实现(1) 和 [mydocker]---docker的四种网络模型与原理实现(2).
1.1 当前环境
vm1
[root@vm1 ~]# cat /proc/sys/net/ipv4/ip_forward
0
[root@vm1 ~]# iptables -t nat -F
[root@vm1 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
[root@vm1 ~]# ovs-vsctl show
91e815a1-1021-4c97-a21c-893ab8c28e37
ovs_version: "2.5.1"
[root@vm1 ~]#
vm2
[root@vm2 ~]# echo 0 > /proc/sys/net/ipv4/ip_forward
[root@vm2 ~]#
[root@vm2 ~]# cat /proc/sys/net/ipv4/ip_forward
0
[root@vm2 ~]# iptables -t nat -F
[root@vm2 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
[root@vm2 ~]# ovs-vsctl show
533800d4-246f-4099-a776-8254610db91f
ovs_version: "2.5.1"
[root@vm2 ~]#
2. ovs 容器在不同子网中
2.1 vm1中设置
[root@vm1 ~]# ip link add docker0 type bridge
[root@vm1 ~]# ip addr add 172.17.1.254/24 dev docker0
[root@vm1 ~]# ip link set docker0 up
[root@vm1 ~]# ip netns add ns1
[root@vm1 ~]# ip link add veth0 type veth peer name veth1
[root@vm1 ~]# brctl addif docker0 veth0
[root@vm1 ~]# ip link set veth1 netns ns1
[root@vm1 ~]# ip link set veth0 up
[root@vm1 ~]# ip netns exec ns1 sh
sh-4.2# ip addr add 172.17.1.1/24 dev veth1
sh-4.2# ip link set veth1 up
sh-4.2# ip link set lo up
sh-4.2# route add default gw 172.17.1.254
sh-4.2# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.17.1.254 0.0.0.0 UG 0 0 0 veth1
172.17.1.0 0.0.0.0 255.255.255.0 U 0 0 0 veth1
sh-4.2# ping -c 1 172.17.1.254
PING 172.17.1.254 (172.17.1.254) 56(84) bytes of data.
64 bytes from 172.17.1.254: icmp_seq=1 ttl=64 time=0.078 ms
--- 172.17.1.254 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.078/0.078/0.078/0.000 ms
sh-4.2# ping -c 1 172.19.0.12
PING 172.19.0.12 (172.19.0.12) 56(84) bytes of data.
64 bytes from 172.19.0.12: icmp_seq=1 ttl=64 time=0.048 ms
--- 172.19.0.12 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.048/0.048/0.048/0.000 ms
sh-4.2# exit
exit
[root@vm1 ~]#
2.2 vm2中设置
[root@vm2 ~]# ip link add docker0 type bridge
[root@vm2 ~]# ip addr add 192.168.2.254/24 dev docker0
[root@vm2 ~]# ip link set docker0 up
[root@vm2 ~]# ip netns add ns1
[root@vm2 ~]# ip link add veth0 type veth peer name veth1
[root@vm2 ~]# brctl addif docker0 veth0
[root@vm2 ~]# ip link set veth1 netns ns1
[root@vm2 ~]# ip link set veth0 up
[root@vm2 ~]# ip netns exec ns1 sh
sh-4.2# ip addr add 192.168.2.1/24 dev veth1
sh-4.2# ip link set veth1 up
sh-4.2# ip link set lo up
sh-4.2# route add default gw 192.168.2.254
sh-4.2# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.2.254 0.0.0.0 UG 0 0 0 veth1
192.168.2.0 0.0.0.0 255.255.255.0 U 0 0 0 veth1
sh-4.2# ping -c 1 192.168.2.254
PING 192.168.2.254 (192.168.2.254) 56(84) bytes of data.
64 bytes from 192.168.2.254: icmp_seq=1 ttl=64 time=0.052 ms
--- 192.168.2.254 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.052/0.052/0.052/0.000 ms
sh-4.2# ping -c 1 172.19.0.8
PING 172.19.0.8 (172.19.0.8) 56(84) bytes of data.
64 bytes from 172.19.0.8: icmp_seq=1 ttl=64 time=0.031 ms
--- 172.19.0.8 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.031/0.031/0.031/0.000 ms
sh-4.2# exit
exit
[root@vm2 ~]#
图片.png
2.3 加入gre配置
很显然目前两个容器是
ping不通的.
[root@vm1 ~]# ping -c 1 192.168.2.254
PING 192.168.2.254 (192.168.2.254) 56(84) bytes of data.
--- 192.168.2.254 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
[root@vm1 ~]# ping -c 1 192.168.2.1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
--- 192.168.2.1 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
[root@vm1 ~]#
2.3.1 增加一个ovs网桥
vm1
[root@vm1 ~]# ovs-vsctl add-br ovs1
[root@vm1 ~]#
[root@vm1 ~]# ovs-vsctl add-port ovs1 rou1 -- set interface rou1 type=internal
[root@vm1 ~]#
[root@vm1 ~]# ifconfig rou1 192.168.1.1/24
[root@vm1 ~]#
[root@vm1 ~]# ovs-vsctl show
91e815a1-1021-4c97-a21c-893ab8c28e37
Bridge "ovs1"
Port "rou1"
Interface "rou1"
type: internal
Port "ovs1"
Interface "ovs1"
type: internal
ovs_version: "2.5.1"
[root@vm1 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.17.1.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 rou1
[root@vm1 ~]#
vm2
[root@vm2 ~]# ovs-vsctl add-br ovs2
[root@vm2 ~]# ovs-vsctl add-port ovs2 rou2 -- set interface rou2 type=internal
[root@vm2 ~]# ifconfig rou2 192.168.1.2/24
[root@vm2 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 rou2
192.168.2.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
[root@vm2 ~]# ovs-vsctl show
533800d4-246f-4099-a776-8254610db91f
Bridge "ovs2"
Port "rou2"
Interface "rou2"
type: internal
Port "ovs2"
Interface "ovs2"
type: internal
ovs_version: "2.5.1"
在vm1中访问vm2中的rou2
[root@vm1 ~]# ping -c 1 192.168.1.1
PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=64 time=0.036 ms
--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.036/0.036/0.036/0.000 ms
[root@vm1 ~]# ping -c 1 192.168.1.2
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
From 192.168.1.1 icmp_seq=1 Destination Host Unreachable
--- 192.168.1.2 ping statistics ---
1 packets transmitted, 0 received, +1 errors, 100% packet loss, time 0ms
[root@vm1 ~]#
图片.png
2.3.2 加入gre tunnel
vm1
[root@vm1 ~]# ovs-vsctl add-port ovs1 gre1 -- set interface gre1 type=gre options:remote_ip=172.19.0.8
[root@vm1 ~]#
[root@vm1 ~]# ovs-vsctl show
91e815a1-1021-4c97-a21c-893ab8c28e37
Bridge "ovs1"
Port "gre1"
Interface "gre1"
type: gre
options: {remote_ip="172.19.0.8"}
Port "rou1"
Interface "rou1"
type: internal
Port "ovs1"
Interface "ovs1"
type: internal
ovs_version: "2.5.1"
[root@vm1 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.17.1.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 rou1
vm2
[root@vm2 ~]# ovs-vsctl add-port ovs2 gre2 -- set interface gre2 type=gre options:remote_ip=172.19.0.12
[root@vm2 ~]# ovs-vsctl show
533800d4-246f-4099-a776-8254610db91f
Bridge "ovs2"
Port "gre2"
Interface "gre2"
type: gre
options: {remote_ip="172.19.0.12"}
Port "rou2"
Interface "rou2"
type: internal
Port "ovs2"
Interface "ovs2"
type: internal
ovs_version: "2.5.1"
[root@vm2 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 rou2
192.168.2.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
[root@vm2 ~]#
图片.png
测试
// 访问vm2中的rou2
[root@vm1 ~]# ping -c 1 192.168.1.2
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
64 bytes from 192.168.1.2: icmp_seq=1 ttl=64 time=1.30 ms
--- 192.168.1.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.309/1.309/1.309/0.000 ms
// 访问本机中的rou1
[root@vm1 ~]# ping -c 1 192.168.1.1
PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=64 time=0.026 ms
--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.026/0.026/0.026/0.000 ms
[root@vm1 ~]#
vm2
// 访问vm1中的rou1
[root@vm2 ~]# ping -c 1 192.168.1.1
PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=64 time=0.691 ms
--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.691/0.691/0.691/0.000 ms
// 访问本机中的rou2
[root@vm2 ~]# ping -c 1 192.168.1.2
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
64 bytes from 192.168.1.2: icmp_seq=1 ttl=64 time=0.028 ms
--- 192.168.1.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.028/0.028/0.028/0.000 ms
[root@vm2 ~]#
2.3.3 将ovs 绑定到 docker0中
vm1
[root@vm1 ~]# brctl addif docker0 ovs1
[root@vm1 ~]# ip link set ovs1 up
[root@vm1 ~]# bridge link
16: veth0 state UP @(null): <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master docker0 state forwarding priority 32 cost 2
22: ovs1 state UNKNOWN : <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master docker0 state forwarding priority 32 cost 100
[root@vm1 ~]#
vm2
[root@vm2 ~]# brctl addif docker0 ovs2
[root@vm2 ~]# ip link set ovs2 up
[root@vm2 ~]# bridge link
16: veth0 state UP @(null): <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master docker0 state forwarding priority 32 cost 2
22: ovs2 state UNKNOWN : <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 master docker0 state forwarding priority 32 cost 100
[root@vm1 ~]#
图片.png
测试, 还是ping不通, 这是因为不在同一个网络上, 在[[docker 网络]ovs gre 测试1]中已经测试过.
[root@vm1 ~]# ping -c 1 192.168.2.1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
--- 192.168.2.1 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
[root@vm1 ~]# ping -c 1 192.168.2.254
PING 192.168.2.254 (192.168.2.254) 56(84) bytes of data.
--- 192.168.2.254 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
[root@vm1 ~]#
2.3.4 加入路由
vm1
[root@vm1 ~]# route add -net 192.168.2.0/24 dev rou1
[root@vm1 ~]#
[root@vm1 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.17.1.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 rou1
192.168.2.0 0.0.0.0 255.255.255.0 U 0 0 0 rou1
vm2
[root@vm2 ~]# route add -net 172.17.1.0/24 dev rou2
[root@vm2 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.19.0.1 0.0.0.0 UG 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 1002 0 0 eth0
172.17.1.0 0.0.0.0 255.255.255.0 U 0 0 0 rou2
172.19.0.0 0.0.0.0 255.255.240.0 U 0 0 0 eth0
192.168.1.0 0.0.0.0 255.255.255.0 U 0 0 0 rou2
192.168.2.0 0.0.0.0 255.255.255.0 U 0 0 0 docker0
[root@vm2 ~]#
测试 从vm1中访问
// ping vm2中的rou2可以通
[root@vm1 ~]# ping -c 1 192.168.2.254
PING 192.168.2.254 (192.168.2.254) 56(84) bytes of data.
64 bytes from 192.168.2.254: icmp_seq=1 ttl=64 time=1.11 ms
--- 192.168.2.254 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 1.111/1.111/1.111/0.000 ms
// ping vm2中的ns1不通 因为vm2中没有开通ip_forward功能
[root@vm1 ~]# ping -c 1 192.168.2.1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
--- 192.168.2.1 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
在vm2中加入ip_forward功能
[root@vm2 ~]# echo 1 > /proc/sys/net/ipv4/ip_forward
再次测试
[root@vm1 ~]# ping -c 1 192.168.2.1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
64 bytes from 192.168.2.1: icmp_seq=1 ttl=63 time=0.709 ms
--- 192.168.2.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.709/0.709/0.709/0.000 ms
同样的道理, 也需要给vm1加入ip_forward功能
[root@vm1 ~]# echo 1 > /proc/sys/net/ipv4/ip_forward
图片.png
2.3.5 容器之间访问
[root@vm1 ~]# ip netns exec ns1 sh
sh-4.2# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 172.17.1.254 0.0.0.0 UG 0 0 0 veth1
172.17.1.0 0.0.0.0 255.255.255.0 U 0 0 0 veth1
// 访问本机的docker0
sh-4.2# ping -c 1 172.17.1.254
PING 172.17.1.254 (172.17.1.254) 56(84) bytes of data.
64 bytes from 172.17.1.254: icmp_seq=1 ttl=64 time=0.050 ms
--- 172.17.1.254 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.050/0.050/0.050/0.000 ms
// 访问本机的ns1
sh-4.2# ping -c 1 172.17.1.1
PING 172.17.1.1 (172.17.1.1) 56(84) bytes of data.
64 bytes from 172.17.1.1: icmp_seq=1 ttl=64 time=0.026 ms
--- 172.17.1.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.026/0.026/0.026/0.000 ms
// 访问本机的ip
sh-4.2# ping -c 1 172.19.0.12
PING 172.19.0.12 (172.19.0.12) 56(84) bytes of data.
64 bytes from 172.19.0.12: icmp_seq=1 ttl=64 time=0.038 ms
--- 172.19.0.12 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.038/0.038/0.038/0.000 ms
// 访问本机的rou1
sh-4.2# ping -c 1 192.168.1.1
PING 192.168.1.1 (192.168.1.1) 56(84) bytes of data.
64 bytes from 192.168.1.1: icmp_seq=1 ttl=64 time=0.044 ms
--- 192.168.1.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.044/0.044/0.044/0.000 ms
// 访问vm2的rou2
sh-4.2# ping -c 1 192.168.1.2
PING 192.168.1.2 (192.168.1.2) 56(84) bytes of data.
64 bytes from 192.168.1.2: icmp_seq=1 ttl=64 time=0.760 ms
--- 192.168.1.2 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.760/0.760/0.760/0.000 ms
// 访问vm2的docker0
sh-4.2# ping -c 1 192.168.2.254
PING 192.168.2.254 (192.168.2.254) 56(84) bytes of data.
64 bytes from 192.168.2.254: icmp_seq=1 ttl=64 time=0.353 ms
--- 192.168.2.254 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.353/0.353/0.353/0.000 ms
// 访问vm2的ns1
sh-4.2# ping -c 1 192.168.2.1
PING 192.168.2.1 (192.168.2.1) 56(84) bytes of data.
64 bytes from 192.168.2.1: icmp_seq=1 ttl=63 time=0.624 ms
--- 192.168.2.1 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.624/0.624/0.624/0.000 ms
// 访问vm2的ip 需要加入iptables规则
sh-4.2# ping -c 1 172.19.0.8
PING 172.19.0.8 (172.19.0.8) 56(84) bytes of data.
--- 172.19.0.8 ping statistics ---
1 packets transmitted, 0 received, 100% packet loss, time 0ms
2.3.6 加入iptables规则
[root@vm1 ~]# iptables -t nat -A POSTROUTING -s 172.17.1.0/24 -o eth0 -j MASQUERADE
[root@vm1 ~]#
[root@vm1 ~]# ip netns exec ns1 ping -c 1 172.19.0.8
PING 172.19.0.8 (172.19.0.8) 56(84) bytes of data.
64 bytes from 172.19.0.8: icmp_seq=1 ttl=63 time=0.380 ms
--- 172.19.0.8 ping statistics ---
1 packets transmitted, 1 received, 0% packet loss, time 0ms
rtt min/avg/max/mdev = 0.380/0.380/0.380/0.000 ms
[root@vm1 ~]#
给vm2中加入
[root@vm2 ~]# iptables -t nat -A POSTROUTING -s 192.168.2.0/24 -o eth0 -j MASQUERADE
图片.png
3. 参考
1. https://blog.csdn.net/wodeamd1/article/details/81282437
2. https://blog.csdn.net/song7999/article/details/80403527
3. Docker 容器与容器云
4. https://blog.csdn.net/qq_27366789/article/details/83348366