1. 目前存在两种静态路由，这两种静态路由都可以出公网



# k ko nbctl lr-route-list vpc2
IPv4 Routes
Route Table <main>:
           192.168.0.0/24              10.5.204.108 src-ip ecmp ecmp-symmetric-reply bfd
           192.168.0.0/24              10.5.204.109 src-ip ecmp ecmp-symmetric-reply bfd
           192.168.0.0/24              10.5.204.121 src-ip ecmp ecmp-symmetric-reply bfd
                0.0.0.0/0              10.5.204.254 dst-ip



# 可以看到vpc内部的pod是通过普通静态路由出去的


[root@pc-node-1 ~]# k exec -it -n vpc2            vpc-1-busybox01     bash
kubectl exec [POD] [COMMAND] is DEPRECATED and will be removed in a future version. Use kubectl exec [POD] -- [COMMAND] instead.
[root@vpc-1-busybox01 /]#
[root@vpc-1-busybox01 /]#
[root@vpc-1-busybox01 /]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
215833: eth0@if215834: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
    link/ether 00:00:00:be:bf:c7 brd ff:ff:ff:ff:ff:ff link-netnsid 0
    inet 192.168.0.2/24 brd 192.168.0.255 scope global eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::200:ff:febe:bfc7/64 scope link
       valid_lft forever preferred_lft forever
[root@vpc-1-busybox01 /]# ping 223.5.5.5
PING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.
64 bytes from 223.5.5.5: icmp_seq=1 ttl=114 time=23.1 ms
64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=23.0 ms
^C
--- 223.5.5.5 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev = 23.058/23.103/23.149/0.158 ms
[root@vpc-1-busybox01 /]# tracepath -n 223.5.5.5
 1?: [LOCALHOST]                                         pmtu 1500
 1:  192.168.0.1                                           0.598ms asymm  2
 1:  192.168.0.1                                           0.348ms asymm  2
 2:  10.5.204.254                                          1.512ms # 注意这一跳
 3:  202.109.248.193                                       2.792ms
 4:  117.30.27.33                                          8.072ms
 5:  218.85.141.133                                        5.779ms asymm  4
 6:  61.154.236.5                                          6.429ms
 7:  202.97.100.217                                       20.525ms
 8:  no reply
 9:  no reply
10:  180.163.53.74                                        31.939ms
11:  116.251.88.154                                       21.910ms asymm 13
12:  116.251.116.81                                       22.668ms

2. 将普通静态路由删除，可以看到才会基于ecmp 静态路由出去



[root@vpc-1-busybox01 /]# ping 223.5.5.5
PING 223.5.5.5 (223.5.5.5) 56(84) bytes of data.
64 bytes from 223.5.5.5: icmp_seq=1 ttl=114 time=23.4 ms
64 bytes from 223.5.5.5: icmp_seq=2 ttl=114 time=23.0 ms
^C
--- 223.5.5.5 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1001ms
rtt min/avg/max/mdev = 23.067/23.253/23.440/0.240 ms
[root@vpc-1-busybox01 /]# tracepath -n 223.5.5.5
 1?: [LOCALHOST]                                         pmtu 1500
 1:  192.168.0.1                                           0.611ms asymm  2
 1:  192.168.0.1                                           0.332ms asymm  2
 2:  no reply
 3:  no reply

# 可以看到是没有基于 10.5.204.254   直接出去

[root@pc-node-1 ~]# tcpdump -i ovnext0 host 223.5.5.5 -netvv
dropped privs to tcpdump
tcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes







^C
0 packets captured
0 packets received by filter
0 packets dropped by kernel
[root@pc-node-1 ~]# exit
[root@pc-node-1 ~]# ip netns exec ovnext bash^C
[root@pc-node-1 ~]# ssh pc-node-2
Last login: Mon Feb 20 08:07:05 2023 from 10.5.32.51
[root@pc-node-2 ~]#  ip netns exec ovnext bash
[root@pc-node-2 ~]#
[root@pc-node-2 ~]#
[root@pc-node-2 ~]#  tcpdump -i ovnext0 host 223.5.5.5 -netvv
dropped privs to tcpdump
tcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes


^C
0 packets captured
1 packet received by filter
0 packets dropped by kernel
[root@pc-node-2 ~]# exit
[root@pc-node-2 ~]# logout
Connection to pc-node-2 closed.
[root@pc-node-1 ~]# ssh pc-node-3
Last login: Mon Feb 20 08:07:22 2023 from 10.5.32.51
[root@pc-node-3 ~]#  ip netns exec ovnext bash
[root@pc-node-3 ~]#  tcpdump -i ovnext0 host 223.5.5.5 -netvv
dropped privs to tcpdump
tcpdump: listening on ovnext0, link-type EN10MB (Ethernet), capture size 262144 bytes
00:00:00:2d:f8:ce > 00:00:00:fd:b2:a4, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 44808, offset 0, flags [DF], proto ICMP (1), length 84)
    10.5.204.111 > 223.5.5.5: ICMP echo request, id 111, seq 71, length 64
00:00:00:fd:b2:a4 > dc:ef:80:5a:44:1a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 44808, offset 0, flags [DF], proto ICMP (1), length 84)
    10.5.204.111 > 223.5.5.5: ICMP echo request, id 111, seq 71, length 64
00:00:00:2d:f8:ce > 00:00:00:fd:b2:a4, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 63, id 45000, offset 0, flags [DF], proto ICMP (1), length 84)
    10.5.204.111 > 223.5.5.5: ICMP echo request, id 111, seq 72, length 64
00:00:00:fd:b2:a4 > dc:ef:80:5a:44:1a, ethertype IPv4 (0x0800), length 98: (tos 0x0, ttl 62, id 45000, offset 0, flags [DF], proto ICMP (1), length 84)
    10.5.204.111 > 223.5.5.5: ICMP echo request, id 111, seq 72, length 64
^C
4 packets captured
4 packets received by filter
0 packets dropped by kernel


# 可以看到我有是哪个下一跳，目前是基于第三个节点出去的，这个是hash的结果