上一篇文章记录过，LSP 的 port-security 本身也是 acl， 而安全组也是 acl 的集合。 但是这两个 annotation 本质上是要分开使用的。所以尝试将这两个功能独立下。
测试
应用如下模版：
root@empty:~/test1/sg# cat sg.yaml
apiVersion: kubeovn.io/v1
kind: SecurityGroup
metadata:
name: sg-example
spec:
allowSameGroupTraffic: true
egressRules:
- ipVersion: ipv4
policy: allow
priority: 1
protocol: all
remoteAddress: 10.16.0.13 # 10.16.0.0/16 配置网段
remoteType: address
ingressRules:
- ipVersion: ipv4
policy: deny
priority: 1
protocol: icmp
remoteAddress: 10.16.0.14
remoteType: address
root@empty:~/test1/sg# cat podsg1.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: podsg1
namespace: default
annotations:
ovn.kubernetes.io/logical_switch: ovn-default
ovn.kubernetes.io/ip_address: 10.16.0.13
ovn.kubernetes.io/port_security: "true"
ovn.kubernetes.io/security_groups: sg-example
spec:
containers:
- name: netshoot
image: nicolaka/netshoot:latest
imagePullPolicy: Never
command:
- sh
- -c
- "sleep infinity"
securityContext:
capabilities:
add:
- NET_ADMIN
root@empty:~/test1/sg# cat podsg2.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: podsg2
namespace: default
annotations:
ovn.kubernetes.io/logical_switch: ovn-default
ovn.kubernetes.io/ip_address: 10.16.0.14
ovn.kubernetes.io/port_security: "true"
ovn.kubernetes.io/security_groups: sg-example
spec:
containers:
- name: netshoot
image: nicolaka/netshoot:latest
imagePullPolicy: Never
command:
- sh
- -c
- "sleep infinity"
securityContext:
capabilities:
add:
- NET_ADMIN
root@empty:~/test1/sg# cat pod-no-port-security.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: podsg-no-port-sc
namespace: default
annotations:
ovn.kubernetes.io/logical_switch: ovn-default
ovn.kubernetes.io/ip_address: 10.16.0.124
ovn.kubernetes.io/port_security: "false"
ovn.kubernetes.io/security_groups: sg-example
spec:
containers:
- name: netshoot
image: nicolaka/netshoot:latest
imagePullPolicy: Never
command:
- sh
- -c
- "sleep infinity"
securityContext:
capabilities:
add:
- NET_ADMIN
root@empty:~/test1/sg#
应用如上脚本之后：
root@empty:~/feat/kube-ovn# k get sg
NAME AGE
sg-example 8m1s
root@empty:~/feat/kube-ovn#
root@empty:~/feat/kube-ovn#
root@empty:~/feat/kube-ovn# k ko nbctl list port_group
_uuid : ed902ca8-0cae-4bc0-be22-454cbaf72fa4
acls : [26c21ac6-cf96-4e22-ab5a-71773b8f7a08, 69504cbc-fbe9-4725-b6f3-d8181c8f5f0e, 6f3666ae-78b2-4b14-bb2a-ec86a623a5d5, 7e39563a-c830-449d-a640-e2fed8550d82, 81102acc-f189-48c0-a880-c7fbc2e96476, 9846ea63-a4f3-4800-b05b-dfc06eaae370, 9ee1a18b-aa33-4310-802c-98ac835be589, a6cbc0d2-72e2-44e1-a7ce-23b261f3af32, a78d7cd0-80ef-49a6-a289-09d5514c5f67, aee4e5e3-5dda-46bd-a797-31d6e8ca2da4, b9d7745a-49fb-4bc0-9fde-3888fffed0f2, cbbb5b5e-24e5-4aad-9b12-69e63bb75912, d18318e2-14a6-4d61-afb7-67cfe04cdf24, d2c810e7-4cdc-402b-a2cd-8d6658f735e7, d41c922f-59e3-4efb-ab5b-abdce6b32b1c, fe3a7964-a81f-46a2-8345-7f083f36f18d]
external_ids : {sg=sg-example, type=security_group}
name : ovn.sg.sg.example
ports : [0fc25e08-b7fa-4c83-be6d-affb50724a08, 6450dd9e-a5bb-40ad-8b86-a36c451dcd3d]
# 可以看到 sg-example 只限制到 两个 port，但是我有三个 pod 都使用了这个安全组。
root@empty:~/feat/kube-ovn# k ko nbctl list logical_switch_port | grep -A 15 -E "0fc25e08-b7fa-4c83-be6d-affb50724a08|6450dd9e-a5bb-40ad-8b86-a36c451dcd3d"
_uuid : 0fc25e08-b7fa-4c83-be6d-affb50724a08
addresses : ["00:00:00:99:09:EF 10.16.0.13"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_sg-example="true", ls=ovn-default, pod="default/podsg1", security_groups=sg-example, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : podsg1.default
options : {}
parent_name : []
port_security : ["00:00:00:99:09:EF 10.16.0.13"]
tag : []
tag_request : []
type : ""
--
_uuid : 6450dd9e-a5bb-40ad-8b86-a36c451dcd3d
addresses : ["00:00:00:50:05:32 10.16.0.14"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_sg-example="true", ls=ovn-default, pod="default/podsg2", security_groups=sg-example, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : podsg2.default
options : {}
parent_name : []
port_security : ["00:00:00:50:05:32 10.16.0.14"]
tag : []
tag_request : []
type : ""
所以目前还不太对，再调调，如下，已符合预期
_uuid : 2c474848-1c26-46e9-a59a-313a845063cb
acls : [078fad57-7d48-4a3a-9b30-a1f7dd6bb7a8, 0d2f325c-bd25-4623-80f3-4c4d93546e8b, 0e2e0994-0f8a-4df1-abaf-44bec6d705b9, 2e549568-0d7b-482c-8efd-ae5920235438, 2fb46b2b-5ea3-4dfe-b9b6-14e8c01fbfab, 4663fd72-a405-40f4-861b-0cb9233ad114, 474b29d9-886f-46fc-9913-d2dac28a7832, 5a100511-4138-4a5f-8f44-bb3749c64ef7, 5bad743f-b44d-4b95-b4df-28d05d055636, 7ea0fd49-0cc4-4f76-a90f-b71a817c5019, 83bd23e0-01dc-41a0-ba47-20cc0eb0a565, 960ddc2f-9b8a-43e1-a7a9-3f509500157c, b2fdf39f-0470-4632-ad39-051d2e272ffd, d7f39ae6-aedf-4fc5-ab93-413e2b7c896e, dc91046a-d2e5-4739-96e5-09950e72bf0a, f08b3d7d-8b42-446e-9384-8f4510e8185c]
external_ids : {sg=sg-example, type=security_group}
name : ovn.sg.sg.example
ports : [45acb4a9-83d6-4a48-8aab-bc06487fc3b8, 54fb6c82-1319-4357-8dff-f11588e73be0, d7d7a7cf-3a20-41f6-98a6-958c672b2e37]
root@empty:~/test1/sg# k ko nbctl list logical_switch_port | grep -A 15 -E "45acb4a9-83d6-4a48-8aab-bc06487fc3b8|54fb6c82-1319-4357-8dff-f11588e73be0|d7d7a7cf-3a20-41f6-98a6-958c672b2e37"
_uuid : 45acb4a9-83d6-4a48-8aab-bc06487fc3b8
addresses : ["00:00:00:3C:2F:24 10.16.0.13"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_sg-example="true", ls=ovn-default, pod="default/podsg1", security_groups=sg-example, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : podsg1.default
options : {}
parent_name : []
port_security : ["00:00:00:3C:2F:24 10.16.0.13"]
tag : []
tag_request : []
type : ""
--
_uuid : 54fb6c82-1319-4357-8dff-f11588e73be0
addresses : ["00:00:00:54:01:9E 10.16.0.124"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_sg-example="true", ls=ovn-default, pod="default/podsg-no-port-sc", security_groups=sg-example, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : podsg-no-port-sc.default
options : {}
parent_name : []
port_security : [] # 如果关闭了 port_security 这个数组是空的
tag : []
tag_request : []
type : ""
--
_uuid : d7d7a7cf-3a20-41f6-98a6-958c672b2e37
addresses : ["00:00:00:D7:80:D4 10.16.0.14"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_sg-example="true", ls=ovn-default, pod="default/podsg2", security_groups=sg-example, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : podsg2.default
options : {}
parent_name : []
port_security : ["00:00:00:D7:80:D4 10.16.0.14"]
tag : []
tag_request : []
type : ""
1. 验证 关闭了 port_security ， 随意 ip 地址都可以发包出来, 不受限制
给测试pod配一个随意ip，然后在 kube-ovn-cni 抓包，发现随意ip的包有发出来
podsg-no-port-sc:~# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
20: eth0@if21: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1400 qdisc noqueue state UP group default
link/ether 00:00:00:54:01:9e brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.16.0.124/16 brd 10.16.255.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::200:ff:fe54:19e/64 scope link
valid_lft forever preferred_lft forever
# 原来的ip 是 10.16.0.124 ，把它改成随意 ip
20: eth0@if21: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1400 qdisc noqueue state UP group default
link/ether 00:00:00:54:01:9e brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.16.0.224/16 scope global eth0
valid_lft forever preferred_lft forever
inet6 fe80::200:ff:fe54:19e/64 scope link
valid_lft forever preferred_lft forever
然后 ping 网关，你发现它是可以通的
root@kube-ovn-worker:/kube-ovn# tcpdump -i any host 10.16.0.224
tcpdump: data link type LINUX_SLL2
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
08:23:22.794644 961df28c4a98_h P IP 10.16.0.224 > 10.16.0.1: ICMP echo request, id 13891, seq 1, length 64
08:23:22.795170 961df28c4a98_h Out IP 10.16.0.1 > 10.16.0.224: ICMP echo reply, id 13891, seq 1, length 64
08:23:27.858699 961df28c4a98_h P ARP, Request who-has 10.16.0.1 tell 10.16.0.224, length 28
08:23:27.859274 961df28c4a98_h Out ARP, Reply 10.16.0.1 is-at 00:00:00:7d:1a:20 (oui Ethernet), length 28
2. 验证 开启了 port_security ， 随意 ip 地址不能发包都出来, 受限制
给测试pod配一个随意ip，然后在 kube-ovn-cni 抓包，发现随意ip的包发不出来
_uuid : 54fb6c82-1319-4357-8dff-f11588e73be0
addresses : ["00:00:00:54:01:9E 10.16.0.124"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_sg-example="true", ls=ovn-default, pod="default/podsg-no-port-sc", security_groups=sg-example, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : podsg-no-port-sc.default
options : {}
parent_name : []
port_security : ["00:00:00:54:01:9E 10.16.0.124"]# 开启之后，这里会加限制
tag : []
tag_request : []
type : ""
--
然后 再ping 就不通了
root@kube-ovn-worker:/kube-ovn# tcpdump -i any host 10.16.0.224
tcpdump: data link type LINUX_SLL2
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on any, link-type LINUX_SLL2 (Linux cooked v2), snapshot length 262144 bytes
08:25:20.342701 961df28c4a98_h P IP 10.16.0.224 > 10.16.0.1: ICMP echo request, id 57990, seq 1, length 64
08:25:25.362536 961df28c4a98_h P
# 关闭安全性之后，又可以通了
符合预期
