测试下 kube-ovn 安全组给 pod 用，输出一些关联数据，最后删除 pod ip，触发 ip 删除，安全组相关的数据清理。
1. 创建 一个测试用的 ns 和 vpc subnet
kubectl create ns sg
kubectl apply -f 01-ns1-vpc1-subnet.yaml
root@empty:~/test/sg# cat 01-ns1-vpc1-subnet.yaml
kind: Vpc
apiVersion: kubeovn.io/v1
metadata:
name: sg
spec:
namespaces:
- sg
---
apiVersion: kubeovn.io/v1
kind: Subnet
metadata:
name: sg-subnet1
spec:
cidrBlock: 10.1.0.0/24
default: false
disableGatewayCheck: true
disableInterConnection: true
enableEcmp: false
gatewayNode: ""
gatewayType: distributed
natOutgoing: false
private: false
protocol: IPv4
provider: ovn
vpc: sg
namespaces:
- sg
# k get sg sg -o yaml
apiVersion: kubeovn.io/v1
kind: SecurityGroup
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"kubeovn.io/v1","kind":"SecurityGroup","metadata":{"annotations":{},"name":"sg"},"spec":{"allowSameGroupTraffic":true,"egressRules":[{"ipVersion":"ipv4","policy":"allow","priority":1,"protocol":"all","remoteAddress":"10.1.0.0/24","remoteType":"address"}],"ingressRules":[{"ipVersion":"ipv4","policy":"deny","priority":1,"protocol":"icmp","remoteAddress":"10.1.0.0/24","remoteType":"address"}]}}
creationTimestamp: "2023-12-19T02:49:16Z"
generation: 1
name: sg
resourceVersion: "2237437"
uid: 1ca31613-d083-4694-b138-a674017ff5a9
spec:
allowSameGroupTraffic: true
egressRules:
- ipVersion: ipv4
policy: allow
priority: 1
protocol: all
remoteAddress: 10.1.0.0/24
remoteType: address
ingressRules:
- ipVersion: ipv4
policy: deny
priority: 1
protocol: icmp
remoteAddress: 10.1.0.0/24
remoteType: address
status:
allowSameGroupTraffic: true
egressLastSyncSuccess: true
egressMd5: 9531e2913e13f6bbd9ebaae12478bb47
ingressLastSyncSuccess: true
ingressMd5: 8465449ebcc7ebbef0b3b4a37bdef2e0
portGroup: ovn.sg.sg # 所有配置了 安全组的 pod 的端口都会添加到该 portGroup ovn.sg.sg
2. 创建并使用安全组
---
apiVersion: kubeovn.io/v1
kind: SecurityGroup
metadata:
name: sg
spec:
allowSameGroupTraffic: true
egressRules:
- ipVersion: ipv4
policy: allow
priority: 1
protocol: all
remoteAddress: 10.1.0.0/24 # 10.16.0.0/16 配置网段
remoteType: address
ingressRules:
- ipVersion: ipv4
policy: deny
priority: 1
protocol: icmp
remoteAddress: 10.1.0.0/24
remoteType: address
该安全组相关的 acl
# k ko nbctl list acl
_uuid : 9d71e4c0-e310-4aa7-9480-33fe8b78cdd8
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && icmp6.type == {130, 133, 135, 136} && icmp6.code == 0 && ip.ttl == 255"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 2d68efe1-87e8-47d0-8984-bb14d52ca5d9
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && icmp6.type == {130, 134, 135, 136} && icmp6.code == 0 && ip.ttl == 255"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 14802f81-6b0b-4d85-bf5d-1d27dbf3db6f
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && udp.src == 547 && udp.dst == 546 && ip6"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 3e067012-24a5-47e7-85db-921825a9b0ac
action : drop
direction : from-lport
external_ids : {parent=ovn.sg.kubeovn_deny_all}
label : 0
log : false
match : "inport == @ovn.sg.kubeovn_deny_all && ip"
meter : []
name : []
options : {}
priority : 2003
severity : []
_uuid : 4a557ba3-f296-46c6-94a3-befa9f2bfdf1
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && ip4 && ip4.src == $ovn.sg.sg.associated.v4"
meter : []
name : []
options : {}
priority : 2004
severity : []
_uuid : ab425fb5-3f82-4c7a-884b-033011b54600
action : allow-related
direction : from-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && ip4 && ip4.dst == $ovn.sg.sg.associated.v4"
meter : []
name : []
options : {}
priority : 2004
severity : []
_uuid : d8937f65-6398-4aeb-b990-e9a1b93a4256
action : allow-related
direction : from-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && ip4 && ip4.dst == 10.1.0.0/24"
meter : []
name : []
options : {}
priority : 2299
severity : []
_uuid : e5291967-85df-4554-8e64-332f814d8803
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && udp.src == 67 && udp.dst == 68 && ip4"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 45526498-47fe-4f6a-b76e-67cf86e7596b
action : drop
direction : to-lport
external_ids : {parent=ovn.sg.kubeovn_deny_all}
label : 0
log : false
match : "outport == @ovn.sg.kubeovn_deny_all && ip"
meter : []
name : []
options : {}
priority : 2003
severity : []
_uuid : fef65298-b96d-441b-9fe2-581a940c7721
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && ip6 && ip6.src == $ovn.sg.sg.associated.v6"
meter : []
name : []
options : {}
priority : 2004
severity : []
_uuid : ebfb3779-ac14-404c-9ab3-61d4f4391ff4
action : allow-related
direction : from-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && ip6 && ip6.dst == $ovn.sg.sg.associated.v6"
meter : []
name : []
options : {}
priority : 2004
severity : []
_uuid : f93bf079-df17-44bc-89a4-8eca03bfa997
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && udp.src == 68 && udp.dst == 67 && ip4"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 99b917b9-ac55-457e-806e-1e326ed83d6c
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && arp"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 2e2df429-f3c5-42c4-8068-093e4c0e8ec3
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && ip.proto == 112"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : a7b95f0a-de59-4ec2-85a0-3821908f0bed
action : drop
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && ip4 && ip4.src == 10.1.0.0/24 && icmp4"
meter : []
name : []
options : {}
priority : 2299
severity : []
_uuid : 118681dd-69d3-4b05-9fba-d016d9734ea7
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && arp"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 0e32e76f-7abf-4bec-b135-9b5889a3b816
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "inport == @ovn.sg.sg && udp.src == 546 && udp.dst == 547 && ip6"
meter : []
name : []
options : {}
priority : 2005
severity : []
_uuid : 43cf0e6f-f28e-4091-91c7-872e301836ab
action : allow-related
direction : to-lport
external_ids : {parent=ovn.sg.sg}
label : 0
log : false
match : "outport == @ovn.sg.sg && ip.proto == 112"
meter : []
name : []
options : {}
priority : 2005
severity : []
pod 使用安全组
使用前，查看 portGroup ovn.sg.sg
root@empty:~/test/sg# k ko nbctl list port-group
_uuid : eda4e58c-d57f-4684-a340-cb2dbf9d2f50
acls : []
external_ids : {np="external/empty"}
name : external.empty
ports : []
_uuid : d00452a8-6a74-4464-afe6-4c547a91744d
acls : []
external_ids : {np="ovn-default/empty"}
name : ovn.default.empty
ports : [0e7f7884-1005-4543-b2eb-46fc51ab22d6, 46a46ca0-b1a5-43b8-bb58-35baeb3ac882, 61139bf7-b2e0-4738-861b-cdd9792441c8, 8e38fb59-c966-4a2d-8196-004c4c0ea748, 9e7c3fbf-c834-4004-acf2-bd86b72b77a9, a3f6ef2b-1c83-4b2c-8a79-7dd1d8bcc53a, eac5c379-0d87-4f20-86d4-0124cf682a68]
_uuid : 1d05da99-17f0-480e-827f-36fd7f9ba06a
acls : [0e32e76f-7abf-4bec-b135-9b5889a3b816, 118681dd-69d3-4b05-9fba-d016d9734ea7, 14802f81-6b0b-4d85-bf5d-1d27dbf3db6f, 2d68efe1-87e8-47d0-8984-bb14d52ca5d9, 2e2df429-f3c5-42c4-8068-093e4c0e8ec3, 43cf0e6f-f28e-4091-91c7-872e301836ab, 4a557ba3-f296-46c6-94a3-befa9f2bfdf1, 99b917b9-ac55-457e-806e-1e326ed83d6c, 9d71e4c0-e310-4aa7-9480-33fe8b78cdd8, a7b95f0a-de59-4ec2-85a0-3821908f0bed, ab425fb5-3f82-4c7a-884b-033011b54600, d8937f65-6398-4aeb-b990-e9a1b93a4256, e5291967-85df-4554-8e64-332f814d8803, ebfb3779-ac14-404c-9ab3-61d4f4391ff4, f93bf079-df17-44bc-89a4-8eca03bfa997, fef65298-b96d-441b-9fe2-581a940c7721]
external_ids : {sg=sg, type=security_group}
name : ovn.sg.sg
ports : [] # 目前 port 都为空，因为还没有 pod 使用该安全组
_uuid : 25e6f4e2-120b-4536-890a-21486d797704
acls : []
external_ids : {np="node/empty"}
name : node.empty
ports : []
_uuid : 638326f6-0a4f-45b2-86fd-052fc55a10d3
acls : [3e067012-24a5-47e7-85db-921825a9b0ac, 45526498-47fe-4f6a-b76e-67cf86e7596b]
external_ids : {sg=kubeovn_deny_all, type=security_group}
name : ovn.sg.kubeovn_deny_all
ports : []
root@empty:~/test/sg# cat 02-sg-netshoot.yaml
---
apiVersion: v1
kind: Pod
metadata:
name: sg-netshoot1
namespace: sg
annotations:
ovn.kubernetes.io/logical_switch: sg-subnet1
ovn.kubernetes.io/port_security: "true"
ovn.kubernetes.io/security_groups: sg
spec:
containers:
- name: netshoot
image: nicolaka/netshoot:latest
imagePullPolicy: Never
command:
- sh
- -c
- "sleep infinity"
securityContext:
capabilities:
add:
- NET_ADMIN
---
apiVersion: v1
kind: Pod
metadata:
name: sg-netshoot2
namespace: sg
annotations:
ovn.kubernetes.io/logical_switch: sg-subnet1
ovn.kubernetes.io/port_security: "true"
ovn.kubernetes.io/security_groups: sg
spec:
containers:
- name: netshoot
image: nicolaka/netshoot:latest
imagePullPolicy: Never
command:
- sh
- -c
- "sleep infinity"
securityContext:
capabilities:
add:
- NET_ADMIN
root@empty:~/test/sg# k get po -n sg -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
sg-netshoot1 1/1 Running 0 6m16s 10.1.0.12 empty <none> <none>
sg-netshoot2 1/1 Running 0 6m16s 10.1.0.13 empty <none> <none>
root@empty:~/test/sg# k ko nbctl list logical_switch_port | grep -C 9 "netshoot"
_uuid : 3dac29b4-09a1-41c8-88b2-fdb7f632a112
addresses : ["00:00:00:96:FA:C7 10.1.0.13"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_default-securitygroup="false", associated_sg_sg="true", ls=sg-subnet1, pod="sg/sg-netshoot2", security_groups=sg, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : sg-netshoot2.sg
options : {}
parent_name : []
port_security : ["00:00:00:96:FA:C7 10.1.0.13"]
tag : []
tag_request : []
type : ""
up : true
_uuid : 4d95ded7-385c-47a7-9f6c-f6987a1c1dc8
addresses : ["00:00:00:20:FF:C5 10.1.0.12"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_default-securitygroup="false", associated_sg_sg="true", ls=sg-subnet1, pod="sg/sg-netshoot1", security_groups=sg, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : sg-netshoot1.sg
options : {}
parent_name : []
port_security : ["00:00:00:20:FF:C5 10.1.0.12"]
tag : []
tag_request : []
type : ""
up : true
root@empty:~/test/sg# k ko nbctl list port-group
...
_uuid : 1d05da99-17f0-480e-827f-36fd7f9ba06a
acls : [0e32e76f-7abf-4bec-b135-9b5889a3b816, 118681dd-69d3-4b05-9fba-d016d9734ea7, 14802f81-6b0b-4d85-bf5d-1d27dbf3db6f, 2d68efe1-87e8-47d0-8984-bb14d52ca5d9, 2e2df429-f3c5-42c4-8068-093e4c0e8ec3, 43cf0e6f-f28e-4091-91c7-872e301836ab, 4a557ba3-f296-46c6-94a3-befa9f2bfdf1, 99b917b9-ac55-457e-806e-1e326ed83d6c, 9d71e4c0-e310-4aa7-9480-33fe8b78cdd8, a7b95f0a-de59-4ec2-85a0-3821908f0bed, ab425fb5-3f82-4c7a-884b-033011b54600, d8937f65-6398-4aeb-b990-e9a1b93a4256, e5291967-85df-4554-8e64-332f814d8803, ebfb3779-ac14-404c-9ab3-61d4f4391ff4, f93bf079-df17-44bc-89a4-8eca03bfa997, fef65298-b96d-441b-9fe2-581a940c7721]
external_ids : {sg=sg, type=security_group}
name : ovn.sg.sg
ports : [3dac29b4-09a1-41c8-88b2-fdb7f632a112, 4d95ded7-385c-47a7-9f6c-f6987a1c1dc8]
_uuid : 25e6f4e2-120b-4536-890a-21486d797704
acls : []
external_ids : {np="node/empty"}
name : node.empty
ports : []
_uuid : 638326f6-0a4f-45b2-86fd-052fc55a10d3
acls : [3e067012-24a5-47e7-85db-921825a9b0ac, 45526498-47fe-4f6a-b76e-67cf86e7596b]
external_ids : {sg=kubeovn_deny_all, type=security_group}
name : ovn.sg.kubeovn_deny_all
ports : [3dac29b4-09a1-41c8-88b2-fdb7f632a112, 4d95ded7-385c-47a7-9f6c-f6987a1c1dc8]
目前 删除 ip ，清理 lsp ipam sg 本身没有问题。 只是 pod 如果依旧在用这个 ip（annos 没变）。 后面 lsp 依旧会被重建出来。 如果 ip 被其他 pod抢走了，这里就会一直报错。
但是我们的这个功能明确在于用户的 pod 不再使用该 ip 的场景下，所以这个逻辑应该不影响。
查看 清理 ip 后的 sg 的 port-group 是否有变化
root@empty:~# k ko nbctl list logical_switch_port | grep -C 10 sg-netshoot
_uuid : 3dac29b4-09a1-41c8-88b2-fdb7f632a112
addresses : ["00:00:00:96:FA:C7 10.1.0.13"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_default-securitygroup="false", associated_sg_sg="true", ls=sg-subnet1, pod="sg/sg-netshoot2", security_groups=sg, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : sg-netshoot2.sg
options : {}
parent_name : []
port_security : ["00:00:00:96:FA:C7 10.1.0.13"]
tag : []
tag_request : []
type : ""
up : true
_uuid : 6873dcf6-6f1f-440c-b632-0a947525b130
addresses : ["00:00:00:20:FF:C5 10.1.0.12"]
dhcpv4_options : []
dhcpv6_options : []
dynamic_addresses : []
enabled : []
external_ids : {associated_sg_default-securitygroup="false", associated_sg_sg="true", ls=sg-subnet1, pod="sg/sg-netshoot1", security_groups=sg, vendor=kube-ovn}
ha_chassis_group : []
mirror_rules : []
name : sg-netshoot1.sg
options : {}
parent_name : []
port_security : ["00:00:00:20:FF:C5 10.1.0.12"]
tag : []
tag_request : []
type : ""
up : true
root@empty:~#
root@empty:~# k ko nbctl list port-group
_uuid : eda4e58c-d57f-4684-a340-cb2dbf9d2f50
acls : []
external_ids : {np="external/empty"}
name : external.empty
ports : []
_uuid : d00452a8-6a74-4464-afe6-4c547a91744d
acls : []
external_ids : {np="ovn-default/empty"}
name : ovn.default.empty
ports : [0e7f7884-1005-4543-b2eb-46fc51ab22d6, 46a46ca0-b1a5-43b8-bb58-35baeb3ac882, 61139bf7-b2e0-4738-861b-cdd9792441c8, 8e38fb59-c966-4a2d-8196-004c4c0ea748, 9e7c3fbf-c834-4004-acf2-bd86b72b77a9, a3f6ef2b-1c83-4b2c-8a79-7dd1d8bcc53a, eac5c379-0d87-4f20-86d4-0124cf682a68]
_uuid : 1d05da99-17f0-480e-827f-36fd7f9ba06a
acls : [0e32e76f-7abf-4bec-b135-9b5889a3b816, 118681dd-69d3-4b05-9fba-d016d9734ea7, 14802f81-6b0b-4d85-bf5d-1d27dbf3db6f, 2d68efe1-87e8-47d0-8984-bb14d52ca5d9, 2e2df429-f3c5-42c4-8068-093e4c0e8ec3, 43cf0e6f-f28e-4091-91c7-872e301836ab, 4a557ba3-f296-46c6-94a3-befa9f2bfdf1, 99b917b9-ac55-457e-806e-1e326ed83d6c, 9d71e4c0-e310-4aa7-9480-33fe8b78cdd8, a7b95f0a-de59-4ec2-85a0-3821908f0bed, ab425fb5-3f82-4c7a-884b-033011b54600, d8937f65-6398-4aeb-b990-e9a1b93a4256, e5291967-85df-4554-8e64-332f814d8803, ebfb3779-ac14-404c-9ab3-61d4f4391ff4, f93bf079-df17-44bc-89a4-8eca03bfa997, fef65298-b96d-441b-9fe2-581a940c7721]
external_ids : {sg=sg, type=security_group}
name : ovn.sg.sg
ports : [3dac29b4-09a1-41c8-88b2-fdb7f632a112, 6873dcf6-6f1f-440c-b632-0a947525b130]
# 可以看到这两个 port-group 正好是上面原 ip 删除后的新的 port id
_uuid : 25e6f4e2-120b-4536-890a-21486d797704
acls : []
external_ids : {np="node/empty"}
name : node.empty
ports : []
_uuid : 638326f6-0a4f-45b2-86fd-052fc55a10d3
acls : [3e067012-24a5-47e7-85db-921825a9b0ac, 45526498-47fe-4f6a-b76e-67cf86e7596b]
external_ids : {sg=kubeovn_deny_all, type=security_group}
name : ovn.sg.kubeovn_deny_all
ports : [3dac29b4-09a1-41c8-88b2-fdb7f632a112, 6873dcf6-6f1f-440c-b632-0a947525b130]
root@empty:~#
