系统环境
本次测试使用的系统为Centos8 Stream,系统内核已升级到了6.6
[root@node4 ~]# uname -a
Linux node4 6.6.11-1.el8.elrepo.x86_64 #1 SMP PREEMPT_DYNAMIC Wed Jan 10 12:37:48 EST 2024 x86_64 x86_64 x86_64 GNU/Linux
创建k8s初始化配置
kubeadm config print init-defaults > kube-init.yaml
修改上一步生成的配置文件
# kube-init.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.3.8 # master的ip
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock
imagePullPolicy: IfNotPresent
name: node4 # 修改为master的节点主机名
taints: null
skipPhases: # 跳过的阶段，这里是不使用kube-proxy
- addon/kube-proxy
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.k8s.io
kind: ClusterConfiguration
kubernetesVersion: 1.28.0
networking:
dnsDomain: cluster.local
serviceSubnet: 10.96.0.0/12
podSubnet: 10.244.0.0/16
scheduler: {}
初始化集群
kubeadm init --config kube-init.yaml
查看Pod
[root@node4 ~]# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system coredns-5dd5756b68-clwxq 0/1 Pending 0 117m
kube-system coredns-5dd5756b68-tsstt 0/1 Pending 0 117m
kube-system etcd-node4 1/1 Running 1 117m
kube-system kube-apiserver-node4 1/1 Running 1 117m
kube-system kube-controller-manager-node4 1/1 Running 0 117m
kube-system kube-scheduler-node4 1/1 Running 1 117m
安装Cilium
下载cilium-cli
wget https://github.com/cilium/cilium-cli/releases/download/v0.15.20/cilium-linux-amd64.tar.gz
解压缩
tar -xf cilium-linux-amd64.tar.gz
移动到/usr/local/bin
mv cilium /usr/local/bin/
安装cilium cni
注意事项
- DEVICES需要根据自己主机进行更改
- k8sServiceHost 需要根据自己主机进行更改
export DEVICES="eth0"
cilium install --version 1.14.5 \
--set k8sServiceHost=192.168.3.8 \
--set k8sServicePort=6443 \
--set cluster.name=prvite-kubernetes \
--set cluster.id=1 \
--set kubeProxyReplacement=true \
--set hubble.enabled=true \
--set hubble.ui.enabled=true \
--set hubble.relay.enabled=true \
--set hubble.relay.service.type=NodePort \
--set hubble.relay.service.nodePort=31234 \
--set nodeinit.enabled=true \
--set rollOutCiliumPods=true \
--set bpfClockProbe=true \
--set operator.replicas=2 \
--set loadBalancer.mode=dsr \
--set tunnel=disabled \
--set autoDirectNodeRoutes=true \
--set ipam.mode="multi-pool" \
--set ipv4NativeRoutingCIDR="10.10.0.0/16" \
--set ipam.operator.autoCreateCiliumPodIPPools.default.ipv4.cidrs='{10.10.0.0/16}' \
--set ipam.operator.autoCreateCiliumPodIPPools.default.ipv4.maskSize=24 \
--set bpf.masquerade=true \
--set enableIpv4Masquerade=true \
--set enableIpv6Masquerade=false \
--set endpointRoutes.enabled=true \
--set-string extraConfig.enable-local-node-route=false \
--set ipMasqAgent.config.nonMasqueradeCIDRs='{10.10.0.0/16}' \
--set ipMasqAgent.config.masqLinkLocal=false \
--set devices=$DEVICES \
--set bpf.preallocateMaps=true \
--set routingMode=native \
--set bandwidthManager.enabled=true \
--set bandwidthManager.bbr=true \
--set loadBalancer.acceleration=native
查看Pod
[root@node4 ~]# kubectl get pods -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system cilium-cwwmc 1/1 Running 0 9m18s
kube-system cilium-node-init-7jdfv 1/1 Running 1 (3m22s ago) 7h25m
kube-system cilium-node-init-7lw4w 1/1 Running 0 9m18s
kube-system cilium-operator-7fb446f6d8-52zrd 1/1 Running 1 (3m22s ago) 7h25m
kube-system cilium-operator-7fb446f6d8-wk56z 1/1 Running 2 (3m31s ago) 7h25m
kube-system cilium-zvzqj 1/1 Running 1 (3m22s ago) 7h25m
kube-system coredns-5dd5756b68-clwxq 1/1 Running 1 (3m22s ago) 9h
kube-system coredns-5dd5756b68-tsstt 1/1 Running 1 (3m22s ago) 9h
kube-system etcd-node4 1/1 Running 2 (3m23s ago) 9h
kube-system hubble-relay-d478c79c8-hhmfv 1/1 Running 1 (3m22s ago) 7h25m
kube-system hubble-ui-6f48889749-sl2d7 2/2 Running 2 (3m22s ago) 7h25m
kube-system kube-apiserver-node4 1/1 Running 2 (3m22s ago) 9h
kube-system kube-controller-manager-node4 1/1 Running 1 (3m22s ago) 9h
kube-system kube-scheduler-node4 1/1 Running 2 (3m22s ago) 9h
遇到的错误
错误一
level=info msg="Updated link /sys/fs/bpf/cilium/socketlb/links/cgroup/cil_sock6_post_bind for program cil_sock6_post_bind" subsys=socketlb
level=fatal msg="Failed to compile XDP program" error="program cil_xdp_entry: attaching XDP program to interface ens33: operation not supported" subsys=datapath-loader
解决
cilium upgrade cilium cilium/cilium \
--version 1.14.5 \
--namespace kube-system \
--reuse-values \
--set loadBalancer.acceleration=disabled
