1.主机配置列表
| 主机名 | K8s节点类型 | ip地址 | 系统版本 |
|---|
| server236 | Master | 192.168.1.236 | Ubuntu2204 |
| Server237 | Master | 192.168.1.237 | Ubuntu2204 |
| Server238 | Master | 192.168.1.238 | Ubuntu2204 |
| Server239 | Slave | 192.168.1.239 | Ubuntu2204 |
| Server240 | Slave | 192.168.1.240 | Ubuntu2204 |
| Server241 | 无 | 192.168.1.241 | Ubuntu2404 |
2. 配置hosts(所有主机)
root@server236:~# vi /etc/hosts
192.168.1.236 server236
192.168.1.237 server237
192.168.1.238 server238
192.168.1.239 server239
192.168.1.240 server240
3.内核转发及网桥过滤(所有主机)
# 添加系统启动时自动加载的内核模块
root@server236:~# vi /etc/modules-load.d/k8s.conf
overlay
br_netfilter
root@server236:~# modprobe overlay
root@server236:~# modprobe br_netfilter
# 立即加载模块
root@server236:~# lsmod | grep -E 'overlay|br_netfilter'
br_netfilter 32768 0
bridge 311296 1 br_netfilter
overlay 151552 0
# 开启桥接流量通过 iptables 和 ip6tables 过滤，启用 IPv4 数据包转发。
root@server236:~# vi /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
# 加载内核参数
sysctl --system
#
root@server236:~# cat > /etc/sysctl.d/k8s.conf << EOF
#开启网桥模式【重要】
net.bridge.bridge-nf-call-iptables=1
#开启网桥模式【重要】
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
# 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.swappiness=0
# 不检查物理内存是否够用
vm.overcommit_memory=1
# 开启 OOM
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
#关闭ipv6【重要】
# net.ipv6.conf.all.disable_ipv6=1
# net.netfilter.nf_conntrack_max=2310720
# 下面的内核参数可以解决ipvs模式下长连接空闲超时的问题
net.ipv4.tcp_keepalive_intvl = 30
net.ipv4.tcp_keepalive_probes = 10
net.ipv4.tcp_keepalive_time = 600
EOF
4.安装ipset与ipvsadm(所有主机)
# 安装ipset及ipvsadm
root@server236:~# apt install -y ipset ipvsadm
# 配置ipvsadm模块加载
root@server236:~# cat << EOF | sudo tee /etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_rr
ip_vs_wrr
ip_vs_sh
nf_conntrack
EOF
# 立即加载ipvs模块
root@server236:~# modprobe -- ip_vs
root@server236:~# modprobe -- ip_vs_rr
root@server236:~# modprobe -- ip_vs_wrr
root@server236:~# modprobe -- ip_vs_sh
root@server236:~# modprobe -- nf_conntrack
# 查看ipvs模块是否加载
lsmod | grep ip_vs
5.时间同步(所有主机)
root@server236:~
root@server236:~
root@server236:~
root@server236:~
0 0 * * * ntpdate time1.aliyun.com
6.禁用swap分区(所有主机)
root@server236:~# swapoff -a && sudo sed -i '/swap/s/^/#/' /etc/fstab
7.禁用linux的透明大页、标准大页(未验证)
root@server236:~# echo never > /sys/kernel/mm/transparent_hugepage/defrag
root@server236:~# echo never > /sys/kernel/mm/transparent_hugepage/enabled
root@server236:~# echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.local
root@server236:~# echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.local
root@server236:~# chmod +x /etc/rc.d/rc.local
8.文件数设置(所有主机)
root@server236:~# ulimit -SHn 65535
root@server236:~# cat >> /etc/security/limits.conf <<EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* seft memlock unlimited
* hard memlock unlimitedd
EOF
9.RKE2 安装与配置
9.1 第一台管理节点配置
9.1.1 下载离线安装文件(最好提前下载，有些东西可能无法从github拉取)
# 创建文件夹下载离线文件
root@server236:~# mkdir -pv /data/rke2-artifacts
# 创建rke2的image文件用来存储离线镜像
root@server236:~# mkdir -p /var/lib/rancher/rke2/agent/images
# 下载v1.34.2版本(根据需要)
root@server236:~# cd /data/rke2-artifacts
root@server236:/data/rke2-artifacts# wget https://github.com/rancher/rke2/releases/download/v1.34.2%2Brke2r1/rke2-images.linux-amd64.tar.zst
root@server236:/data/rke2-artifacts# wget https://github.com/rancher/rke2/releases/download/v1.34.2%2Brke2r1/rke2.linux-amd64.tar.gz
root@server236:/data/rke2-artifacts# wget https://github.com/rancher/rke2/releases/download/v1.34.2%2Brke2r1/sha256sum-amd64.txt
# 下载v1.29.15版本(根据需要)
root@server236:~# cd /data/rke2-artifacts
root@server236:/data/rke2-artifacts# wget https://github.com/rancher/rke2/releases/download/v1.29.15%2Brke2r1/rke2-images.linux-amd64.tar.zst
root@server236:/data/rke2-artifacts# wget https://github.com/rancher/rke2/releases/download/v1.29.15%2Brke2r1/rke2.linux-amd64.tar.gz
root@server236:/data/rke2-artifacts# wget https://github.com/rancher/rke2/releases/download/v1.29.15%2Brke2r1/sha256sum-amd64.txt
# 将离线镜像拷贝到指定文件夹
root@server236:/data/rke2-artifacts# cp rke2-images.linux-amd64.tar.zst /var/lib/rancher/rke2/agent/images
9.1.2 配置 RKE2 配置文件
root@server236:~# mkdir -p /etc/rancher/rke2 && vim /etc/rancher/rke2/config.yaml
token: 123456
node-name: server236
tls-san: 192.168.1.236
system-default-registry: "registry.cn-hangzhou.aliyuncs.com" #容器镜像仓库
kube-proxy-arg: # 部署负载均衡器
- proxy-mode=ipvs
- ipvs-strict-arp=true
#node-taint:
# - "CriticalAddonsOnly=true:NoExecute"
node-label:
- "node=Master"
- "Master=server236"
# 解释说明
- token 表示自定义的一个 token 标识。
- node-name 表示配置节点的名称，该名称是全局唯一的，用于 DNS 路由。
- tls-san 表示 TLS 证书上添加的主机名或 IPv4/IPv6 地址，处处填写本机 IP，该参数是为了避免固定地址的证书错误。
- system-default-registry 表示使用国内镜像。
- node-taint： 给主节点打上污点标记，使主节点不执行pod的任务，如果只想从节点执行工作任务，主节点不执行，取消这个注释
9.1.3 配置使用私服仓库(可选,这里配置的是容器私服不是rke2需要镜像)
# cat /etc/rancher/rke2/registries.yaml
mirrors:
www.aaa.com: # 仓库名
endpoint:
- "http://www.aaa.com" # 仓库地址
configs:
"http://www.aaa.com": # 仓库安全配置
auth:
username: admin
password: 12345
说明：
- mirrors 字段说明
表示当访问镜像时，会把 docker.io 重定向到国内的镜像网站 https://docker.mirrors.ustc.edu.cn
- configs 字段说明
该段配置表示有镜像仓库，企业自己搭建的 harbor 仓库，如果没有私有仓库，则 configs 段配置可以省略。
- www.kubemsb.com 填写镜像仓库的地址。
- auth 块下的 username 和 password 填写仓库的登录账号密码。
如果镜像仓库访问时使用 https（使用了 tls），则需要填写 tls 的信息，如不验证 CA 证书，则 tls 下需要填写 insecure_skip_verify: true。如果需要验证，则需要填写 cert_file、key_file 和 ca_file 这三个参数。
9.1.4 安装Server端
# 下载安装脚本并授权(官网 脚本curl -sfL https://get.rke2.io)
root@server237:~# cd /data/rke2-artifacts
root@server237:/data/rke2-artifacts# wget --no-check-certificate https://rancher-mirror.rancher.cn/rke2/install.sh
root@server237:/data/rke2-artifacts# chmod +x install.sh
# 指定版本国内源-主节点指定安装版本(使用线上脚本安装，安装过程会失败还不知道什么情况)
# root@server236:~
root@server237:/data/rke2-artifacts# INSTALL_RKE2_TYPE="server" INSTALL_RKE2_MIRROR=cn INSTALL_RKE2_VERTSION=v1.34.2+rke2r1 INSTALL_RKE2_ARTIFACT_PATH=/data/rke2-artifacts sh install.sh
[INFO] staging local checksums from /data/rke2-artifacts/sha256sum-amd64.txt
[INFO] staging zst airgap image tarball from /data/rke2-artifacts/rke2-images.linux-amd64.tar.zst
[INFO] staging tarball from /data/rke2-artifacts/rke2.linux-amd64.tar.gz
[INFO] verifying airgap tarball
[INFO] installing airgap tarball to /var/lib/rancher/rke2/agent/images
[INFO] verifying tarball
[INFO] unpacking tarball file to /usr/local
# 查找 RKE2 安装位置：
root@server236:~# find / -name rke2
/usr/local/share/rke2
/usr/local/bin/rke2
/etc/rancher/rke2
9.1.5 启动rke2Server开始初始化
# 执行时间会比较久
root@server236:~# systemctl enable --now rke2-server.service
# 如果想查看初识话状态可以使用下面命令
root@server236:~# journalctl -u rke2-server -f
# 查看安装生成的 token
root@server236:/data/rke2-artifacts# cat /var/lib/rancher/rke2/server/node-token
# 查看集群信息(kubeconfig)
root@server236:/data/rke2-artifacts# cat /etc/rancher/rke2/rke2.yaml
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: |
LS0tLS1CRUdJTiBDRVJUSU...（省略的内容表示证书数据）
server: https://127.0.0.1:6443
name: default
contexts:
- context:
cluster: default
user: default
name: default
current-context: default
10.1.6 配置kubectl命令
# 在安装完成后 kubectl等二进制命令文件都在一个目录当中需要添加环境变量来使系统能够正常调用
root@server237:/data/rke2-artifacts# ls /var/lib/rancher/rke2/bin
containerd containerd-shim containerd-shim-runc-v1 containerd-shim-runc-v2 crictl ctr kubectl kubelet runc
root@server236:/data/rke2-artifacts# vim /etc/profile.d/rke2.sh
export PATH=$PATH:/var/lib/rancher/rke2/bin
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
root@server236:/data/rke2-artifacts# source /etc/profile
# 配置ctr命令和crictl能够正常使用
root@server236:/data/rke2-artifacts# echo 'alias ctr="/var/lib/rancher/rke2/bin/ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io"' >> /etc/profile
root@server236:/data/rke2-artifacts# echo 'export CRI_CONFIG_FILE=/var/lib/rancher/rke2/agent/etc/crictl.yaml' >> /etc/profile
root@server236:/data/rke2-artifacts# source /etc/profile
# 测试命令
root@server237:/data/rke2-artifacts# kubectl get nodes
NAME STATUS ROLES AGE VERSION
server237 Ready control-plane,etcd 21m v1.34.2+rke2r1
root@server239:/data/rke2-artifacts# crictl images
IMAGE TAG IMAGE ID SIZE
docker.io/rancher/hardened-addon-resizer 1.8.23-build20251016 682ffb21c0403 48.1MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-addon-resizer 1.8.23-build20251016 682ffb21c0403 48.1MB
docker.io/rancher/hardened-calico v3.30.3-build20251015 02dad26543033 686MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-calico v3.30.3-build20251015 02dad26543033 686MB
docker.io/rancher/hardened-cluster-autoscaler v1.10.2-build20251015 88a47d41f8b82 48.7MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-cluster-autoscaler v1.10.2-build20251015 88a47d41f8b82 48.7MB
docker.io/rancher/hardened-coredns v1.13.1-build20251015 f4ccc0f5d7de0 93.1MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-coredns v1.13.1-build20251015 f4ccc0f5d7de0 93.1MB
docker.io/rancher/hardened-dns-node-cache 1.26.7-build20251016 b15f8e9b31de6 70MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-dns-node-cache 1.26.7-build20251016 b15f8e9b31de6 70MB
docker.io/rancher/hardened-etcd v3.6.5-k3s1-build20251017 405516f27f18a 47.6MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-etcd v3.6.5-k3s1-build20251017 405516f27f18a 47.6MB
docker.io/rancher/hardened-flannel v0.27.4-build20251015 cd9765145172e 69.6MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-flannel v0.27.4-build20251015 cd9765145172e 69.6MB
docker.io/rancher/hardened-k8s-metrics-server v0.8.0-build20251015 be573c48df116 71.5MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-k8s-metrics-server v0.8.0-build20251015 be573c48df116 71.5MB
docker.io/rancher/hardened-kubernetes v1.34.2-rke2r1-build20251112 7b491b625ab3a 640MB
registry.cn-hangzhou.aliyuncs.com/rancher/hardened-kubernetes v1.34.2-rke2r1-build20251112 7b491b625ab3a 640MB
docker.io/rancher/klipper-helm v0.9.10-build20251111 1d0854c41ff27 181MB
registry.cn-hangzhou.aliyuncs.com/rancher/klipper-helm v0.9.10-build20251111 1d0854c41ff27 181MB
docker.io/rancher/klipper-lb v0.4.13 f7415d0003cb6 12.7MB
registry.cn-hangzhou.aliyuncs.com/rancher/klipper-lb v0.4.13 f7415d0003cb6 12.7MB
docker.io/rancher/mirrored-ingress-nginx-kube-webhook-certgen v1.6.4 884bd0ac01c8f 43.1MB
registry.cn-hangzhou.aliyuncs.com/rancher/mirrored-ingress-nginx-kube-webhook-certgen v1.6.4 884bd0ac01c8f 43.1MB
docker.io/rancher/mirrored-pause 3.6 6270bb605e12e 301kB
registry.cn-hangzhou.aliyuncs.com/rancher/mirrored-pause 3.6 6270bb605e12e 301kB
docker.io/rancher/mirrored-sig-storage-snapshot-controller v8.2.0 25f7295428a63 71.9MB
registry.cn-hangzhou.aliyuncs.com/rancher/mirrored-sig-storage-snapshot-controller v8.2.0 25f7295428a63 71.9MB
docker.io/rancher/nginx-ingress-controller v1.13.4-hardened1 f184bd6520d3a 707MB
registry.cn-hangzhou.aliyuncs.com/rancher/nginx-ingress-controller v1.13.4-hardened1 f184bd6520d3a 707MB
docker.io/rancher/rke2-cloud-provider v1.34.2-0.20251010190833-cf0d35a732d1-build20251017 35741f48740f4 82.1MB
registry.cn-hangzhou.aliyuncs.com/rancher/rke2-cloud-provider v1.34.2-0.20251010190833-cf0d35a732d1-build20251017 35741f48740f4 82.1MB
docker.io/rancher/rke2-runtime v1.34.2-rke2r1 0fc9035fae438 306MB
registry.cn-hangzhou.aliyuncs.com/rancher/rke2-runtime v1.34.2-rke2r1 0fc9035fae438 306MB
10.2 第二台管理节点加入
# 其他步骤都相同，配置文件稍作修改
root@server236:~# mkdir -p /etc/rancher/rke2 && vim /etc/rancher/rke2/config.yaml
server: https://192.168.1.236:9345
token: 123456
node-name: server237
tls-san: 192.168.1.237
system-default-registry: "registry.cn-hangzhou.aliyuncs.com" #容器镜像仓库
kube-proxy-arg: # 部署负载均衡器
- proxy-mode=ipvs
- ipvs-strict-arp=true
#node-taint:
# - "CriticalAddonsOnly=true:NoExecute"
node-label:
- "node=Master"
- "Master=server237"
10.3 第三台管理节点加入
# 其他步骤都相同，配置文件稍作修改
root@server236:~# mkdir -p /etc/rancher/rke2 && vim /etc/rancher/rke2/config.yaml
server: https://192.168.1.236:9345
token: 123456
node-name: server238
tls-san: 192.168.1.238
system-default-registry: "registry.cn-hangzhou.aliyuncs.com" #容器镜像仓库
kube-proxy-arg: # 部署负载均衡器
- proxy-mode=ipvs
- ipvs-strict-arp=true
#node-taint:
# - "CriticalAddonsOnly=true:NoExecute"
node-label:
- "node=Master"
- "Master=server238"
10.4 Work节点加入(所有从节点都可以这样操作)
# 命令基本相同，初始化命令有所不同。
root@server239:/data/rke2-artifacts# wget --no-check-certificate https://rancher-mirror.rancher.cn/rke2/install.sh
root@server239:/data/rke2-artifacts# chmod +x install.sh
root@server239:/data/rke2-artifacts# INSTALL_RKE2_TYPE="agent" INSTALL_RKE2_MIRROR=cn INSTALL_RKE2_VERTSION=v1.34.2+rke2r1 INSTALL_RKE2_ARTIFACT_PATH=/data/rke2-artifacts sh install.sh
# Work节点配置文件
root@server239:~# mkdir -p /etc/rancher/rke2 && vim /etc/rancher/rke2/config.yaml
server: https://192.168.1.236:9345
token: 123456
node-name: server239
tls-san: 192.168.1.239
system-default-registry: "registry.cn-hangzhou.aliyuncs.com" #容器镜像仓库
kube-proxy-arg: # 部署负载均衡器
- proxy-mode=ipvs
- ipvs-strict-arp=true
#node-taint:
# - "CriticalAddonsOnly=true:NoExecute"
node-label:
- "node=Master"
- "Master=server239"
# agent初始化
root@server236:~# systemctl enable --now rke2-agent.service
# 如果想查看初识话状态可以使用下面命令
root@server236:~# journalctl -u rke2-agent -f
