1.1 服务器准备
我们准备4台服务器，两台node节点，两台master节点
192.168.2.251 server251
192.168.2.253 server253
192.168.2.252 node252
192.168.2.249 node249
在每台服务器执行 hostname+名称 修改主机hostname
hostname node252
关闭防火墙：
$ systemctl stop firewalld
$ systemctl disable firewalld
关闭 selinux：
$ sed -i 's/enforcing/disabled/' /etc/selinux/config # 永久
$ setenforce 0 # 临时
关闭 swap：
$ swapoff -a # 临时
$ sed -ri 's/.*swap.*/#&/' /etc/fstab # 永久
注意：如果重启服务器，swap一般重启后要执行临时关闭命令
1.1.1 在每个master节点安装keepalive用于健康检查
[root@# server253 opt]# yum install -y keepalived
[root@# server253 opt]# cat > /etc/keepalived/keepalived.conf << EOFglobal_defs {
notification_email {
acassen @firewall.loc
failover @firewall.loc
sysadmin @firewall.loc
}
notification_email_from Alexandre.Cassen @firewall.loc
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id NGINX_MASTER
}
vrrp_script check_nginx {
script "/etc/keepalived/check_nginx.sh"
}
vrrp_instance VI_1 {
state MASTER
interface ens192
virtual_router_id 51 # VRRP 路由 ID 实例， 每个实例是唯一的
priority 100 # 优先级， 备服务器设置 90
advert_int 1 # 指定 VRRP 心跳包通告间隔时间， 默认 1 秒
authentication {
auth_type PASS
auth_pass 1111
}
# 虚拟 IP
virtual_ipaddress {
192.168.2.88/24
}
track_script {
check_nginx
}
}
EOF
[root@# server253 opt]# vim /etc/keepalived/keepalived.conf
[root@# server253 opt]# systemctl start keepalived.service
[root@# server253 opt]# systemctl enable keepalived.serviceCreated symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /usr/lib/systemd/system/keepalived.servic e.
[root@# server253 opt]# ip a s ens1922: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:0c:29:11:91:cc brd ff:ff:ff:ff:ff:ff
inet 192.168.2.252/24 brd 192.168.2.255 scope global noprefixroute ens192
valid_lft forever preferred_lft forever
inet 192.168.2.88/24 scope global secondary ens192
valid_lft forever preferred_lft forever
inet6 fe80::d70:4f81:5aee:f432/64 scope link noprefixroute
valid_lft forever preferred_lft forever
ens192这个每个电脑不一样，一般在 /etc/sysconfig/network-script/下有个ens开头的文件，可以查看里面内容，我的虚拟机内是这个：
vi /etc/sysconfig/network-script/ifcfg-ens192 :TYPE=EthernetPROXY_METHOD=noneBROWSER_ONLY=noBOOTPROTO=staticIPADDR=192.168.2.213NETMASK=255.255.255.0GATEWAY=192.168.3.1DNS1=8.8.8.8DEFROUTE=yesIPV4_FAILURE_FATAL=noIPV6INIT=yesIPV6_AUTOCONF=yesIPV6_DEFROUTE=yesIPV6_FAILURE_FATAL=noIPV6_ADDR_GEN_MODE=stable-privacyNAME=ens192UUID=988c62a9-c219-4fd2-8323-ea34c80aadecDEVICE=ens192ONBOOT=yes
1.1.2 在master251和master253 修改 /etc/hosts，加入
192.168.2.251 server251
192.168.2.252 node252
192.168.2.253 server253
192.168.2.249 node249
1.1.3 每个节点指定版本安装 yum install -y kubelet-1.20.2 kubeadm-1.20.2 kubectl-1.20.2
先把原有安装的k8s卸载
1
systemctl stop kubelet
systemctl stop etcd
2
kubeadm reset -f
3 清除相关配置
modprobe -r ipip
lsmod
rm -rf ~/.kube/
rm -rf /etc/kubernetes/
rm -rf /etc/systemd/system/kubelet.service.d
rm -rf /etc/systemd/system/kubelet.service
rm -rf /usr/bin/kube*
rm -rf /etc/cni
rm -rf /opt/cni
rm -rf /var/lib/etcd
rm -rf /var/etcd
4 删除kubeadmi kubelet kubectl
yum -y remove kube*
yum install -y kubelet-1.20.2 kubeadm-1.20.2 kubectl-1.20.2
1.1.4 设置kubelet开机启动，然后启动kublet
systemctl enable kubelet
1.2 k8s集群安装
1.2.1 方式一：执行kubeadm init, 注意IP修改为每个master节点ip
[root@server253 kubernetes]# kubeadm init --apiserver-advertise-address=192.168.2.253 --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.20.2 --service-cidr=10.96.0.0/12 --pod-network-cidr=10.244.0.0/16
出现这个结果表示成功，注意需要按照提示执行三个命令，最后一句用于node节点加入master
#master节点253Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.2.253:6443 --token vzakyn.jcxip00b5kimfson --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5a
如果加入master节点，需要在后面加上 --control-plane
kubeadm join 192.168.2.253:6443 --token vzakyn.jcxip00b5kimfson --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5a --control-plane
方式二：
将192.168.2.253 server253追加到每个节点/etc/hosts末尾
#先在每个节点服务器将server253座位master节点配置到hosts
[root@node249 ~]# vim /etc/hosts
[root@node249 ~]# ping server253
PING server253 (192.168.2.253) 56(84) bytes of data.
64 bytes from server253 (192.168.2.253): icmp_seq=1 ttl=64 time=0.130 ms
64 bytes from server253 (192.168.2.253): icmp_seq=2 ttl=64 time=0.104 ms
^C
--- server253 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1000ms
rtt min/avg/max/mdev = 0.104/0.117/0.130/0.013 ms
#初始化中使用control-plane-endpoint参数
[root@server253 kubernetes]# kubeadm init \
--apiserver-advertise-address=192.168.2.253 \
--control-plane-endpoint=server253 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.20.2 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16
这种方式后期加入节点使用命令就可以不用ip，而是用hosts配置的master的主机名称
kubeadm join server253:6443 --token mbd8fr.limeyyhyhsbu81nz --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5a
1.2.2 多个master节点 虚拟ip（virtual ip）不是每个节点都有
我们通过命令查看那个master节点获取了vip
[root@server253 kubernetes]# ip a s ens1922: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:0c:29:f9:21:ff brd ff:ff:ff:ff:ff:ff
inet 192.168.2.253/24 brd 192.168.2.255 scope global noprefixroute ens192
valid_lft forever preferred_lft forever
inet 192.168.2.88/24 scope global secondary ens192
valid_lft forever preferred_lft forever
inet6 fe80::7629:62c4:282:eab5/64 scope link noprefixroute
valid_lft forever preferred_lft forever
[root@server251 kubernetes]# ip a s ens192
2: ens192: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
link/ether 00:0c:29:4a:59:0e brd ff:ff:ff:ff:ff:ff
inet 192.168.2.251/24 brd 192.168.2.255 scope global noprefixroute ens192
valid_lft forever preferred_lft forever
inet6 fe80::12c1:3319:5a1c:7b81/64 scope link noprefixroute
valid_lft forever preferred_lft forever
上述两个服务器执行结果说明目前vip在 192.168.2.253服务器，那么我们将节点加入到192.168.2.253master
在node252执行 kubeadm join，为了测试后续加入节点，我们暂时不在node249节点执行
#在252 node节点[root@#node252 kubernetes]# kubeadm join 192.168.2.253:6443 --token vzakyn.jcxip00b5kimfson --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5aW0516 16:39:27.721379 25829 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
[WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[WARNING SystemVerification]: this Docker version is not on the list of validated versions: 20.10.15. Latest validated version: 19.03
[WARNING Hostname]: hostname "node252" could not be reached
[WARNING Hostname]: hostname "node252": lookup node252 on 8.8.8.8:53: no such host
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
Run 'kubectl get nodes' on the control-plane to see this node join the cluster
然后回到server253查看节点情况
[root@server253 kubernetes]# kubectl get nodesNAME STATUS ROLES AGE VERSION
node252 NotReady <none> 7s v1.18.0
server253 NotReady master 115m v1.18.0
1.2.3 安装 Pod网络插件（二选一）
- 插件1：calico
查找k8s对应版本
**calico链接 System requirements (tigera.io)
记得在calico.yaml文件中添加
- name: IP_AUTODETECTION_METHOD
value: "interface=ens.*" # ens 根据实际网卡开头配置，支持正则表达式**
- name: CLUSTER_TYPE
value: "k8s,bgp"
- name: IP_AUTODETECTION_METHOD
value: "interface=ens.*"
# Auto-detect the BGP IP address.
- name: IP
value: "autodetect"
# Enable IPIP
- name: CALICO_IPV4POOL_IPIP
value: "Always"
[root@server253 kubernetes]# kubectl apply -f calico.yaml
[root@server253 kubernetes]# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-75d555c48-hcb6q 0/1 ContainerCreating 0 15s
kube-system calico-node-78cd5 0/1 PodInitializing 0 15s
kube-system calico-node-gmcbd 0/1 Running 0 15s
kube-system calico-node-jmpzg 0/1 Init:2/3 0 15s
kube-system coredns-7ff77c879f-77xql 0/1 Pending 0 28m
kube-system coredns-7ff77c879f-nc8rz 0/1 ContainerCreating 0 28m
kube-system etcd-server253 1/1 Running 0 28m
kube-system kube-apiserver-server253 1/1 Running 0 28m
kube-system kube-controller-manager-server253 1/1 Running 0 28m
kube-system kube-proxy-bc7h4 1/1 Running 0 10m
kube-system kube-proxy-dxj8p 1/1 Running 0 11m
kube-system kube-proxy-vlvdx 1/1 Running 0 28m
kube-system kube-scheduler-server253 1/1 Running 0 28m
插件2： kube-flannel
从上面内容我们看到STATUS都为NotReady，所以需要安装kube-flannel
[root@server253 kubernetes]# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
namespace/kube-flannel created
clusterrole.rbac.authorization.k8s.io/flannel created
clusterrolebinding.rbac.authorization.k8s.io/flannel created
serviceaccount/flannel created
configmap/kube-flannel-cfg created
daemonset.apps/kube-flannel-ds created
然后查看相关pod，注意namespace不是system
[root@server253 kubernetes]# kubectl get pods -n kube-flannelNAME READY STATUS RESTARTS AGE
kube-flannel-ds-gvnwg 0/1 Init:1/2 0 47s
kube-flannel-ds-v97ck 0/1 Init:1/2 0 47s
最后查看集群nodes
[root@server253 kubernetes]# kubectl get nodesNAME STATUS ROLES AGE VERSION
node252 Ready <none> 42m v1.18.0
server253 Ready master 157m v1.18.0
Status转态为Ready表示成功
卸载kube-flannel 安装 calico
#第一步，在master节点删除flannel
kubectl delete -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
#第二步，在node节点清理flannel网络留下的文件
ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
rm -f /etc/cni/net.d/*
注：执行完上面的操作，重启kubelet
#第三步，应用calico相关的yaml文件
1.2.4 创建一个Pod并暴露端口进行外网访问
[root@server253 kubernetes]# kubectl create deployment nginx --image=nginxdeployment.apps/nginx created
[root@server253 kubernetes]# kubectl get podNAME READY STATUS RESTARTS AGE
nginx-f89759699-kb6mc 0/1 ContainerCreating 0 12s
暴露端口
[root@server253 kubernetes]# kubectl expose deployment nginx --port=80 --type=NodePortservice/nginx exposed
查看pod和service，获取对应pod外网访问端口：如下是：32130
[root@server253 kubernetes]# kubectl get pod,svcNAME READY STATUS RESTARTS AGE
pod/nginx-f89759699-kb6mc 1/1 Running 0 2m26s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 164m
service/nginx NodePort 10.106.105.75 <none> 80:32130/TCP 13s
在浏览器访问：192.168.2.253:32130查看是否能够访问nginx，或者在其他节点使用CUrl进行访问：
[root@node249 containers]# curl http://192.168.2.253:32130/
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>
<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>
<p><em>Thank you for using nginx.</em></p>
</body>
</html>
能够正常访问nginx，说明我们创建pod成功！
1.3 安装k8s dashboard
1.3.1 安装
[root@localhost kubernetes]# kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.3.1/aio/deploy/recommended.yaml
namespace/kubernetes-dashboard created
serviceaccount/kubernetes-dashboard created
1.3.2 修改端口类型 并查看端口：如下是：31296
[root@localhost kubernetes]# kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
service/kubernetes-dashboard edited
[root@localhost kubernetes]# kubectl get svc -A |grep kubernetes-dashboard
kubernetes-dashboard dashboard-metrics-scraper ClusterIP 10.96.254.240 <none> 8000/TCP 6m56s
kubernetes-dashboard kubernetes-dashboard NodePort 10.96.246.89 <none> 443:31296/TCP 6m56s
1.3.3 创建dashboard访问账号
在/etc/kubernetes/下再创建一个自己存放配置文件夹
[root@localhost kubernetes]# mkdir self
[root@localhost kubernetes]# cd self/
[root@localhost self]# vim dash.yaml
在dash.yaml中写入
apiVersion: v1
kind: ServiceAccount
metadata:
name: admin-user
namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: admin-user
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: admin-user
namespace: kubernetes-dashboard
1.3.4 创建dashboard 的pod
[root@localhost self]# kubectl apply -f dash.yaml
serviceaccount/admin-user created
clusterrolebinding.rbac.authorization.k8s.io/admin-user created
1.3.5 生成dashboard 访问token
[root@localhost self]# kubectl -n kubernetes-dashboard get secret $(kubectl -n kubernetes-dashboard get sa/admin-user -o jsonpath="{.secrets[0].name}") -o go-template="{{.data.token | base64decode}}"
eyJhbGciOiJSUzI1NiIsImtpZCI6Ik42T1ZuWWpSVVpZVF9wQlA4OXdncjF5WE5WZjdXQnBXRHM3S1JBTGhpS0kifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLWp4bXI4Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiIzNThmZTc5NC1kNDE5LTRmZmEtOWEzZS03MGUyOGNkMTI0YTAiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.Y9EmG24GAPxsnZr3Bd03loFRPoEmSYXlp57yu26rTPTqN6rOMnoLbfmNSQ2O5jeHd0hMAzLB5sqG5rig8XIarjb7c4Gjsx19syZTcQ860nt7mbiXVjf228iXPdWukdONoCbmjd0hGe2ZN_Sq9nH9M5EJSC_txKeHSz8IO-EheJut2PidiUnaXtLqxHmplbxfQT8WBZqP-mLgZpJ_vj6tqv1ECmFzZ-Iav4ZFBRzSa1mzWkcFFbVf3dkdFUwn9u43SgW1r92dc0vm4C_3AzAS8Q1SBZNUmlRXC8N3OpT7uAYtOzafYssA1PhHfHVBX214fhnCqaucAQquK8XuZaEWag
1.4 浏览器访问k8s dashboard
通过1.3.2查看的dashboard的端口号加上ip进行访问
注意是https+ip+端口如（https://192.168.3.253:31296），如果**有些浏览器无法打开https的地址，请更换浏览器**
复制输入1.3.5生成的token，点击【登录】
查看running的pods
[root@server253 ~]# kubectl get pods --namespace=kubernetes-dashboard
NAME READY STATUS RESTARTS AGE
dashboard-metrics-scraper-78f5d9f487-hzvvz 1/1 Running 0 60m
dashboard-metrics-scraper-78f5d9f487-lnq6n 1/1 Terminating 0 143d
kubernetes-dashboard-6bc5cb8879-58jjd 1/1 Terminating 0 143d
kubernetes-dashboard-6bc5cb8879-wrbt9 1/1 Running 0 60m
查看kubernates-dashboard service配置
kubectl edit svc kubernetes-dashboard -n kubernetes-dashboard
查看密钥
[root@server253 ~]# kubectl get secret -n kubernetes-dashboard
NAME TYPE DATA AGE
admin-user-token-jxmr8 kubernetes.io/service-account-token 3 143d
default-token-vbzsv kubernetes.io/service-account-token 3 143d
kubernetes-dashboard-certs Opaque 0 143d
kubernetes-dashboard-csrf Opaque 1 143d
kubernetes-dashboard-key-holder Opaque 2 143d
kubernetes-dashboard-token-v76cd kubernetes.io/service-account-token 3 143d
生成新的密钥（使用上面kubernetes-dashboard-token-XXX）
[root@server253 ~]# kubectl describe secret kubernetes-dashboard-token-v76cd -n kubernetes-dashboard
Name: kubernetes-dashboard-token-v76cd
Namespace: kubernetes-dashboard
Labels: <none>
Annotations: kubernetes.io/service-account.name: kubernetes-dashboard
kubernetes.io/service-account.uid: 12a666d1-fa41-4042-8460-0d32791ba1aa
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 20 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6Ik42T1ZuWWpSVVpZVF9wQlA4OXdncjF5WE5WZjdXQnBXRHM3S1JBTGhpS0kifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZC10b2tlbi12NzZjZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjEyYTY2NmQxLWZhNDEtNDA0Mi04NDYwLTBkMzI3OTFiYTFhYSIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDprdWJlcm5ldGVzLWRhc2hib2FyZDprdWJlcm5ldGVzLWRhc2hib2FyZCJ9.ElGoSA9TRzkj8VkNQXgTfeI_FzhjBJcViSbYrR9blQosGmbLe5R0ltx6bSd8EzX-4oowi-lGfjLEZbEk8FEhIK1Yeff3w4sfOmBpL8KF3flxHfBB5zCrKf7x_KcC2-N9_a5PLlgm2_zGwizMODlL68xia1Vf4jchgQU9mgG1o8xNDj1e6cj-XQCgCTouDgRAu0-6RxJP-m8IGJDEjl9FMfBbziCalD2kqG5uvtQWvUFfP53TxQANAnMqLia1OpLZAXbzEvENtHzTXeZe5iRd7DUpa0tpSVwp0sdMJn-6mt83DaByg-wzPTGl0_VMeIisHcz04TYn01Ft1oio0ra7Tw
2.集群加入新的node节点
先获取加入集群的命令：在master节点执行：
[root@server253 kubernetes]# kubeadm token create --print-join-command
W1007 17:01:39.376097 26566 configset.go:202] WARNING: kubeadm cannot validate component configs for API groups [kubelet.config.k8s.io kubeproxy.config.k8s.io]
kubeadm join 192.168.2.253:6443 --token mbd8fr.limeyyhyhsbu81nz --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5a
现在执行node249加入master节点操作
[root@node249 containers]# kubeadm join 192.168.2.253:6443 --token vzakyn.jcxip00b5kimfson --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5a
W0516 17:42:43.018243 4753 join.go:346] [preflight] WARNING: JoinControlPane.controlPlane settings will be ignored when control-plane flag is not set.
[preflight] Running pre-flight checks
[WARNING Service-Docker]: docker service is not enabled, please run 'systemctl enable docker.service'
[WARNING Hostname]: hostname "node249" could not be reached
[WARNING Hostname]: hostname "node249": lookup node249 on 8.8.8.8:53: no such host
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.18" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.
Run 'kubectl get nodes' on the control-plane to see this node join the cluster.
遇到问题：查看nodes，这个时候node节点 Status为NotReady，我们怎么处理呢？
[root@localhost kubernetes]# kubectl get nodes
NAME STATUS ROLES AGE VERSION
node249 NotReady <none> 90s v1.18.0
node252 Ready <none> 64m v1.18.0
server253 Ready master 3h v1.18.0
遇到问题：
[ERROR FileContent--proc-sys-net-ipv4-ip_forward]: /proc/sys/net/ipv4/ip_forward contents are not set to 1
解决：
[root@localhost ~]# sysctl -w net.ipv4.ip_forward=1
3.集群加入master节点
首先，我们将第一个master的证书文件复制到新增master节点
[root@server253 kubernetes]# ssh root@192.168.2.251 mkdir -p /etc/kubernetes/pki/etcd
root@192.168.2.251's password:
[root@server253 kubernetes]# scp admin.conf root@192.168.2.251:/etc/kubernetes
root@192.168.2.251's password:
admin.conf 100% 5449 18.6MB/s 00:00
[root@server253 kubernetes]# scp pki/{ca.*,sa.*,front-proxy-ca.*} root@192.168.2.251:/etc/kubernetes/pki
root@192.168.2.251's password:
ca.crt 100% 1025 4.4MB/s 00:00
ca.key 100% 1679 8.7MB/s 00:00
sa.key 100% 1679 8.3MB/s 00:00
sa.pub 100% 451 2.8MB/s 00:00
front-proxy-ca.crt 100% 1038 6.0MB/s 00:00
front-proxy-ca.key 100% 1675 7.9MB/s 00:00
[root@server253 kubernetes]# scp pki/etcd/ca.* root@192.168.2.251:/etc/kubernetes/pki/etcd/
root@192.168.2.251's password:
ca.crt 100% 1017 3.8MB/s 00:00
ca.key
然后在新增节点执行kubeadm join命令，来源于第一个master kubeadmin init， 注意后面添加--control-plane
kubeadm join 192.168.2.253:6443 --token vzakyn.jcxip00b5kimfson --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5a --control-plane
如果之前命令过期，可以重新生成新的join命令
kubeadm token create --print-join-command
添加过程中如果报错：unable to add a new control plane instance a cluster that doesn't have a stable controlPlaneEndpoint address
先查看 kubeadm config view或者 kubectl describe cm kubeadm-config -n kube-system
[root@server253 kubernetes]# kubeadm config view
apiServer:
extraArgs:
authorization-mode: Node,RBAC
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.18.1
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16
serviceSubnet: 10.96.0.0/12
scheduler: {}
[root@server253 kubernetes]# kubectl describe cm kubeadm-config -n kube-system
Name: kubeadm-config
Namespace: kube-system
Labels: <none>
Annotations: <none>
Data
====
ClusterConfiguration:
----
apiServer:
extraArgs:
authorization-mode: Node,RBAC
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.18.1
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16
serviceSubnet: 10.96.0.0/12
scheduler: {}
ClusterStatus:
----
apiEndpoints:
server253:
advertiseAddress: 192.168.2.253
bindPort: 6443
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterStatus
BinaryData
====
Events: <none>
发现kubeadm-config内没有controlPlaneEndpoint
所以我们在第一个master配置一下controlPlaneEndpoint
[root@server253 kubernetes]# kubectl edit cm kubeadm-config -n kube-system
添加后能够通过kubeadm config view查看内容已变化
[root@server253 kubernetes]# kubeadm config view
apiServer:
extraArgs:
authorization-mode: Node,RBAC
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta2
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns:
type: CoreDNS
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.18.1
controlPlaneEndpoint: 192.168.2.253:6443 #########这里变了
networking:
dnsDomain: cluster.local
podSubnet: 10.244.0.0/16
serviceSubnet: 10.96.0.0/12
scheduler: {}
再次回到新增master节点执行kubadm join 注意后面添加--control-plane
kubeadm join 192.168.2.253:6443 --token vzakyn.jcxip00b5kimfson --discovery-token-ca-cert-hash sha256:7eeb949618f9bfe4021795e460243851ca67abab018a3a59a947a754cc245a5a --control-plane
等待成功后我们再次查看kubeadm-config，能够发现参数自动生成了apiEndpoints
