一、系统环境初始化

1.1、ssh免密登录与组件安装

kube-proxy使用ipvs模式，ipvsadm为ipvs的管理工具
etcd集群各机器需要时间同步，chrony用于系统时间同步

[root@VM-16-14-centos ~]# vim /etc/hosts

10.206.16.6 node1
10.206.16.4 node2
10.206.16.14 master
10.206.16.14 harbor01.io

[root@VM-16-14-centos ~]# ssh node1
[root@VM-16-14-centos ~]# ssh node2

➜  .ssh ansible master,node1,node2 -m command -a "yum install -y epel-release"

➜  .ssh ansible master,node1,node2 -m command -a "yum install -y chrony conntrack ipvsadm ipset jq iptables curl sysstat libseccomp wget socat git vim  lrzsz wget man tree rsync gcc gcc-c++ cmake telnet"

1.2、firewalld、iptables、selinux、swap配置

➜  .ssh ansible master,node1,node2 -m command -a "systemctl stop firewalld"

➜  .ssh ansible master,node1,node2 -m command -a "systemctl disable firewalld"

#➜  .ssh ansible master,node1,node2 -m command -a "iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat"
node2 | FAILED | rc=2 >>
Bad argument `iptables'
Try `iptables -h' or 'iptables --help' for more information.non-zero return code
node1 | FAILED | rc=2 >>
Bad argument `iptables'
Try `iptables -h' or 'iptables --help' for more information.non-zero return code
master | FAILED | rc=2 >>
Bad argument `iptables'
Try `iptables -h' or 'iptables --help' for more information.non-zero return code
➜  .ssh ansible master,node1,node2 -m command -a "iptables -P FORWARD ACCEPT"

➜  .ssh ansible master,node1,node2 -m command -a "swapoff -a"

➜  .ssh ansible master,node1,node2 -m command -a "iptables -F"
➜  .ssh ansible master,node1,node2 -m command -a "iptables -X"
➜  .ssh ansible master,node1,node2 -m command -a "iptables -F -t nat "
➜  .ssh ansible master,node1,node2 -m command -a "iptables -X -t nat"

➜  .ssh ansible master,node1,node2 -m command -a "iptables -P FORWARD ACCEPT"

➜  .ssh ansible master,node1,node2 -m command -a "swapoff -a"

[root@VM-16-14-centos ~]# sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
[root@VM-16-6-centos ~]# sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
[root@VM-16-4-centos ~]# sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab

➜  .ssh ansible master,node1,node2 -m command -a "setenforce 0"
node2 | FAILED | rc=1 >>
setenforce: SELinux is disablednon-zero return code
node1 | FAILED | rc=1 >>
setenforce: SELinux is disablednon-zero return code
master | FAILED | rc=1 >>
setenforce: SELinux is disablednon-zero return code

[root@VM-16-14-centos ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@VM-16-6-centos ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
[root@VM-16-4-centos ~]# sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

1.3、sysctl

关闭tcp_tw_recycle，否则与NAT冲突，可能导致服务不通


[root@VM-16-14-centos ~]# cat > sysctl.conf <<EOF
> net.bridge.bridge-nf-call-iptables=1
> net.bridge.bridge-nf-call-ip6tables=1
> net.ipv4.ip_forward=1
> net.ipv4.tcp_tw_recycle=0
> vm.swappiness=0
> vm.overcommit_memory=1
> vm.panic_on_oom=0
> fs.inotify.max_user_instances=8192
> fs.inotify.max_user_watches=1048576
> fs.file-max=52706963
> fs.nr_open=52706963
> net.ipv6.conf.all.disable_ipv6=1
> net.netfilter.nf_conntrack_max=2310720
> EOF


[root@VM-16-6-centos ~]# cat > sysctl.conf <<EOF
> net.bridge.bridge-nf-call-iptables=1
> net.bridge.bridge-nf-call-ip6tables=1
> net.ipv4.ip_forward=1
> net.ipv4.tcp_tw_recycle=0
> vm.swappiness=0
> vm.overcommit_memory=1
> vm.panic_on_oom=0
> fs.inotify.max_user_instances=8192
> fs.inotify.max_user_watches=1048576
> fs.file-max=52706963
> fs.nr_open=52706963
> net.ipv6.conf.all.disable_ipv6=1
> net.netfilter.nf_conntrack_max=2310720
> EOF


[root@VM-16-4-centos ~]# cat > sysctl.conf <<EOF
> net.bridge.bridge-nf-call-iptables=1
> net.bridge.bridge-nf-call-ip6tables=1
> net.ipv4.ip_forward=1
> net.ipv4.tcp_tw_recycle=0
> vm.swappiness=0
> vm.overcommit_memory=1
> vm.panic_on_oom=0
> fs.inotify.max_user_instances=8192
> fs.inotify.max_user_watches=1048576
> fs.file-max=52706963
> fs.nr_open=52706963
> net.ipv6.conf.all.disable_ipv6=1
> net.netfilter.nf_conntrack_max=2310720
> EOF


➜  .ssh ansible master,node1,node2 -m command -a "cp sysctl.conf  /etc/sysctl.d/sysctl.conf" -v
➜  .ssh ansible master,node1,node2 -m command -a "sysctl -p /etc/sysctl.d/sysctl.conf" -v
No config file found; using defaults
node1 | FAILED | rc=255 >>
net.ipv4.ip_forward = 1
vm.swappiness = 0
vm.overcommit_memory = 1
vm.panic_on_oom = 0
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 1048576
fs.file-max = 52706963
fs.nr_open = 52706963
net.ipv6.conf.all.disable_ipv6 = 1
net.netfilter.nf_conntrack_max = 2310720sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: 没有那个文件或目录
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: 没有那个文件或目录
sysctl: cannot stat /proc/sys/net/ipv4/tcp_tw_recycle: 没有那个文件或目录non-zero return code


node2 | FAILED | rc=255 >>
net.ipv4.ip_forward = 1
vm.swappiness = 0
vm.overcommit_memory = 1
vm.panic_on_oom = 0
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 1048576
fs.file-max = 52706963
fs.nr_open = 52706963
net.ipv6.conf.all.disable_ipv6 = 1
net.netfilter.nf_conntrack_max = 2310720sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: 没有那个文件或目录
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: 没有那个文件或目录
sysctl: cannot stat /proc/sys/net/ipv4/tcp_tw_recycle: 没有那个文件或目录non-zero return code


master | FAILED | rc=255 >>
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
vm.swappiness = 0
vm.overcommit_memory = 1
vm.panic_on_oom = 0
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 1048576
fs.file-max = 52706963
fs.nr_open = 52706963
net.ipv6.conf.all.disable_ipv6 = 1
net.netfilter.nf_conntrack_max = 2310720sysctl: cannot stat /proc/sys/net/ipv4/tcp_tw_recycle: 没有那个文件或目录non-zero return code

1.4、IPVS模块


[root@VM-16-14-centos ~]# cat >>/etc/modules-load.d/kubernetes.conf<<EOF
> ip_vs_dh
> ip_vs_ftp
> ip_vs
> ip_vs_lblc
> ip_vs_lblcr
> ip_vs_lc
> ip_vs_nq
> ip_vs_pe_sip
> ip_vs_rr
> ip_vs_sed
> ip_vs_sh
> ip_vs_wlc
> ip_vs_wrr
> nf_conntrack_ipv4
> overlay
> br_netfilter
> EOF


[root@VM-16-6-centos ~]# cat >>/etc/modules-load.d/kubernetes.conf<<EOF
> ip_vs_dh
> ip_vs_ftp
> ip_vs
> ip_vs_lblc
> ip_vs_lblcr
> ip_vs_lc
> ip_vs_nq
> ip_vs_pe_sip
> ip_vs_rr
> ip_vs_sed
> ip_vs_sh
> ip_vs_wlc
> ip_vs_wrr
> nf_conntrack_ipv4
> overlay
> br_netfilter
> EOF


[root@VM-16-4-centos ~]# cat >>/etc/modules-load.d/kubernetes.conf<<EOF
> ip_vs_dh
> ip_vs_ftp
> ip_vs
> ip_vs_lblc
> ip_vs_lblcr
> ip_vs_lc
> ip_vs_nq
> ip_vs_pe_sip
> ip_vs_rr
> ip_vs_sed
> ip_vs_sh
> ip_vs_wlc
> ip_vs_wrr
> nf_conntrack_ipv4
> overlay
> br_netfilter
> EOF
[root@VM-16-4-centos ~]#

➜  .ssh ansible master,node1,node2 -m command -a "systemctl enable systemd-modules-load.service" -v
No config file found; using defaults
node1 | CHANGED | rc=0 >>
The unit files have no installation config (WantedBy, RequiredBy, Also, Alias
settings in the [Install] section, and DefaultInstance for template units).
This means they are not meant to be enabled using systemctl.
Possible reasons for having this kind of units are:
1) A unit may be statically enabled by being symlinked from another unit's
   .wants/ or .requires/ directory.
2) A unit's purpose may be to act as a helper for some other unit which has
   a requirement dependency on it.
3) A unit may be started when needed via activation (socket, path, timer,
   D-Bus, udev, scripted systemctl call, ...).
4) In case of template units, the unit is meant to be enabled with some
   instance name specified.
node2 | CHANGED | rc=0 >>
The unit files have no installation config (WantedBy, RequiredBy, Also, Alias
settings in the [Install] section, and DefaultInstance for template units).
This means they are not meant to be enabled using systemctl.
Possible reasons for having this kind of units are:
1) A unit may be statically enabled by being symlinked from another unit's
   .wants/ or .requires/ directory.
2) A unit's purpose may be to act as a helper for some other unit which has
   a requirement dependency on it.
3) A unit may be started when needed via activation (socket, path, timer,
   D-Bus, udev, scripted systemctl call, ...).
4) In case of template units, the unit is meant to be enabled with some
   instance name specified.
master | CHANGED | rc=0 >>
The unit files have no installation config (WantedBy, RequiredBy, Also, Alias
settings in the [Install] section, and DefaultInstance for template units).
This means they are not meant to be enabled using systemctl.
Possible reasons for having this kind of units are:
1) A unit may be statically enabled by being symlinked from another unit's
   .wants/ or .requires/ directory.
2) A unit's purpose may be to act as a helper for some other unit which has
   a requirement dependency on it.
3) A unit may be started when needed via activation (socket, path, timer,
   D-Bus, udev, scripted systemctl call, ...).
4) In case of template units, the unit is meant to be enabled with some
   instance name specified.
   
➜  .ssh ansible master,node1,node2 -m command -a "lsmod | grep -e ip_vs -e nf_conntrack_ipv4" -v
No config file found; using defaults
node1 | FAILED | rc=1 >>
Usage: lsmodnon-zero return code
node2 | FAILED | rc=1 >>
Usage: lsmodnon-zero return code
master | FAILED | rc=1 >>
Usage: lsmodnon-zero return code

➜  .ssh ansible master,node1,node2 -m command -a "reboot" -v

[root@VM-16-14-centos ~]# lsmod | grep -e ip_vs -e nf_conntrack_ipv4
[root@VM-16-6-centos ~]# lsmod | grep -e ip_vs -e nf_conntrack_ipv4
[root@VM-16-4-centos ~]# lsmod | grep -e ip_vs -e nf_conntrack_ipv4

1.5、系统配置初始化

➜  .ssh ansible master,node1,node2 -m command -a "timedatectl set-timezone Asia/Shanghai"

➜  .ssh ansible master,node1,node2 -m command -a "systemctl enable chronyd"

➜  .ssh ansible master,node1,node2 -m command -a "systemctl start chronyd"

➜  .ssh ansible master,node1,node2 -m command -a "systemctl stop postfix"
node1 | FAILED | rc=5 >>
Failed to stop postfix.service: Unit postfix.service not loaded.non-zero return code
node2 | FAILED | rc=5 >>
Failed to stop postfix.service: Unit postfix.service not loaded.non-zero return code
master | FAILED | rc=5 >>
Failed to stop postfix.service: Unit postfix.service not loaded.non-zero return code

➜  .ssh ansible master,node1,node2 -m command -a "systemctl disable postfix"
node2 | FAILED | rc=1 >>
Failed to disable unit: Unit file postfix.service does not exist.non-zero return code
master | FAILED | rc=1 >>
Failed to disable unit: Unit file postfix.service does not exist.non-zero return code
node1 | FAILED | rc=1 >>
Failed to disable unit: Unit file postfix.service does not exist.non-zero return code

[root@VM-16-14-centos ~]# mkdir -p /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert
[root@VM-16-6-centos ~]# mkdir -p /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert
[root@VM-16-4-centos ~]# mkdir -p /opt/k8s/{bin,work} /etc/{kubernetes,etcd}/cert

➜  .ssh ansible master,node1,node2 -m command -a "cat /etc/redhat-release"
node1 | CHANGED | rc=0 >>
CentOS Linux release 8.4.2105
node2 | CHANGED | rc=0 >>
CentOS Linux release 8.4.2105
master | CHANGED | rc=0 >>
CentOS Linux release 8.4.2105
➜  .ssh ansible master,node1,node2 -m command -a "uname -a"
node1 | CHANGED | rc=0 >>
Linux VM-16-6-centos 4.18.0-305.19.1.el8_4.x86_64 #1 SMP Wed Sep 15 15:39:39 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
node2 | CHANGED | rc=0 >>
Linux VM-16-4-centos 4.18.0-305.19.1.el8_4.x86_64 #1 SMP Wed Sep 15 15:39:39 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
master | CHANGED | rc=0 >>
Linux VM-16-14-centos 4.18.0-305.19.1.el8_4.x86_64 #1 SMP Wed Sep 15 15:39:39 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux

03 K8S 安装准备

一、系统环境初始化

1.1、ssh免密登录与组件安装

1.2、firewalld、iptables、selinux、swap配置

1.3、sysctl

1.4、IPVS模块

1.5、系统配置初始化

1.6、制作镜像、并自动创建快照[master/node1]