k8s

297 阅读9分钟

本文以参与[新人创作礼]活动,一起开启掘金创作之路

Kubernetes 环境安装

安装必备工具

yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y

关闭防火墙、Selinux、dnsmasq、swap、同步时间

systemctl disable --now firewalld
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager
setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

setenforce 0 && sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' > /etc/timezone
ntpdate time2.aliyun.com

# 加入到crontab中
crontab -e
*/5 * * * * ntpdate time2.aliyun.com

# 临时生效
ulimit -SHn 65535

# 永久生效
vim /etc/security/limits.conf

* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited

master1 节点免密登陆其它节点

ssh-keygen -t rsa for i in master1 master2 master3 node1 node2;do ssh-copy-id -i .ssh/id_rsa.pub $i;done

更新内核

离线升级

yum update -y --exclude=kernel* && reboot

wget https://ossjc-1252545319.cos.ap-shanghai.myqcloud.com/other/linux/kernel-ml-4.18.9/kernel-ml-4.18.9.tar.gz
[root@localhost ~]# tar xf kernel-ml-4.18.9.tar.gz
[root@localhost ~]# cd kernel-ml-4.18.9
[root@k8s-master kernel-ml-4.18.9]# yum localinstall kernel-ml*
Loaded plugins: fastestmirror
Examining kernel-ml-4.18.9-1.el7.elrepo.x86_64.rpm: kernel-ml-4.18.9-1.el7.elrepo.x86_64
Marking kernel-ml-4.18.9-1.el7.elrepo.x86_64.rpm to be installed
Examining kernel-ml-devel-4.18.9-1.el7.elrepo.x86_64.rpm: kernel-ml-devel-4.18.9-1.el7.elrepo.x86_64
Marking kernel-ml-devel-4.18.9-1.el7.elrepo.x86_64.rpm to be installed
Examining linux-firmware-20180911-69.git85c5d90.el7.noarch.rpm: linux-firmware-20180911-69.git85c5d90.el7.noarch
linux-firmware-20180911-69.git85c5d90.el7.noarch.rpm: does not update installed package.
Resolving Dependencies
--> Running transaction check
---> Package kernel-ml.x86_64 0:4.18.9-1.el7.elrepo will be installed
---> Package kernel-ml-devel.x86_64 0:4.18.9-1.el7.elrepo will be installed
--> Finished Dependency Resolution

Dependencies Resolved

==========================================================================================================================================================================================================
 Package                                    Arch                              Version                                        Repository                                                              Size
==========================================================================================================================================================================================================
Installing:
 kernel-ml                                  x86_64                            4.18.9-1.el7.elrepo                            /kernel-ml-4.18.9-1.el7.elrepo.x86_64                                  203 M
 kernel-ml-devel                            x86_64                            4.18.9-1.el7.elrepo                            /kernel-ml-devel-4.18.9-1.el7.elrepo.x86_64                             45 M

Transaction Summary
==========================================================================================================================================================================================================
Install  2 Packages

Total size: 248 M
Installed size: 248 M
Is this ok [y/d/N]: y
Downloading packages:
Running transaction check
Running transaction test
Transaction test succeeded
Running transaction
  Installing : kernel-ml-devel-4.18.9-1.el7.elrepo.x86_64                                                                                                                                             1/2
  Installing : kernel-ml-4.18.9-1.el7.elrepo.x86_64                                                                                                                                                                                                                         2/2
  Verifying  : kernel-ml-4.18.9-1.el7.elrepo.x86_64                                                                                                                                                                                                                         1/2
  Verifying  : kernel-ml-devel-4.18.9-1.el7.elrepo.x86_64                                                                                                                                                                                                                   2/2

Installed:
  kernel-ml.x86_64 0:4.18.9-1.el7.elrepo                                                                                              kernel-ml-devel.x86_64 0:4.18.9-1.el7.elrepo

Complete!

设置默认启动新内核 并重新生成 grub2

# 查看当前内核
[root@k8s-master kernel-ml-4.18.9]# cat /boot/grub2/grub.cfg |grep ^menuentry
menuentry 'CentOS Linux (4.18.9-1.el7.elrepo.x86_64) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-1127.el7.x86_64-advanced-abb47b92-d268-4d8c-a9df-43bf44522cab' {
menuentry 'CentOS Linux (3.10.0-1127.el7.x86_64) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-3.10.0-1127.el7.x86_64-advanced-abb47b92-d268-4d8c-a9df-43bf44522cab' {
menuentry 'CentOS Linux (0-rescue-63f20fbd07a048dc88574233e1ad966b) 7 (Core)' --class centos --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'gnulinux-0-rescue-63f20fbd07a048dc88574233e1ad966b-advanced-abb47b92-d268-4d8c-a9df-43bf44522cab' {

# 设置启动内核
[root@localhost kernel-ml-4.18.9]# grub2-set-default 0
或者
[root@localhost kernel-ml-4.18.9]# grub2-set-default 'CentOS Linux (4.18.9-1.el7.elrepo.x86_64) 7 (Core)'

[root@localhost kernel-ml-4.18.9]# grub2-mkconfig -o /boot/grub2/grub.cfg

#重启系统
[root@localhost kernel-ml-4.18.9]# reboot

确认使用的内核版本

[root@localhost ~]# uname -r
4.18.9-1.el7.elrepo.x86_64

在线升级

下载公钥

rpm --import www.elrepo.org/RPM-GPG-KEY… 安装 elrepo repo rpm -Uvh www.elrepo.org/elrepo-rele… 仓库启用后,列出可用的内核相关包: yum --disablerepo="*" --enablerepo="elrepo-kernel" list available 查询所有 kernel 包

rpm -qa|grep kernel
kernel-devel-3.10.0-862.6.3.el7.x86_64
kernel-tools-libs-3.10.0-862.6.3.el7.x86_64
kernel-tools-3.10.0-862.6.3.el7.x86_64
kernel-devel-3.10.0-862.el7.x86_64
abrt-addon-kerneloops-2.1.11-50.el7.centos.x86_64
kernel-3.10.0-862.6.3.el7.x86_64
kernel-headers-3.10.0-862.6.3.el7.x86_64

根据上述查询结果,删除所有旧内核相关包

yum remove kernel-devel kernel-tools-libs kernel-tools kernel-headers 卸载上述软件包会连带卸载以下安装包需要更新后,重新安装。kernel-devel 会连带删除 systemtap 和 systemtap-devel。kernel-headers 则和下面都有关系。

compat-glibc   x86_64   1:2.12-4.el7.centos
compat-glibc-headers  x86_64   1:2.12-4.el7.centos
gcc x86_64   4.8.5-28.el7_5.1
gcc-c++x86_64   4.8.5-28.el7_5.1
gcc-gfortran   x86_64   4.8.5-28.el7_5.1
glibc-develx86_64   2.17-222.el7
glibc-headers  x86_64   2.17-222.el7
libquadmath-devel  x86_64   4.8.5-28.el7_5.1
libtool x86_64   2.4.2-22.el7_3
systemtap   x86_64   3.2-8.el7_5
systemtap-develx86_64   3.2-8.el7_5

安装新内核

yum --enablerepo=elrepo-kernel install -y kernel-ml.x86_64 安装新内核相关软件 yum --disablerepo=* --enablerepo=elrepo-kernel install -y kernel-ml-devel kernel-ml-tools kernel-ml-tools-libs kernel-ml-tools-libs-devel kernel-ml-headers 查看默认内核 grub2-editenv list

将新内核设为默认 grbu 启动项

grub2-set-default 0

生成 grub 配置文件

grub2-mkconfig -o /etc/grub2.cfg
reboot   #不重启也可以

删除旧内核

yum remove kernel-3.10.0-862.6.3.el7.x86_64

安装更新旧内核时被卸载的软件

yum install -y compat-glibc compat-glibc-headers gcc gcc-c++ gcc-gfortran glibc-devel glibc-headers libquadmath-devel libtool systemtap systemtap-devel

查看默认内核

grub2-editenv list
saved_entry=CentOS Linux (4.18.11-1.el7.elrepo.x86_64) 7 (Core)

安装 ipvs

所有节点安装 ipvsadm

yum install ipvsadm ipset sysstat conntrack libsecomp -y

所有节点设置 ipvs 模块,在内核 4.19+版本中 nf_conntrack_ipv4 已经改成 nf_conntrack, 4.18 以下使用 nf_conntrack_ipv4 即可:

modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_contrack

vim /etc/modules-load.d/ipvs.conf

# 加入以下内容
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack_ipv4
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip

然后执行

systemctl enable --now systemd-modules-load.service

检查是否加载

lsmod | grep -e ip_vs -e nf_conntrack_ipv4

K8S 集群需要的内核参数

cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF

#重新加载配置
sysctl --system

报错1:显示/proc/sys/net/bridge/bridge-nf-call-iptables:没有这个文件或者目录
#加载网桥过滤模块
modprobe br_netfilter

报错2:显示sysctl: cannot stat /proc/sys/net/netfilter/nf_conntrack_max: 没有那个文件或目录
modprobe ip_conntrack

#查看网桥过滤模块是否加载成功
lsmod | grep br_netfilter

Kubernetes 组件安装

docker 安装

# 安装docker所需的工具
yum install -y yum-utils device-mapper-persistent-data lvm2
# 配置阿里云的docker源
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 指定安装这个版本的docker-ce
yum install -y docker-ce-18.09.9-3.el7
# 启动docker
systemctl enable docker && systemctl start docker
  • docker 加速
创建 /etc/docker/daemon.json
由于新版 kubelet 建议使用 systemd,所以可以把 docker 的 CgroupDriver 改成 systemd

cat > /etc/docker/daemon.conf << EOF
{
	"registry-mirrors": ["https://yp8sluaa.mirror.aliyuncs.com"]
}
EOF

  • 设置开启启动
sudo systemctl daemon-reload
sudo systemctl restart docker
systemctl enable docker
  • 查看 docker 信息

    docker info

安装 k8s v1.16.0 master 管理节点

如果还没安装 docker,请参照本文步骤二安装 docker-ce 18.09.9(所有机器)安装。如果没设置 k8s 环境准备条件,请参照本文步骤三设置 k8s 环境准备条件(所有机器)执行。

以上两个步骤检查完毕之后,继续以下步骤。

安装 kubeadm、kubelet、kubectl 由于官方 k8s 源在 google,国内无法访问,这里使用阿里云 yum 源

# 执行配置k8s阿里云源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 安装kubeadm、kubectl、kubelet
yum install -y kubectl-1.16.0-0 kubeadm-1.16.0-0 kubelet-1.16.0-0

# 启动kubelet服务
systemctl enable kubelet && systemctl start kubelet

初始化 k8s 以下这个命令开始安装 k8s 需要用到的 docker 镜像,因为无法访问到国外网站,所以这条命令使用的是国内的阿里云的源(registry.aliyuncs.com/google_containers)。另一个非常重要的是:这里的--apiserver-advertise-address 使用的是 master 和 node 间能互相 ping 通的 ip,我这里是 192.168.99.104,刚开始在这里被坑了一个晚上,你请自己修改下 ip 执行。这条命令执行时会卡在[preflight] You can also perform this action in beforehand using ''kubeadm config images pull,大概需要 2 分钟,请耐心等待。

# 下载管理节点中用到的6个docker镜像,你可以使用docker images查看到
# 这里需要大概两分钟等待,会卡在[preflight] You can also perform this action in beforehand using ''kubeadm config images pull
kubeadm init --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.16.0 --apiserver-advertise-address 192.168.99.104 --pod-network-cidr=10.244.0.0/16 --token-ttl 0

上面安装完后,会提示你输入如下命令,复制粘贴过来,执行即可。

# 上面安装完成后,k8s会提示你输入如下命令,执行
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

记住 node 加入集群的命令 上面 kubeadm init 执行成功后会返回给你 node 节点加入集群的命令,等会要在 node 节点上执行,需要保存下来,如果忘记了,可以使用如下命令获取。

kubeadm token create --print-join-command

以上,安装 master 节点完毕。可以使用 kubectl get nodes 查看一下,此时 master 处于 NotReady 状态,暂时不用管。

安装 k8s v1.16.0 node 工作节点

如果还没安装 docker,请参照本文步骤二安装 docker-ce 18.09.9(所有机器)安装。如果没设置 k8s 环境准备条件,请参照本文步骤三设置 k8s 环境准备条件(所有机器)执行。

以上两个步骤检查完毕之后,继续以下步骤。

安装 kubeadm、kubelet

# 执行配置k8s阿里云源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 安装kubeadm、kubectl、kubelet
yum install -y  kubeadm-1.16.0-0 kubelet-1.16.0-0

# 启动kubelet服务
systemctl enable kubelet && systemctl start kubelet

加入集群 这里加入集群的命令每个人都不一样,可以登录 master 节点,使用 kubeadm token create --print-join-command 来获取。获取后执行如下。

kubeadm join 192.168.99.104:6443 --token ncfrid.7ap0xiseuf97gikl \
    --discovery-token-ca-cert-hash sha256:47783e9851a1a517647f1986225f104e81dbfd8fb256ae55ef6d68ce9334c6a2

加入成功后,可以在 master 节点上使用 kubectl get nodes 命令查看到加入的节点。

安装 flannel(master 机器)

以上步骤安装完后,机器搭建起来了,但状态还是 NotReady 状态,如下图,master 机器需要安装 flanneld。 下载官方 fannel 配置文件 使用 wget 命令,地址为:(raw.githubusercontent.com/coreos/flan…

安装 fannel

kubectl apply -f kube-flannel.yml

大功告成

至此,k8s 集群搭建完成,如下图节点已为 Ready 状态,大功告成,完结撒花。

附录

这是 kube-flannel.yml 文件的内容,已经将无法访问的地址(quay.io)全部改为国内可以访问的地址(quay-mirror.qiniu.com)。我们新建一个 kube-flannel.yml 文件,复制粘贴该内容即可。

---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
  name: psp.flannel.unprivileged
  annotations:
    seccomp.security.alpha.kubernetes.io/allowedProfileNames: docker/default
    seccomp.security.alpha.kubernetes.io/defaultProfileName: docker/default
    apparmor.security.beta.kubernetes.io/allowedProfileNames: runtime/default
    apparmor.security.beta.kubernetes.io/defaultProfileName: runtime/default
spec:
  privileged: false
  volumes:
    - configMap
    - secret
    - emptyDir
    - hostPath
  allowedHostPaths:
    - pathPrefix: "/etc/cni/net.d"
    - pathPrefix: "/etc/kube-flannel"
    - pathPrefix: "/run/flannel"
  readOnlyRootFilesystem: false
  # Users and groups
  runAsUser:
    rule: RunAsAny
  supplementalGroups:
    rule: RunAsAny
  fsGroup:
    rule: RunAsAny
  # Privilege Escalation
  allowPrivilegeEscalation: false
  defaultAllowPrivilegeEscalation: false
  # Capabilities
  allowedCapabilities: ['NET_ADMIN']
  defaultAddCapabilities: []
  requiredDropCapabilities: []
  # Host namespaces
  hostPID: false
  hostIPC: false
  hostNetwork: true
  hostPorts:
  - min: 0
    max: 65535
  # SELinux
  seLinux:
    # SELinux is unused in CaaSP
    rule: 'RunAsAny'
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
rules:
  - apiGroups: ['extensions']
    resources: ['podsecuritypolicies']
    verbs: ['use']
    resourceNames: ['psp.flannel.unprivileged']
  - apiGroups:
      - ""
    resources:
      - pods
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes/status
    verbs:
      - patch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: flannel
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: flannel
subjects:
- kind: ServiceAccount
  name: flannel
  namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: flannel
  namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: kube-flannel-cfg
  namespace: kube-system
  labels:
    tier: node
    app: flannel
data:
  cni-conf.json: |
    {
      "name": "cbr0",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "flannel",
          "delegate": {
            "hairpinMode": true,
            "isDefaultGateway": true
          }
        },
        {
          "type": "portmap",
          "capabilities": {
            "portMappings": true
          }
        }
      ]
    }
  net-conf.json: |
    {
      "Network": "10.244.0.0/16",
      "Backend": {
        "Type": "vxlan"
      }
    }
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-amd64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - amd64
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-amd64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-amd64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
            add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-arm64
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - arm64
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-arm64
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-arm64
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-arm
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - arm
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-arm
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-arm
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-ppc64le
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - ppc64le
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-ppc64le
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-ppc64le
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kube-flannel-ds-s390x
  namespace: kube-system
  labels:
    tier: node
    app: flannel
spec:
  selector:
    matchLabels:
      app: flannel
  template:
    metadata:
      labels:
        tier: node
        app: flannel
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: beta.kubernetes.io/os
                    operator: In
                    values:
                      - linux
                  - key: beta.kubernetes.io/arch
                    operator: In
                    values:
                      - s390x
      hostNetwork: true
      tolerations:
      - operator: Exists
        effect: NoSchedule
      serviceAccountName: flannel
      initContainers:
      - name: install-cni
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-s390x
        command:
        - cp
        args:
        - -f
        - /etc/kube-flannel/cni-conf.json
        - /etc/cni/net.d/10-flannel.conflist
        volumeMounts:
        - name: cni
          mountPath: /etc/cni/net.d
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      containers:
      - name: kube-flannel
        image: quay-mirror.qiniu.com/coreos/flannel:v0.11.0-s390x
        command:
        - /opt/bin/flanneld
        args:
        - --ip-masq
        - --kube-subnet-mgr
        resources:
          requests:
            cpu: "100m"
            memory: "50Mi"
          limits:
            cpu: "100m"
            memory: "50Mi"
        securityContext:
          privileged: false
          capabilities:
             add: ["NET_ADMIN"]
        env:
        - name: POD_NAME
          valueFrom:
            fieldRef:
              fieldPath: metadata.name
        - name: POD_NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
        volumeMounts:
        - name: run
          mountPath: /run/flannel
        - name: flannel-cfg
          mountPath: /etc/kube-flannel/
      volumes:
        - name: run
          hostPath:
            path: /run/flannel
        - name: cni
          hostPath:
            path: /etc/cni/net.d
        - name: flannel-cfg
          configMap:
            name: kube-flannel-cfg