高可用 k3s Kubernetes集群搭建

全干攻城狮_蒲
445 阅读2分钟

本文已参与「新人创作礼」活动， 一起开启掘金创作之路。

1. 先决条件

1.1 Kubernetes集群

1.1.1 高可用 k3s Kubernetes集群

1.1.1.1 先决条件

两个以上节点、一个负载均衡器、一个 DNS 记录和一个外部 MySQL 数据库

此处我使用一台额外的机器部署依赖的服务(使用云服务时可购买现成的服务)

1.1.1.1.1 安装docker

此处我使用docker部署依赖的服务, 在要搭建k3s集群的机器也部署docker

安装脚本

# 安装
curl https://releases.rancher.com/install-docker/20.10.sh -o docker.sh
sh docker.sh --mirror Aliyun

mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": [
  	"https://docker.mirrors.ustc.edu.cn/",
  	"https://hub-mirror.c.163.com/"
  ]
}
EOF
systemctl daemon-reload
systemctl restart docker
1.1.1.1.2 启动mysql服务
docker run --name mysql -v /my/data/mysql:/var/lib/mysql -p 3306:3306 --restart=always -e MYSQL_ROOT_PASSWORD=UYDWBuadIWEH -d mysql:5.7.22
1.1.1.1.3 启动nginx服务
mkdir /my/data/nginx
tee /my/data/nginx/nginx.conf <<-'EOF'
worker_processes 1;
worker_rlimit_nofile 10000;

events {
    worker_connections 2000;
}

stream {
    upstream rancher_servers_http {
        least_conn;
        server 192.169.101.1:80 max_fails=3 fail_timeout=5s;
        server 192.169.101.2:80 max_fails=3 fail_timeout=5s;
        server 192.169.101.3:80 max_fails=3 fail_timeout=5s;
    }
    server {
        listen 80;
        proxy_pass rancher_servers_http;
    }

    upstream rancher_servers_https {
        least_conn;
        server 192.169.101.1:443 max_fails=3 fail_timeout=5s;
        server 192.169.101.2:443 max_fails=3 fail_timeout=5s;
        server 192.169.101.3:443 max_fails=3 fail_timeout=5s;
    }
    server {
        listen     443;
        proxy_pass rancher_servers_https;
    }
}
EOF

docker run --name rancher-nginx -p 80:80 -p 443:443 --restart=always -v /my/data/nginx/nginx.conf:/etc/nginx/nginx.conf:ro -d nginx:1.21.3

# docker exec rancher-nginx nginx -s reload

# 查看53端口是否占用
netstat -anp | grep 53
# 解决占用问题
systemctl stop systemd-resolved
vi /etc/systemd/resolved.conf
sed -i s/#DNS=/DNS=114.114.114.114/ /etc/systemd/resolved.conf
sed -i s/#DNSStubListener=yes/DNSStubListener=no/ /etc/systemd/resolved.conf
ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf

mkdir -r/my/data/dns-server
tee /my/data/dns-server/dnsmasq.conf <<-'EOF'
#dnsmasq config, for a complete example, see:
#  http://oss.segetech.com/intra/srv/dnsmasq.conf
#log all dns queries
log-queries
#dont use hosts nameservers
no-resolv
#use cloudflare as default nameservers, prefer 1^4
server=1.0.0.1
server=1.1.1.1
strict-order
#serve all .company queries using a specific nameserver
server=/company/10.0.0.1
#explicitly define host-ip mappings
address=/myhost.company/10.0.0.2
EOF

docker run \
	--name dnsmasq \
	--privileged \
	-d \
	-p 53:53/udp \
	-p 5380:8080 \
	-v /my/data/dns-server/dnsmasq.conf:/etc/dnsmasq.conf \
	--log-opt "max-size=100m" \
	-e "HTTP_USER=foo" \
	-e "HTTP_PASS=bar" \
	--restart always \
	jpillora/dnsmasq


docker run -d \
  --restart always \
  --privileged \
  --name coredns \
  -p 53:53/tcp \
  -p 53:53/udp \
  -v /my/data/coredns/hosts:/etc/hosts \
  -v /my/data/coredns/Corefile:/Corefile \
  coredns/coredns

docker run -d -p 53:53/tcp -p 53:53/udp -v /my/data/dns-server/:/etc/ --restart=always --cap-add=NET_ADMIN --name dns-server andyshinn/dnsmasq:2.75

echo nameserver 114.114.114.114 >> /my/data/dns-server/resolv.dnsmasq
echo 192.169.100.0 my-rancher-k3s >> /my/data/dns-server/dnsmasqhosts
echo resolv-file=/etc/resolv.dnsmasq >> /my/data/dns-server/dnsmasq.conf
echo addn-hosts=/etc/dnsmasqhosts >> /my/data/dns-server/dnsmasq.conf

docker restart dns-server

1.1.1.2 搭建k3s集群

master 节点, 两个以上

卸载命令: /usr/local/bin/k3s-uninstall.sh

# 数据库名含特殊字符可能会启动失败
curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | \
  INSTALL_K3S_VERSION=v1.21.4-k3s1 INSTALL_K3S_MIRROR=cn sh -s - server \
  --docker --datastore-endpoint="mysql://root:UYDWBuadIWEH@tcp(192.169.100.0:3306)/rancherk3s"

# 查看token
cat /var/lib/rancher/k3s/server/node-token

# 其它master节点需要使用第一个的token
curl -sfL http://rancher-mirror.cnrancher.com/k3s/k3s-install.sh | \
  K3S_TOKEN=K107c78f42c9da1e76f7ab709ac864e1e2cb6cc807c58fb37c84e8dc2aa516ef02d::server:4e1a69ffcd5f09d6055078213c9ddaef \
  INSTALL_K3S_VERSION=v1.21.4-k3s1 INSTALL_K3S_MIRROR=cn sh -s - server \
  --docker --datastore-endpoint="mysql://root:UYDWBuadIWEH@tcp(192.169.100.0:3306)/rancherk3s"

# 我是在master1上操作的
cp /etc/rancher/k3s/k3s.yaml ~/.kube/config

wget http://rancher-mirror.cnrancher.com/helm/v3.7.0/helm-v3.7.0-linux-amd64.tar.gz
tar -zxvf helm-v3.7.0-linux-amd64.tar.gz
mv linux-amd64/helm /usr/local/bin/helm
helm help

helm repo add rancher-stable http://rancher-mirror.oss-cn-beijing.aliyuncs.com/server-charts/stable

wget https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.crds.yaml
# 或者
wget https://shrill-pond-3e81.hunsh.workers.dev/https://github.com/jetstack/cert-manager/releases/download/v1.0.4/cert-manager.crds.yaml

kubectl apply --validate=false -f cert-manager.crds.yaml

kubectl create namespace cert-manager
helm repo add jetstack https://charts.jetstack.io
helm repo update
helm install cert-manager jetstack/cert-manager --namespace cert-manager --version v1.0.4
# 查看是否正确部署
kubectl get pods --namespace cert-manager

kubectl describe -n cert-manager cert-manager-cainjector-55db655cd8-54n7f

# 由于没有DNS, 在所有节点执行
echo 192.169.100.0 my-rancher-k3s >> /etc/hosts

kubectl create namespace cattle-system
helm install rancher rancher-stable/rancher \
 --namespace cattle-system \
 --set hostname=my-rancher-k3s \
 --set replicas=3

kubectl -n cattle-system rollout status deploy/rancher

kubectl describe p -n cattle-system rancher-5dfdd779dc-xll86
avatar
文章
阅读
粉丝