创建一个工作目录
后面操作进入该工作目录进行
[root@master ~]# mkdir etcd-ssl
安装cfssl工具
该工具用于自动生成证书
cd /root/etcd-ssl
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.0/cfssl_1.6.0_linux_amd64 -O cfssl
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.0/cfssljson_1.6.0_linux_amd64 -O cfssljson
wget https://github.com/cloudflare/cfssl/releases/download/v1.6.0/cfssl-certinfo_1.6.0_linux_amd64 -O cfssl-certinfo
chmod +x cfssl*
mv cfssl* /usr/local/bin/
chmod +x cfssl*
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
创建ca证书请求文件
cat > ca-csr.json <<EOF
{
"CN": "etcd-ca",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd-ca",
"OU": "etcd-ca"
}
],
"ca": {
"expiry": "87600h"
}
}
EOF
生成ca证书
cfssl gencert -initca ca-csr.json | cfssljson -bare ca
# 当前目录文件内容如下
[root@master etcd-ssl]# ls
ca.csr ca-csr.json ca-key.pem ca.pem
配置ca证书策略
cat > ca-config.json <<EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"etcd-ca": {
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
],
"expiry": "87600h"
}
}
}
}
EOF
配置etcd证书请求文件
cat > etcd-csr.json <<EOF
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"etcd0-0.etcd",
"etcd1-0.etcd",
"etcd2-0.etcd"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [{
"C": "CN",
"ST": "Beijing",
"L": "Beijing",
"O": "etcd",
"OU": "etcd"
}]
}
EOF
生成etcd证书
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=etcd-ca etcd-csr.json | cfssljson -bare etcd
创建configMap
将上面创建的etcd证书文件内容拷贝到configMap中
# etcd-configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: etcd-config
namespace: etcd
data:
ca.pem: |
-----BEGIN CERTIFICATE-----
MIIDwjCCAqqgAwIBAgIUJ/mL02xZcxV60lzVXdVn5tioc2wwDQYJKoZIhvcNAQEL
BQAwZzELMAkGA1UEBhMCQ04xEDAOBgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0Jl
aWppbmcxEDAOBgNVBAoTB2V0Y2QtY2ExEDAOBgNVBAsTB2V0Y2QtY2ExEDAOBgNV
BAMTB2V0Y2QtY2EwHhcNMjMwOTI4MjMzOTAwWhcNMzMwOTI1MjMzOTAwWjBnMQsw
CQYDVQQGEwJDTjEQMA4GA1UECBMHQmVpamluZzEQMA4GA1UEBxMHQmVpamluZzEQ
MA4GA1UEChMHZXRjZC1jYTEQMA4GA1UECxMHZXRjZC1jYTEQMA4GA1UEAxMHZXRj
ZC1jYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMre09xibaPj45k7
gJvT1vDlZlBW5VZpWvV1amenZdeyfp7EeDmVdOXYPMn8h13ujGxTsaechMTniBCv
YUsj1pbhvaInDtlnk7Yt6TJtB3hRn/x/lH1ObVSRdDwnr5yYYarlaAp8RAIEgxaV
qqncx99b8vImHFInB32olex4SexNMHN1YJ2nkP18BeecljKc2rUjf2hXDr73clmI
OIVVB++E8uRSJ5bQ284Fn3cfPtBnEtQUKEM2qX5U6mr055GjMCE5E9yi+0qdi4hd
8h2mSPl/+tBJajMV1wZUQztWzpKbbPR69HgsmIAiNFgH12A4nwx0erSlqZvmOIJI
gh3yRm0CAwEAAaNmMGQwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8C
AQIwHQYDVR0OBBYEFCAwd62YchSEBgOvFdMdhADd98j1MB8GA1UdIwQYMBaAFCAw
d62YchSEBgOvFdMdhADd98j1MA0GCSqGSIb3DQEBCwUAA4IBAQBBTun2406dZ//1
eqScPYjry63n0ryz2e7AUmhlQwDLCdnZ4Mu7wQculyFvvlU8Ianpc8Xu7nVf0rd+
yy74L4+TUwE1M0f0XblfI1piegWt9BwdkxSa8ssVoCUMMFoJmILzI6XlSY6NNkns
pYAkEFQkAtbtTv8cs1GdSBjyjusarZ1D5Jkr7csOxYuRnr6ARMQoe75qGASRdn4q
X3KD+Oof23EJ6QqzsSRBc0Z+aY+nYsYsO5J3j+YDCvtwbPoOj67W862xhlqA0Q92
+gmCooeOBim1qlP6AwORZPdbOmtLmZZYwWVZed2Y/COJ+6r+RY/Gi2Gyg+pXuHh/
AeO0acoO
-----END CERTIFICATE-----
etcd.pem: |
-----BEGIN CERTIFICATE-----
MIIEDzCCAvegAwIBAgIUJfKa9B64G++7KJdxhIzP7pIPD3owDQYJKoZIhvcNAQEL
BQAwZzELMAkGA1UEBhMCQ04xEDAOBgNVBAgTB0JlaWppbmcxEDAOBgNVBAcTB0Jl
aWppbmcxEDAOBgNVBAoTB2V0Y2QtY2ExEDAOBgNVBAsTB2V0Y2QtY2ExEDAOBgNV
BAMTB2V0Y2QtY2EwHhcNMjMwOTI4MjM0NjAwWhcNMzMwOTI1MjM0NjAwWjBeMQsw
CQYDVQQGEwJDTjEQMA4GA1UECBMHQmVpamluZzEQMA4GA1UEBxMHQmVpamluZzEN
MAsGA1UEChMEZXRjZDENMAsGA1UECxMEZXRjZDENMAsGA1UEAxMEZXRjZDCCASIw
DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALwweLJVJ50ws5Q0FO2ebrbDmjBq
2ZsAVSry4clbhR0aWZ9TsRIAhDIxLSpQj1r+x5RT1OkLla44Q4fv5CWMWuWyDa5E
+i9L3anu+JR1pIylx3A0/bqmfoe70K+SEVKEttqeStUW0S9PkJ4oBEQ5JvcfCdMA
fmLyViBsrP/h2amEgY8J12zSZMfore2wujM0KDBpAZDlKLlhlVNtKgGSZX+BadU4
QIDnRO8xHaOgKyV5znNFDHtptXPBLD9VKNTfsOSjggRDzQF9x0YQ9y5txLXP18bk
mWWl1QEUditAIdlJqdCucPe70qQy56SY10pAasIvajGQIGvUnRk9FTzVNPUCAwEA
AaOBuzCBuDAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsG
AQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFEWjVtV6Tkjdph61Er7fK3Uh
JsAxMB8GA1UdIwQYMBaAFCAwd62YchSEBgOvFdMdhADd98j1MDkGA1UdEQQyMDCC
DGV0Y2QwLTAuZXRjZIIMZXRjZDEtMC5ldGNkggxldGNkMi0wLmV0Y2SHBH8AAAEw
DQYJKoZIhvcNAQELBQADggEBALib9QKdT3hooszQmDk7FUrjg8wRV+NJ1+uSlYTR
eACKP90Oub/6Due141062yYXnfvo5zUNe9F+g8vr83eWdCdaCN8aPFuDq3fd59rK
CtTVqOdoBcypdQY1GLpchczTD3B1AZkWjY7LNWJnYs2UkizbcpgEkyiYK7mQ0dgk
sSbvt69eGcjE7OFM+ZjvRU+Jl9d55rdREsKsCD2JAKFod1hRfD2crCZtPPxAs/ok
jQVL/mpPYwoLHj/lOY2935VT04v6dFb0P5zEwFfmA2FrGTeSFfDP3YHuszJL5mIO
e5wum7N0tRKmetY3tKlbpitEspMSzoX9+ieSLADTfS/HM9A=
-----END CERTIFICATE-----
etcd-key.pem: |
-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEAvDB4slUnnTCzlDQU7Z5utsOaMGrZmwBVKvLhyVuFHRpZn1Ox
EgCEMjEtKlCPWv7HlFPU6QuVrjhDh+/kJYxa5bINrkT6L0vdqe74lHWkjKXHcDT9
uqZ+h7vQr5IRUoS22p5K1RbRL0+QnigERDkm9x8J0wB+YvJWIGys/+HZqYSBjwnX
bNJkx+it7bC6MzQoMGkBkOUouWGVU20qAZJlf4Fp1ThAgOdE7zEdo6ArJXnOc0UM
e2m1c8EsP1Uo1N+w5KOCBEPNAX3HRhD3Lm3Etc/XxuSZZaXVARR2K0Ah2Ump0K5w
97vSpDLnpJjXSkBqwi9qMZAga9SdGT0VPNU09QIDAQABAoIBAQCaTYQpdPkOQih8
lFe0nftZj/iVSocyOIucs5m+aHQ08K4bnZvKjvHV1cyarYNdsRoj3LjufLk2KJv2
khQXVYYSYHQnOBeLQEF4XKEl2XAagrOMpWaF7I5C7RSnYLUjW/4yMyhYoGTUzgBe
LaGR7uqsJZ8Ai58Vsm48F9+hz6WQ/9JT7Yse5vNx3fTuXQJSBTWbF26/f/cgV9Uz
+Y+aJSJALiqO8aCfWjftiW5PutXWDaZHENQXiGLoDkP2eAsOoxV/g6kAQqnLY87R
RZ373X5WeqvXrGvHI+dft7S5TqxE6eHABgb74BggC+OcPPJLM8LxUDpsHaGaOOIO
NC3YTJXhAoGBAMK1BcJVU+ZeajCW82FjxGITDsenrM6MvTvuzAE4jNqoqK1W8t1L
c5YrbRrEoqRd/xZXzdlvDp8tjvJyIA6dx5f7uYjEuoBdElQfv1sIQyL3Z5mhS5H7
0ZcHr0VxF8wdVMAjCqwIgB86/a6t9pV4VpitlZYdGIMxRqK+cZk2dhtfAoGBAPdu
Mm5xnqY8SUpmeINPYbZVskt8YK56KtInFYl/GXRAYKGtDY4eV6KvLbdderygqhw0
qX4W7f+E4qDVpjZZLOMLeevQYi+5blX4OE6VZk4bSC3mXkgSqvk9lR1I/HDt5H4n
j9dfLFxS5KLDLfvOm1s46CGeIOGt1RM1TQI8IuQrAoGAfcf1hXob7ik2DleluFd8
CSNom+AdCX3vzWTRk0RV8fecqAYQdJcGf49kNzOqIea7IFLunzVYS8phmwvQ28Bf
rTYnwEvec4gsi/3KX2NyA4ex/JQvPopf9mI3fvO1PN4B3vGENmhzZDl2oj6tmheY
UZ9pkAURzcaqa/39Ys3E9J8CgYEAjY5X/+O09joLXCtfyxABQQWqKZGQ5oEyOmTt
52n1sBKctiKBXMtzK7m5F+y9KWlpqRVf2qlXpBXsakORAgrmqUaJjCX95nvZ6glG
fX7vtsPY+ZmQWv+DHXDj/IsjEaCoVpxfqMHkaTVrZMrxZdx/+SKHe6Qx/6sp+gFG
e37YH10CgYEAjgwpSzVBJS0GRU732mk+vxwnNb47qOqLSHODZWocWqoDSaYqfPm6
zhJOEz1OJy+bXmR+cMKmxaf5HinXAkSSnQHZl2yErqOq3+MzxSmxcpHi3t7VWiXp
9lZFPZJOCg9YmfcwfT0yA21TLdID5SYHZgdntggHRTzi1TCBD+VD5nI=
-----END RSA PRIVATE KEY-----
应用configMap文件
[root@master etcd-ssl]# kubectl apply -f etcd-configmap.yaml
configmap/etcd-config created
配置etcd集群yaml文件
# etcd-cluster.yaml
apiVersion: v1
kind: Service
metadata:
annotations:
service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
labels:
name: etcd-operator
name: etcd
namespace: etcd
spec:
clusterIP: None
ports:
- name: client
port: 2379
protocol: TCP
targetPort: 2379
- name: peer
port: 2380
protocol: TCP
targetPort: 2380
publishNotReadyAddresses: true
selector:
name: etcd-operator
sessionAffinity: None
type: ClusterIP
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
name: etcd-operator
name: etcd0
namespace: etcd
spec:
replicas: 1
selector:
matchLabels:
name: etcd-operator
serviceName: etcd
template:
metadata:
labels:
name: etcd-operator
spec:
containers:
- command:
- /usr/local/bin/etcd
- --data-dir
- /data/etcd_data
- --auto-compaction-retention
- "1"
- --quota-backend-bytes
- "8589934592"
- --listen-client-urls
- https://0.0.0.0:2379
- --advertise-client-urls
- https://etcd0-0.etcd:2379
- --listen-peer-urls
- https://0.0.0.0:2380
- --initial-advertise-peer-urls
- https://etcd0-0.etcd:2380
- --initial-cluster-token
- etcd-cluster
- --initial-cluster
- etcd0=https://etcd0-0.etcd:2380,etcd1=https://etcd1-0.etcd:2380,etcd2=https://etcd2-0.etcd:2380
- --initial-cluster-state
- new
- --enable-pprof
- --election-timeout
- "5000"
- --heartbeat-interval
- "250"
- --name
- etcd0
- --logger
- zap
- --cert-file=/etc/etcd/ssl/etcd.pem
- --key-file=/etc/etcd/ssl/etcd-key.pem
- --trusted-ca-file=/etc/etcd/ssl/ca.pem
- --peer-cert-file=/etc/etcd/ssl/etcd.pem
- --peer-key-file=/etc/etcd/ssl/etcd-key.pem
- --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
- --peer-client-cert-auth
- --client-cert-auth
image: astraw99/etcd:v3.4.13
imagePullPolicy: Always
name: app
volumeMounts:
- name: etcd-volume
mountPath: /data/etcd0
- name: config
mountPath: /etc/etcd/ssl
volumes:
- hostPath:
path: /var/tmp
type: Directory
name: etcd-volume
- name: config
configMap:
name: etcd-config
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
name: etcd-operator
name: etcd1
namespace: etcd
spec:
replicas: 1
selector:
matchLabels:
name: etcd-operator
serviceName: etcd
template:
metadata:
labels:
name: etcd-operator
spec:
containers:
- command:
- /usr/local/bin/etcd
- --data-dir
- /data/etcd_data
- --auto-compaction-retention
- "1"
- --quota-backend-bytes
- "8589934592"
- --listen-client-urls
- https://0.0.0.0:2379
- --advertise-client-urls
- https://etcd1-0.etcd:2379
- --listen-peer-urls
- https://0.0.0.0:2380
- --initial-advertise-peer-urls
- https://etcd1-0.etcd:2380
- --initial-cluster-token
- etcd-cluster
- --initial-cluster
- etcd0=https://etcd0-0.etcd:2380,etcd1=https://etcd1-0.etcd:2380,etcd2=https://etcd2-0.etcd:2380
- --initial-cluster-state
- new
- --enable-pprof
- --election-timeout
- "5000"
- --heartbeat-interval
- "250"
- --name
- etcd1
- --logger
- zap
- --cert-file=/etc/etcd/ssl/etcd.pem
- --key-file=/etc/etcd/ssl/etcd-key.pem
- --trusted-ca-file=/etc/etcd/ssl/ca.pem
- --peer-cert-file=/etc/etcd/ssl/etcd.pem
- --peer-key-file=/etc/etcd/ssl/etcd-key.pem
- --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
- --peer-client-cert-auth
- --client-cert-auth
image: astraw99/etcd:v3.4.13
imagePullPolicy: Always
name: app
volumeMounts:
- name: etcd-volume
mountPath: /data/etcd1
- name: config
mountPath: /etc/etcd/ssl
volumes:
- hostPath:
path: /var/tmp
type: Directory
name: etcd-volume
- name: config
configMap:
name: etcd-config
---
apiVersion: apps/v1
kind: StatefulSet
metadata:
labels:
name: etcd-operator
name: etcd2
namespace: etcd
spec:
replicas: 1
selector:
matchLabels:
name: etcd-operator
serviceName: etcd
template:
metadata:
labels:
name: etcd-operator
spec:
containers:
- command:
- /usr/local/bin/etcd
- --data-dir
- /data/etcd_data
- --auto-compaction-retention
- "1"
- --quota-backend-bytes
- "8589934592"
- --listen-client-urls
- https://0.0.0.0:2379
- --advertise-client-urls
- https://etcd2-0.etcd:2379
- --listen-peer-urls
- https://0.0.0.0:2380
- --initial-advertise-peer-urls
- https://etcd2-0.etcd:2380
- --initial-cluster-token
- etcd-cluster
- --initial-cluster
- etcd0=https://etcd0-0.etcd:2380,etcd1=https://etcd1-0.etcd:2380,etcd2=https://etcd2-0.etcd:2380
- --initial-cluster-state
- new
- --enable-pprof
- --election-timeout
- "5000"
- --heartbeat-interval
- "250"
- --name
- etcd2
- --logger
- zap
- --cert-file=/etc/etcd/ssl/etcd.pem
- --key-file=/etc/etcd/ssl/etcd-key.pem
- --trusted-ca-file=/etc/etcd/ssl/ca.pem
- --peer-cert-file=/etc/etcd/ssl/etcd.pem
- --peer-key-file=/etc/etcd/ssl/etcd-key.pem
- --peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
- --peer-client-cert-auth
- --client-cert-auth
image: astraw99/etcd:v3.4.13
imagePullPolicy: Always
name: app
volumeMounts:
- name: etcd-volume
mountPath: /data/etcd2
- name: config
mountPath: /etc/etcd/ssl
volumes:
- hostPath:
path: /var/tmp
type: Directory
name: etcd-volume
- name: config
configMap:
name: etcd-config
应用yaml文件
[root@master etcd-ssl]# kubectl apply -f etcd-cluster.yaml
进入Pod
kubectl exec -it -n etcd etcd0-0 -- sh
查看集群的健康状态
# /usr/local/bin/etcdctl --write-out=table --cacert=/etc/etcd/ssl/ca.pem --cert=/etc/etcd/ssl/etcd.pem --key=/etc/etcd/ssl/etcd-key.pem --endpoints=https://etcd0-0.etcd:2379,https://etcd1-0.etcd:2379,https://etcd2-0.etcd:2379 endpoint health
+---------------------------+--------+--------------+-------+
| ENDPOINT | HEALTH | TOOK | ERROR |
+---------------------------+--------+--------------+-------+
| https://etcd0-0.etcd:2379 | true | 104.419034ms | |
| https://etcd1-0.etcd:2379 | true | 305.007436ms | |
| https://etcd2-0.etcd:2379 | true | 655.207207ms | |
+---------------------------+--------+--------------+-------+
