Prerequisites
Deployment By Manifest
Preparation
| IP Address | Hostname | OS Release | Remark |
|---|---|---|---|
| 10.2.102.241 | k8s-master | Debian 11.x | 主节点，负责管理集群 |
| 10.2.102.242 | k8s-node1 | Debian 11.x | 工作节点1，运行应用的 Pod |
| 10.2.102.243 | k8s-node2 | Debian 11.x | 工作节点2，运行应用的 Pod |
| 10.2.102.250 | nfs-server | Debian 11.x | NFS 服务器，为集群提供持久化存储服务 |
Deploy NFS
部署 NFS 服务端
# 安装 NFS 服务端软件
root@localhost:~# apt-get install -y nfs-kernel-server
# 创建三个共享文件夹
root@localhost:~# mkdir -p /mnt/k8s/mongo/pv{1..3}
# 更改目录权限
root@localhost:~# chmod 777 /mnt/k8s/mongo/pv{1..3}
# 编辑 /etc/exports 文件
root@localhost:~# cat >> /etc/exports <<EOF
/mnt/k8s/mongo/pv1 10.2.102.0/24(rw,sync,no_subtree_check,no_root_squash)
/mnt/k8s/mongo/pv2 10.2.102.0/24(rw,sync,no_subtree_check,no_root_squash)
/mnt/k8s/mongo/pv3 10.2.102.0/24(rw,sync,no_subtree_check,no_root_squash)
EOF
# 应用新的设置
root@localhost:~# exportfs -a
# 重启 NFS 服务
root@localhost:~# systemctl restart nfs-kernel-server
# 检查可用的共享资源
root@localhost:~# exportfs -v
/mnt/k8s/mongo/pv1
10.2.102.0/24(rw,wdelay,no_root_squash,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
/mnt/k8s/mongo/pv2
10.2.102.0/24(rw,wdelay,no_root_squash,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
/mnt/k8s/mongo/pv3
10.2.102.0/24(rw,wdelay,no_root_squash,no_subtree_check,sec=sys,rw,secure,no_root_squash,no_all_squash)
参考挂载选项
| 选项 | 描述 |
|---|---|
| ro | 只读模式。客户端只可读取共享文件，不能进行写操作。 |
| rw | 读写模式。客户端可以读取和写入共享文件。 |
| sync | 同步模式。NFS 服务器将等待所有的写操作都写入磁盘才返回结果，保证了数据的一致性。 |
| async | 异步模式。NFS 服务器在接收到写请求后立即返回结果，不等待数据写入磁盘，可能会导致数据不一致。 |
| no_root_squash | 取消 Root Squash 。NFS 服务器将客户端的 root 用户视为本地的 root 用户。 |
| root_squash | 启用 Root Squash。NFS 服务器将客户端的 root 用户视为名为 nobody 的普通用户。 |
| subtree_check | 子目录检查。NFS 服务器将检查客户端对某个子目录的请求是否也在共享目录下。 |
| no_subtree_check | 取消子目录检查。NFS 服务器不将进行子目录检查，有助于提高性能。 |
验证 NFS 客户端
在各节点安装 NFS 客户端，并进行本地测试：
# 安装 NFS 客户端软件
root@k8s-master:~# apt-get install -y nfs-common
# 查看 NFS 服务端可用于共享的目录
root@k8s-master:~# showmount -e 10.2.102.250
Export list for 10.2.102.250:
/mnt/k8s/mongo/pv3 10.2.102.0/24
/mnt/k8s/mongo/pv2 10.2.102.0/24
/mnt/k8s/mongo/pv1 10.2.102.0/24
# 尝试挂载一个共享目录到本地
root@k8s-master:~# mount -t nfs 10.2.102.250:/mnt/k8s/mongo/pv1 /mnt
Created symlink /run/systemd/system/remote-fs.target.wants/rpc-statd.service → /lib/systemd/system/rpc-statd.service.
# 检查已挂载的本地文件系统
root@k8s-master:~# df -h /mnt
Filesystem Size Used Avail Use% Mounted on
10.2.102.250:/mnt/k8s/mongo/pv1 64G 4.8G 59G 8% /mnt
# 测试共享目录的读写权限
root@k8s-master:~# cd /mnt && touch test && rm test && cd ~
# 卸载文件系统
root@k8s-master:~# umount /mnt
Deploy K8S
K8S 集群搭建过程略，检查集群节点状态及信息：
# 查看集群节点信息
root@k8s-master:~# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
k8s-master Ready control-plane 44d v1.28.2 10.2.102.241 <none> Debian GNU/Linux 12 (bookworm) 6.1.0-7-amd64 containerd://1.7.8
k8s-node1 Ready <none> 44d v1.28.2 10.2.102.242 <none> Debian GNU/Linux 12 (bookworm) 6.1.0-7-amd64 containerd://1.7.8
k8s-node2 Ready <none> 44d v1.28.2 10.2.102.243 <none> Debian GNU/Linux 12 (bookworm) 6.1.0-7-amd64 containerd://1.7.8
# 查看所有的命名空间
root@k8s-master:~# kubectl get ns
NAME STATUS AGE
calico-apiserver Active 44d
calico-system Active 44d
default Active 44d
kube-node-lease Active 44d
kube-public Active 44d
kube-system Active 44d
test Active 39d
tigera-operator Active 44d
# 在每个节点上添加主机名和IP地址映射
root@k8s-master:~# cat >> /etc/hosts <<EOF
10.2.102.241 k8s-master
10.2.102.242 k8s-node1
10.2.102.243 k8s-node2
10.2.102.250 nfs-server
EOF
Deploy ReplicaSet
- 创建
mongo-replica命名空间
# 创建命名空间
root@k8s-master:~# kubectl create ns mongo-replica
namespace/mongo-replica created
# 查看相关信息
root@k8s-master:~# kubectl describe ns mongo-replica
Name: mongo-replica
Labels: kubernetes.io/metadata.name=mongo-replica
Annotations: <none>
Status: Active
No resource quota.
No LimitRange resource.
- 编写
pv-mongo-repl.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv-mongo-repl-1 # 资源名称
labels:
app: mongo # 为资源添加标签，方便管理
spec:
capacity:
storage: 10Gi # 存储容量（注意：虽然 K8S 允许你声明超过实际存储量的存储容量，但为了避免可能的问题，建议你总是确保声明的存储容量不超过 NFS 服务器实际的可用空间）
accessModes:
- ReadWriteOnce # 存储访问模式，ReadWriteOnce 表示支持单个节点进行读写操作
storageClassName: mongo-storage # 存储类，PVC 所需 matched storageClass 才能绑定该 PV
nfs:
server: nfs-server # NFS 服务器的主机名或 IP 地址，注意确保该地址可以被所有 Kubernetes 集群节点解析
path: /mnt/k8s/mongo/pv1 # NFS 服务器上的共享目录路径
persistentVolumeReclaimPolicy: Retain # 取回策略，Retain 表示在 PVC 被删除后，对应的 PV 仍然保留，需要手动清理
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv-mongo-repl-2
labels:
app: mongo
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
storageClassName: mongo-storage
nfs:
server: nfs-server
path: /mnt/k8s/mongo/pv2
persistentVolumeReclaimPolicy: Retain
---
apiVersion: v1
kind: PersistentVolume
metadata:
name: pv-mongo-repl-3
labels:
app: mongo
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteOnce
storageClassName: mongo-storage
nfs:
server: nfs-server
path: /mnt/k8s/mongo/pv3
persistentVolumeReclaimPolicy: Retain
- 创建
pv持久卷
# 创建 PersistentVolume 资源
root@k8s-master:~/test/mongo-repl# kubectl apply -f pv-mongo-repl.yaml
persistentvolume/pv-mongo-repl-1 created
persistentvolume/pv-mongo-repl-2 created
persistentvolume/pv-mongo-repl-3 created
# 查看 PersistentVolume 的状态
root@k8s-master:~/test/mongo-repl# kubectl get pv
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
pv-mongo-repl-1 10Gi RWO Retain Available mongo-storage 102s
pv-mongo-repl-2 10Gi RWO Retain Available mongo-storage 102s
pv-mongo-repl-3 10Gi RWO Retain Available mongo-storage 102s
# Tips: 当有 PVC 去申请使用这些存储后，你可以在 NFS 服务器上看到这些挂载点了
- 编写
sts-mongo-repl.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: mongo # 定义 StatefulSet 的名称
namespace: mongo-replica # 定义使用的 Kubernetes 命名空间
spec:
serviceName: mongo # 定义 Headless Service 的名称，用于服务发现
replicas: 3 # 定义副本数量
selector:
matchLabels:
app: mongo # 匹配 Pod 的标签，用于确定 StatefulSet 管理哪些 Pod
template:
metadata:
labels:
app: mongo # 在 Pod 模板中定义的标签，用于匹配 StatefulSet 的 selector
spec:
terminationGracePeriodSeconds: 10 # 定义 Pod 终止的优雅期限
containers:
- name: mongo
image: mongo:4.0 # 定义 MongoDB 容器使用的镜像版本
imagePullPolicy: IfNotPresent
command:
- mongod
- "--bind_ip_all" # 监听所有网络接口
- "--replSet"
- rs0 # 启用 MongoDB 副本集，并指定副本集的名称
ports:
- containerPort: 27017 # 定义容器暴露的端口
volumeMounts:
- name: mongo-volume
mountPath: /data/db # 定义 MongoDB 数据存储的挂载路径
- name: mongo-sidecar
image: cvallance/mongo-k8s-sidecar # 这个 sidecar 容器可以帮助在 MongoDB Pod 成功启动后，自动配置 MongoDB 的副本集
env:
- name: MONGO_SIDECAR_POD_LABELS
value: "app=mongo"
volumeClaimTemplates:
- metadata:
name: mongo-volume # 定义持久卷声明（PersistentVolumeClaim）的名称
labels:
app: mongo
spec:
accessModes: [ "ReadWriteOnce" ] # 定义持久卷声明的访问模式
storageClassName: mongo-storage # 定义存储类的名称，用于动态分配存储
resources:
requests:
storage: 10Gi # 定义请求的存储容量
---
apiVersion: v1
kind: Service
metadata:
name: mongo-headless-svc # 定义 Headless Service 的名称
namespace: mongo-replica # 定义使用的 Kubernetes 命名空间
labels:
app: mongo
spec:
ports:
- port: 27017
targetPort: 27017
clusterIP: None # 定义 Headless Service，clusterIP 设置为 None，表示没有 Cluster IP
selector:
app: mongo # 匹配 StatefulSet 管理的 Pod 的标签
- 编写
svc-lb-mongo-repl.yaml
apiVersion: v1
kind: Service
metadata:
name: mongo-lb
namespace: mongo-replica
annotations:
metallb.universe.tf/address-pool: ip-pool
spec:
ports:
- port: 27017
targetPort: 27017
selector:
app: mongo
type: LoadBalancer
- 运行
MongoDB服务，检查相关资源
# 应用配置
root@k8s-master:~/test/mongo-repl# kubectl apply -f sts-mongo-repl.yaml
statefulset.apps/mongodb created
service/mongodb-headless-svc created
# 查看全部资源（将会每一秒刷新一次，实时显示最新状态）
root@k8s-master:~/test/mongo-repl# watch -n 1 -x kubectl get all -n mongo-replica
Every 1.0s: kubectl get all -n mongo-replica k8s-master: Thu Dec 28 14:59:04 2023
NAME READY STATUS RESTARTS AGE
pod/mongo-0 2/2 Running 0 9s
pod/mongo-1 2/2 Running 0 6s
pod/mongo-2 0/2 ContainerCreating 0 3s
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/mongo-headless-svc ClusterIP None <none> 27017/TCP 9s
NAME READY AGE
statefulset.apps/mongo 2/3 9s
# 查看 statefulset 资源
root@k8s-master:~/test/mongo-repl# kubectl get sts -n mongo-replica
NAME READY AGE
mongo 3/3 81s
# 查看 pod 资源
root@k8s-master:~/test/mongo-repl# kubectl get pod -n mongo-replica -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-0 2/2 Running 0 98s 10.244.169.144 k8s-node2 <none> <none>
mongo-1 2/2 Running 0 94s 10.244.36.93 k8s-node1 <none> <none>
mongo-2 2/2 Running 0 84s 10.244.169.145 k8s-node2 <none> <none>
root@k8s-master:~/test/mongo-repl# kubectl get pvc -n mongo-replica
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
mongo-volume-mongo-0 Bound pv-mongo-repl-1 10Gi RWO mongo-storage 114s
mongo-volume-mongo-1 Bound pv-mongo-repl-2 10Gi RWO mongo-storage 110s
mongo-volume-mongo-2 Bound pv-mongo-repl-3 10Gi RWO mongo-storage 100s
# 查看 headless service 资源
root@k8s-master:~/test/mongo-repl# kubectl get svc -n mongo-replica
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mongo-headless-svc ClusterIP None <none> 27017/TCP 2m12s
root@k8s-master:~/test/mongo-repl# kubectl exec -n mongo-replica -c mongo -it mongo-0 -- /bin/bash
root@mongo-0:/# mongo mongo-0.mongo.mongo-replica.svc.cluster.local:27017
MongoDB shell version v4.0.28
connecting to: mongodb://mongo-0.mongo.mongo-replica.svc.cluster.local:27017/test?gssapiServiceName=mongodb
Implicit session: session { "id" : UUID("a96a02e9-6a59-4a4d-9070-5af8a2087f46") }
MongoDB server version: 4.0.28
Server has startup warnings:
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten]
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten] ** WARNING: Access control is not enabled for the database.
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten] ** Read and write access to data and configuration is unrestricted.
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten] ** WARNING: You are running this process as the root user, which is not recommended.
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten]
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten]
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten] ** WARNING: /sys/kernel/mm/transparent_hugepage/enabled is 'always'.
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten] ** We suggest setting it to 'never'
2023-12-28T06:32:26.987+0000 I CONTROL [initandlisten]
---
Enable MongoDB's free cloud-based monitoring service, which will then receive and display
metrics about your deployment (disk utilization, CPU, operation statistics, etc).
The monitoring data will be available on a MongoDB website with a unique URL accessible to you
and anyone you share the URL with. MongoDB may use this information to make product
improvements and to suggest MongoDB products and deployment options to you.
To enable free monitoring, run the following command: db.enableFreeMonitoring()
To permanently disable this reminder, run the following command: db.disableFreeMonitoring()
---
>
- 手动创建副本集
// 方式1
use admin
var cfg = { _id: "rs0", members:[{ _id: 0, host: 'mongo-0.mongo.mongo-replica.svc.cluster.local:27017', priority: 3 }, { _id: 1, host: 'mongo-1.mongo.mongo-replica.svc.cluster.local:27017', priority: 2 },{ _id: 2, host: 'mongo-2.mongo.mongo-replica.svc.cluster.local:27017', priority: 1 }] }
rs.initiate(cfg)
rs.status()
// 方式2
rs.initiate()
var cfg = rs.conf()
cfg.members[0].host="mongo-0.mongo.mongo-replica.svc.cluster.local:27017"
rs.reconfig(cfg)
rs.status()
rs.add("mongo-1.mongo.mongo-replica.svc.cluster.local:27017")
rs.add("mongo-2.mongo.mongo-replica.svc.cluster.local:27017")
rs.status()
Using MongoDB Operator
MongoDB Community Operator 可创建 ReplicaSet 复制集，支持节点弹性扩缩，当前不支持备份功能。
Basic Introduction
Architecture
Documentation
Community Operator：
Content：
- MongoDB Community Kubernetes Operator Architecture
- Install and Upgrade the Community Kubernetes Operator
- Deploy and Configure MongoDBCommunity Resources
- Create Database Users
- Secure MongoDBCommunity Resources
More Config：
Configuration Scheme
- 集群部署模式选型：
| 一主一从一选举 ✅ | 一主两从 ❌ | |
|---|---|---|
| 冗余性 | 较低，如果从节点故障则无法提供服务 | 高，即使一个节点故障，也能保持服务连续性 |
| 读能力 | 一般，由两个数据节点提供 | 较高，由三个数据节点提供 |
| 数据一致性 | 很好，数据只需要在两个节点间同步 | 较差，需要在三个节点间同步，可能存在一致性问题 |
| 维护成本 | 较低，仲裁器不持有数据，资源需求低 | 较高，需要维护三个数据节点 |
| 资源开销 | 以 100% 为资源基准 | 上浮到 150% - 200% 资源消耗 |
| 故障恢复 | 一般，如果从节点故障，集群将无法提供服务 | 较好，即使一个从节点故障，剩余节点仍可提供服务 |
- 标准用户角色分配：
mindmap
Permissions
root
readWriteAnyDatabase
userAdminAnyDatabase
clusterAdmin
clusterManager
clusterMonitor
hostManager
dbAdminAnyDatabase
restore
backup
admin
readWriteAnyDatabase
dbAdminAnyDatabase
userAdminAnyDatabase
user
readWrite
read
Deployment Steps
Install using Helm charts
通过 Helm charts 离线安装 mongodb-community-operator：
# 下载 MongoDB Community Operator 的 Helm chart 到本地
root@k8s-client:~# helm pull mongodb/community-operator --untar
root@k8s-client:~# ll community-operator/
total 24
-rw-r--r-- 1 root root 239 Dec 29 15:24 Chart.lock
drwxr-xr-x 3 root root 4096 Dec 29 15:24 charts
-rw-r--r-- 1 root root 564 Dec 29 15:24 Chart.yaml
-rw-r--r-- 1 root root 3181 Dec 29 15:24 README.md
drwxr-xr-x 2 root root 4096 Dec 29 15:24 templates
-rw-r--r-- 1 root root 3260 Dec 29 15:24 values.yaml
# 按需修改 Helm chart 的默认配置
root@k8s-client:~# vi community-operator/values.yaml
# 将整个 community-operator 文件夹打包，并安装到 Kubernetes 集群
root@k8s-client:~# helm install mongo-community-operator ./community-operator --namespace mongo-repl --create-namespace
通过 Helm charts 在线安装 mongodb-community-operator：
# 将一个新的 Helm 仓库添加到 Helm 客户端
root@k8s-client:~# helm repo add mongodb https://mongodb.github.io/helm-charts
# 更新获取到仓库中最新的 charts
root@k8s-client:~# helm repo update
# 安装 mongo-community-operator 的 Helm chart 到 mongo-repl 命名空间下
root@k8s-client:~# helm install mongo-community-operator mongodb/community-operator --namespace mongo-repl --create-namespace
NAME: mongo-community-operator
LAST DEPLOYED: Fri Dec 29 13:13:42 2023
NAMESPACE: mongo-repl
STATUS: deployed
REVISION: 1
TEST SUITE: None
# 列出在命名空间 mongo-repl 中的所有 Helm releases
root@k8s-client:~# helm list -n mongo-repl
NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
mongo-community-operator mongo-repl 1 2023-12-29 13:13:42.012360971 +0800 CST deployed community-operator-0.9.0 0.9.0
# 检查刚安装的 mongo-community-operator Helm chart 已创建了对应的 operator Deployment
root@k8s-client:~# kubectl get deployments -n mongo-repl
NAME READY UP-TO-DATE AVAILABLE AGE
mongodb-kubernetes-operator 1/1 1 1 2m57s
# 检查刚安装的 mongo-community-operator Helm chart 已创建了对应的 operator Pod
root@k8s-client:~# kubectl get pods -n mongo-repl
NAME READY STATUS RESTARTS AGE
mongodb-kubernetes-operator-584c9fcb79-qg8xk 1/1 Running 0 3m8s
Create ReplicaSet
- 编写
mongo-community-operator.yaml
---
apiVersion: mongodbcommunity.mongodb.com/v1
kind: MongoDBCommunity
metadata:
name: mongodb
namespace: mongo-repl
spec:
members: 2
arbiters: 1
type: ReplicaSet
version: "6.0.5"
security:
authentication:
modes: ["SCRAM"]
statefulSet:
spec:
template:
spec:
# resources can be specified by applying an override
# per container name.
containers:
- name: mongod
resources:
limits:
cpu: 1000m
memory: 1Gi
requests:
cpu: 500m
memory: 512Mi
- name: mongodb-agent
resources:
limits:
cpu: "0.2"
memory: 256Mi
requests:
cpu: "0.2"
memory: 200Mi
volumeClaimTemplates:
- metadata:
name: data-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: longhorn
volumeMode: Filesystem
- metadata:
name: logs-volume
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 10Gi
storageClassName: longhorn
volumeMode: Filesystem
users:
- name: root
db: admin
passwordSecretRef:
name: root-password
roles:
- name: root
db: admin
scramCredentialsSecretName: root-scram
additionalConnectionStringConfig:
readPreference: secondary
- name: admin
db: admin
passwordSecretRef:
name: admin-password
roles:
- name: userAdminAnyDatabase
db: admin
- name: readWriteAnyDatabase
db: admin
- name: dbAdminAnyDatabase
db: admin
scramCredentialsSecretName: admin-scram
additionalConnectionStringConfig:
readPreference: secondary
- name: appuser
db: mydb # assuming 'mydb' is your application database
passwordSecretRef:
name: appuser-password
roles:
- name: readWrite
db: mydb
scramCredentialsSecretName: appuser-scram
additionalMongodConfig:
# the additional config passed to the mongod process can be specified
# either in nested or dot notation
net:
bindIp: 0.0.0.0
port: 27017
storage:
wiredTiger:
engineConfig:
cacheSizeGB: 2
journalCompressor: zlib
security:
authorization: enabled
setParameter:
failIndexKeyTooLong: false
additionalConnectionStringConfig:
readPreference: primary
# the user credentials will be generated from this secret
# once the credentials are generated, this secret is no longer required
---
apiVersion: v1
kind: Secret
metadata:
name: root-password # corresponds to spec.users.passwordSecretRef.name in the MongoDB CRD
namespace: mongo-repl
type: Opaque
stringData:
password: root@12345 # corresponds to spec.users.passwordSecretRef.key in the MongoDB CRD
---
apiVersion: v1
kind: Secret
metadata:
name: admin-password
namespace: mongo-repl
type: Opaque
stringData:
password: admin@12345
---
apiVersion: v1
kind: Secret
metadata:
name: appuser-password
namespace: mongo-repl
type: Opaque
stringData:
password: app@12345
- 应用
mongo-community-operator.yaml
# 应用加载 mongo-community-operator.yaml
root@k8s-client:~# kubectl apply -f mongo-community-operator.yaml
mongodbcommunity.mongodbcommunity.mongodb.com/mongodb created
secret/root-password created
secret/admin-password created
secret/appuser-password created
# watch 是基于 linux 命令周期性地执行并显示后面命令的结果
root@k8s-client:~# watch -n 1 -x kubectl get pods -n mongo-repl
NAME READY STATUS RESTARTS AGE
mongodb-0 0/2 Init:0/2 0 11s
mongodb-arb-0 0/2 Init:0/2 0 11s
mongodb-kubernetes-operator-584c9fcb79-qg8xk 1/1 Running 0 7m3s
# k8s 中 --watch 选项是基于事件驱动，只有在有事件发生时才会有输出
root@k8s-client:~# kubectl get mongodbcommunity -n mongo-repl --watch
NAME PHASE VERSION
mongodb Pending
mongodb Running 6.0.5
# 可以看到创建了 PSA 模型的 mongodb replicaSet 集群
root@k8s-client:~# kubectl get pods -n mongo-repl
NAME READY STATUS RESTARTS AGE
mongodb-0 2/2 Running 0 2m8s
mongodb-1 2/2 Running 0 76s
mongodb-arb-0 2/2 Running 0 2m8s
mongodb-kubernetes-operator-584c9fcb79-qg8xk 1/1 Running 0 9m
# 社区 Kubernetes Operator 创建包含用户连接字符串和凭据secret
# 遵循以下命名约定：<metadata.name>-<auth-db>-<username>
root@k8s-client:~# kubectl get secret -n mongo-repl
NAME TYPE DATA AGE
admin-password Opaque 1 3m32s
admin-scram-scram-credentials Opaque 6 3m32s
appuser-password Opaque 1 3m32s
appuser-scram-scram-credentials Opaque 6 3m32s
mongodb-admin-admin Opaque 4 104s
mongodb-admin-root Opaque 4 104s
mongodb-agent-password Opaque 1 3m32s
mongodb-config Opaque 1 3m32s
mongodb-keyfile Opaque 1 3m32s
mongodb-mydb-appuser Opaque 4 104s
root-password Opaque 1 3m32s
root-scram-scram-credentials Opaque 6 3m32s
sh.helm.release.v1.mongo-community-operator.v1 helm.sh/release.v1 1 9m
# 需在主机上提前下载 jq 命令：apt-get update && apt-get install -y jq
root@k8s-client:~# kubectl get secret mongodb-admin-root -n mongo-repl -o json | jq -r '.data | with_entries(.value |= @base64d)'
{
"connectionString.standard": "mongodb://root:root%4012345@mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017,mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017/admin?replicaSet=mongodb&ssl=false&readPreference=secondary",
"connectionString.standardSrv": "mongodb+srv://root:root%4012345@mongodb-svc.mongo-repl.svc.cluster.local/admin?replicaSet=mongodb&ssl=false&readPreference=secondary",
"password": "root@12345",
"username": "root"
}
Expose Service
- 编写
mongo-external-service.yaml
---
kind: Service
apiVersion: v1
metadata:
name: mongo-svc0
namespace: mongo-repl
annotations:
note: "NodePort is used for development and testing environments"
spec:
type: NodePort
selector:
app: mongodb-svc
statefulset.kubernetes.io/pod-name: mongodb-0
ports:
- protocol: TCP
nodePort: 31181
port: 31181
targetPort: 27017
---
kind: Service
apiVersion: v1
metadata:
name: mongo-svc1
namespace: mongo-repl
annotations:
note: "NodePort is used for development and testing environments"
spec:
type: NodePort
selector:
app: mongodb-svc
statefulset.kubernetes.io/pod-name: mongodb-1
ports:
- nodePort: 31182
port: 31182
targetPort: 27017
- 应用
mongo-external-service.yaml
# 服务加载 mongo-external-service.yaml
root@k8s-client:~# kubectl apply -f mongo-external-service.yaml
service/mongo-svc0 created
service/mongo-svc1 created
# 查看当前服务状态
root@k8s-client:~# kubectl get svc -n mongo-repl
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
mongo-svc0 NodePort 10.43.208.111 <none> 31181:31181/TCP 20s
mongo-svc1 NodePort 10.43.20.130 <none> 31182:31182/TCP 20s
mongodb-svc ClusterIP None <none> 27017/TCP 4h25m
# 查看Node的IP信息
root@k8s-client:~# kubectl get nodes -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
kube-dev01 Ready controlplane,etcd,worker 127d v1.26.7 10.93.40.51 <none> CentOS Linux 7 (Core) 3.10.0-1160.92.1.el7.x86_64 docker://24.0.2
kube-dev02 Ready controlplane,etcd,worker 127d v1.26.7 10.93.40.52 <none> CentOS Linux 7 (Core) 3.10.0-1160.92.1.el7.x86_64 docker://24.0.2
kube-dev03 Ready controlplane,etcd,worker 127d v1.26.7 10.93.40.53 <none> CentOS Linux 7 (Core) 3.10.0-1160.92.1.el7.x86_64 docker://24.0.2
- 使用
K8S Node + Service Port进行登录测试
# 其它服务器可联通 K8S Node 暴露出来的端口
[root@localhost ~]# telnet 10.93.40.51 31181
Trying 10.93.40.51...
Connected to 10.93.40.51.
Escape character is '^]'.
^C
[root@localhost ~]# telnet 10.93.40.51 31182
Trying 10.93.40.51...
Connected to 10.93.40.51.
Escape character is '^]'.
^C
[root@localhost ~]# mongosh --host 10.93.40.51 --port 31181 -u root -p 'root@12345' --authenticationDatabase admin
Current Mongosh Log ID: 658ecbdf3be8f6c4953bcf36
Connecting to: mongodb://<credentials>@10.93.40.51:31181/?directConnection=true&authSource=admin&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
To help improve our products, anonymous usage data is collected and sent to MongoDB periodically (https://www.mongodb.com/legal/privacy-policy).
You can opt-out by running the disableTelemetry() command.
------
The server generated these startup warnings when booting
2023-12-29T08:54:52.230+00:00: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem
2023-12-29T08:54:52.230+00:00: The configured WiredTiger cache size is more than 80% of available RAM. See http://dochub.mongodb.org/core/faq-memory-diagnostics-wt
2023-12-29T08:54:52.927+00:00: You are running on a NUMA machine. We suggest launching mongod like this to avoid performance problems: numactl --interleave=all mongod [other options]
2023-12-29T08:54:52.927+00:00: /sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never'
2023-12-29T08:54:52.927+00:00: /sys/kernel/mm/transparent_hugepage/defrag is 'always'. We suggest setting it to 'never'
2023-12-29T08:54:52.927+00:00: vm.max_map_count is too low
------
Deprecation warnings:
- Using mongosh on the current operating system is deprecated, and support may be removed in a future release.
See https://www.mongodb.com/docs/mongodb-shell/install/#supported-operating-systems for documentation on supported platforms.
mongodb [direct: primary] test>
在生产环境中为了简化 MongoDB 连接的需求，希望让客户端（用户）无需了解 MongoDB 的集群结构，只通过一个固定 IP+Port 就能连接上对应的服务。Kubernetes 提供了几种方式来满足这种需求：
- 当你创建
LoadBalancer类型的Service时，云提供商（例如AWS，GCP，Azure等）会提供一个公共IP或者DNS名称，并设置一个负载均衡器，流量通过这个负载均衡器路由到你的服务。 Ingress是一组路由规则，它定义了外部请求如何访问集群内部的服务。在你的情况下，你可以设置一个Ingress Controller，将所有的请求指向你的MongoDB服务。- 如上述不支持，可以考虑使用
HAProxy或者Nginx作为反向代理和负载均衡器。这样，所有的请求都会发送到这个代理，然后由代理转发到正确的服务。 - 还可以考虑使用
MongoDB的服务发现功能，通过连接字符串进行连接，当应用程序使用MongoDB的驱动连接到集群时，这个连接字符串只需包含一个集群中的任一节点地址，驱动程序会自动发现集群中其它的节点。Connection String类似如下：
mongodb://<user>:<password>@<host1>:<port>,<host2>:<port>/<dbname>?replicaSet=<replica-name>
Cluster Operation
Client Connection
- 编写
mongo-client-deployment.yaml
---
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: mongo-client
name: mongo-client
namespace: mongo-repl
spec:
replicas: 1
selector:
matchLabels:
app: mongo-client
template:
metadata:
labels:
app: mongo-client
spec:
containers:
- image: mongo
name: mongo-client
env:
- name: "CONNECTION_STRING"
valueFrom:
secretKeyRef:
# name: <metadata.name>-<auth-db>-<username>
name: mongodb-admin-root
key: connectionString.standardSrv
- 应用
mongo-client-deployment.yaml
# 应用加载 mongo-client-deployment.yaml
root@k8s-client:~# kubectl apply -f mongo-client-deployment.yaml
deployment.apps/mongo-client created
# 查看当前的 pod
root@k8s-client:~# kubectl get pods -n mongo-repl
NAME READY STATUS RESTARTS AGE
mongo-client-5879dfbfcd-p52zc 1/1 Running 0 34s
mongodb-0 2/2 Running 0 2m18s
mongodb-1 2/2 Running 0 86s
mongodb-arb-0 2/2 Running 0 2m18s
mongodb-kubernetes-operator-584c9fcb79-qg8xk 1/1 Running 0 9m10s
# 进入 pod 内部 shell
root@k8s-client:~# kubectl exec -it mongo-client-5b74bfbbc8-qmjzh -n mongo-repl -- /bin/bash
# 列出客户端提供的 mongo 相关工具命令
root@mongo-client-5b74bfbbc8-qmjzh:/# ls `echo "$PATH" | sed 's/:/ /g'` | grep -i mongo
mongod
mongodump
mongoexport
mongofiles
mongoimport
mongorestore
mongos
mongosh
mongostat
mongotop
mongod
mongodump
mongoexport
mongofiles
mongoimport
mongorestore
mongos
mongosh
mongostat
mongotop
# 使用 MongoDB SRV connection string 进行登录
root@mongo-client-5b74bfbbc8-qmjzh:/# mongosh "mongodb+srv://root:root%4012345@mongodb-svc.mongo-repl.svc.cluster.local/admin?replicaSet=mongodb&ssl=false&readPreference=secondary"
Current Mongosh Log ID: 658e95dc23c0825f9f980a1a
Connecting to: mongodb+srv://<credentials>@mongodb-svc.mongo-repl.svc.cluster.local/admin?replicaSet=mongodb&ssl=false&readPreference=secondary&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
To help improve our products, anonymous usage data is collected and sent to MongoDB periodically (https://www.mongodb.com/legal/privacy-policy).
You can opt-out by running the disableTelemetry() command.
------
The server generated these startup warnings when booting
2023-12-29T08:54:52.230+00:00: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem
2023-12-29T08:54:52.230+00:00: The configured WiredTiger cache size is more than 80% of available RAM. See http://dochub.mongodb.org/core/faq-memory-diagnostics-wt
2023-12-29T08:54:52.927+00:00: You are running on a NUMA machine. We suggest launching mongod like this to avoid performance problems: numactl --interleave=all mongod [other options]
2023-12-29T08:54:52.927+00:00: /sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never'
2023-12-29T08:54:52.927+00:00: /sys/kernel/mm/transparent_hugepage/defrag is 'always'. We suggest setting it to 'never'
2023-12-29T08:54:52.927+00:00: vm.max_map_count is too low
------
mongodb [primary] admin>
- 连接
Mongosh验证配置和集群状态
// 查看数据库的当前连接状态，包括连接用户的信息和角色等
mongodb [primary] admin> db.runCommand({connectionStatus : 1})
{
authInfo: {
authenticatedUsers: [ { user: 'root', db: 'admin' } ],
authenticatedUserRoles: [ { role: 'root', db: 'admin' } ]
},
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703843334, i: 1 }),
signature: {
hash: Binary.createFromBase64('wtVTIZ9uuK36t3Km3jZyKimjxAU=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703843334, i: 1 })
}
// 列出 MongoDB 上的所有数据库
mongodb [primary] admin> show dbs
admin 172.00 KiB
config 176.00 KiB
local 484.00 KiB
// 切换当前的数据库到 admin 数据库
mongodb [primary] admin> use admin
already on db admin
// 查看所有数据库的用户信息
mongodb [primary] admin> db.system.users.find()
[
{
_id: 'admin.mms-automation',
userId: UUID('03cc6896-7d21-4c15-9232-59b850037354'),
user: 'mms-automation',
db: 'admin',
credentials: {
'SCRAM-SHA-256': {
iterationCount: 15000,
salt: 'j+8GOsmttVvlXKtBwt7VQ0J9aAChc/RRwoSxwQ==',
storedKey: '2JcvTsOAuxsM2Lgt7CcNNZROVr73IxJr+ZtyqVZx9eM=',
serverKey: 'rCYjGVIILlYMObuZv7OH11I0oCax4022CjiPBGqZXUo='
}
},
roles: [
{ role: 'backup', db: 'admin' },
{ role: 'readWriteAnyDatabase', db: 'admin' },
{ role: 'restore', db: 'admin' },
{ role: 'dbAdminAnyDatabase', db: 'admin' },
{ role: 'userAdminAnyDatabase', db: 'admin' },
{ role: 'clusterAdmin', db: 'admin' }
]
},
{
_id: 'admin.root',
user: 'root',
db: 'admin',
credentials: {
'SCRAM-SHA-256': {
iterationCount: 15000,
salt: 'mCCf6in3BYhljlLs3q495Eo1xtfwpERLqoepLQ==',
storedKey: 'nLiWzq1jrODdjPPgVqGQl9hE8PmBnuYvykXHZFpKBY4=',
serverKey: 'SkzaF+2gGV3ZL5C3bWQTksF5IQdfVXYIuyH2L4EufLI='
}
},
roles: [ { role: 'root', db: 'admin', minFcv: '' } ],
authenticationRestrictions: [],
userId: UUID('a7100736-2abc-404c-a5a1-e7d17233c13f')
},
{
_id: 'admin.admin',
user: 'admin',
db: 'admin',
credentials: {
'SCRAM-SHA-256': {
iterationCount: 15000,
salt: '5wD7fQwKCAjNi34vrsq5ERHEqnIn117L5B3EOA==',
storedKey: 'kfkGNAOZK3H+NUIgIbJ3fPSFSXfGG2TjaeBhiznxXxs=',
serverKey: 'mWBKpWkUqCc6taTpG4tnOfiBdiPl9cjGWwBuhgrP1Yo='
}
},
roles: [
{ role: 'dbAdminAnyDatabase', db: 'admin', minFcv: '' },
{ role: 'readWriteAnyDatabase', db: 'admin', minFcv: '' },
{ role: 'userAdminAnyDatabase', db: 'admin', minFcv: '' }
],
authenticationRestrictions: [],
userId: UUID('75a11c21-3029-4eef-88f6-47788e5bdc62')
},
{
_id: 'mydb.appuser',
user: 'appuser',
db: 'mydb',
credentials: {
'SCRAM-SHA-256': {
iterationCount: 15000,
salt: 'hvmm+8ssMZHpdYFeGXER/XSICN1Lh+Fwd09Akw==',
storedKey: 'uobhKe8VO1RQvEWfIAphcIc+jL46bUWQXKLwfjvY0bM=',
serverKey: 'L1w/NwAIzIMKqLADDBZseMDjeLcYq05lcUXIyT/nbuI='
}
},
roles: [ { role: 'readWrite', db: 'mydb', minFcv: '' } ],
authenticationRestrictions: [],
userId: UUID('522098c5-269e-46e0-9733-cab2df84db9b')
}
]
// 返回副本集的状态信息
mongodb [primary] admin> rs.status()
{
set: 'mongodb',
date: ISODate('2023-12-29T10:00:46.780Z'),
myState: 1,
term: Long('1'),
syncSourceHost: '',
syncSourceId: -1,
heartbeatIntervalMillis: Long('2000'),
majorityVoteCount: 2,
writeMajorityCount: 2,
votingMembersCount: 3,
writableVotingMembersCount: 2,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1703844044, i: 1 }), t: Long('1') },
lastCommittedWallTime: ISODate('2023-12-29T10:00:44.718Z'),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1703844044, i: 1 }), t: Long('1') },
appliedOpTime: { ts: Timestamp({ t: 1703844044, i: 1 }), t: Long('1') },
durableOpTime: { ts: Timestamp({ t: 1703844044, i: 1 }), t: Long('1') },
lastAppliedWallTime: ISODate('2023-12-29T10:00:44.718Z'),
lastDurableWallTime: ISODate('2023-12-29T10:00:44.718Z')
},
lastStableRecoveryTimestamp: Timestamp({ t: 1703843993, i: 6 }),
electionCandidateMetrics: {
lastElectionReason: 'electionTimeout',
lastElectionDate: ISODate('2023-12-29T08:56:04.371Z'),
electionTerm: Long('1'),
lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 1703840154, i: 1 }), t: Long('-1') },
lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1703840154, i: 1 }), t: Long('-1') },
numVotesNeeded: 2,
priorityAtElection: 1,
electionTimeoutMillis: Long('10000'),
numCatchUpOps: Long('0'),
newTermStartDate: ISODate('2023-12-29T08:56:04.488Z'),
wMajorityWriteAvailabilityDate: ISODate('2023-12-29T08:56:05.065Z')
},
members: [
{
_id: 0,
name: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 1,
stateStr: 'PRIMARY',
uptime: 3954,
optime: { ts: Timestamp({ t: 1703844044, i: 1 }), t: Long('1') },
optimeDate: ISODate('2023-12-29T10:00:44.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T10:00:44.718Z'),
lastDurableWallTime: ISODate('2023-12-29T10:00:44.718Z'),
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
electionTime: Timestamp({ t: 1703840164, i: 1 }),
electionDate: ISODate('2023-12-29T08:56:04.000Z'),
configVersion: 1,
configTerm: 1,
self: true,
lastHeartbeatMessage: ''
},
{
_id: 1,
name: 'mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 2,
stateStr: 'SECONDARY',
uptime: 3892,
optime: { ts: Timestamp({ t: 1703844044, i: 1 }), t: Long('1') },
optimeDurable: { ts: Timestamp({ t: 1703844044, i: 1 }), t: Long('1') },
optimeDate: ISODate('2023-12-29T10:00:44.000Z'),
optimeDurableDate: ISODate('2023-12-29T10:00:44.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T10:00:44.718Z'),
lastDurableWallTime: ISODate('2023-12-29T10:00:44.718Z'),
lastHeartbeat: ISODate('2023-12-29T10:00:45.581Z'),
lastHeartbeatRecv: ISODate('2023-12-29T10:00:44.952Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
syncSourceId: 0,
infoMessage: '',
configVersion: 1,
configTerm: 1
},
{
_id: 100,
name: 'mongodb-arb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 7,
stateStr: 'ARBITER',
uptime: 3892,
lastHeartbeat: ISODate('2023-12-29T10:00:45.582Z'),
lastHeartbeatRecv: ISODate('2023-12-29T10:00:45.581Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 1
}
],
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703844044, i: 1 }),
signature: {
hash: Binary.createFromBase64('MrGmoETGp3SW23/VZ+UnlYOQ2ms=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703844044, i: 1 })
}
- 其它账号验证连接
root@k8s-client:~# kubectl get pods -n mongo-repl -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-client-5b74bfbbc8-qmjzh 1/1 Running 0 54m 10.42.1.148 kube-dev02 <none> <none>
mongodb-0 2/2 Running 0 98m 10.42.1.103 kube-dev02 <none> <none>
mongodb-1 2/2 Running 0 97m 10.42.0.129 kube-dev01 <none> <none>
mongodb-arb-0 2/2 Running 0 98m 10.42.2.20 kube-dev03 <none> <none>
mongodb-kubernetes-operator-584c9fcb79-qg8xk 1/1 Running 0 179m 10.42.1.17 kube-dev02 <none> <none>
root@k8s-node1:~# kubectl exec -it mongo-client-5b74bfbbc8-qmjzh -n mongo-repl -- /bin/bash
root@mongo-client-5b74bfbbc8-qmjzh:/#
// 是用 root 用户密码作为凭证进行登录
root@mongo-client-5b74bfbbc8-qmjzh:/# mongosh --host 10.42.0.129 --port 27017 -u root -p 'root@12345' --authenticationDatabase admin
Current Mongosh Log ID: 658ea0827976736f70dc71ca
Connecting to: mongodb://<credentials>@10.42.0.129:27017/?directConnection=true&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
------
The server generated these startup warnings when booting
2023-12-29T08:55:50.137+00:00: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem
2023-12-29T08:55:50.137+00:00: The configured WiredTiger cache size is more than 80% of available RAM. See http://dochub.mongodb.org/core/faq-memory-diagnostics-wt
2023-12-29T08:55:50.860+00:00: You are running on a NUMA machine. We suggest launching mongod like this to avoid performance problems: numactl --interleave=all mongod [other options]
2023-12-29T08:55:50.860+00:00: /sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never'
2023-12-29T08:55:50.860+00:00: /sys/kernel/mm/transparent_hugepage/defrag is 'always'. We suggest setting it to 'never'
2023-12-29T08:55:50.860+00:00: vm.max_map_count is too low
------
mongodb [direct: secondary] test>
// 连接业务普通用户时，需要注意要指定库名
root@mongo-client-5b74bfbbc8-qmjzh:/# mongosh --host 10.42.0.129 --port 27017 -u appuser -p 'app@12345' mydb
Current Mongosh Log ID: 658ea1770e38c2f427d201a6
Connecting to: mongodb://<credentials>@10.42.0.129:27017/mydb?directConnection=true&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
mongodb [direct: secondary] mydb>
Data Consistency
主备节点数据同步一致性校验：
// 连接到 primary 节点
[root@localhost ~]# mongosh --host 10.93.40.51 --port 31181 -u appuser -p 'app@12345' mydb
Current Mongosh Log ID: 658eea9af9379d5e496cd74f
Connecting to: mongodb://<credentials>@10.93.40.51:31181/mydb?directConnection=true&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
Deprecation warnings:
- Using mongosh on the current operating system is deprecated, and support may be removed in a future release.
See https://www.mongodb.com/docs/mongodb-shell/install/#supported-operating-systems for documentation on supported platforms.
// 新建 tt 集合，并插入 10 条文档
mongodb [direct: primary] mydb> for (var i = 1; i <= 10; i++) { db.tt.insertOne({ field: i }); }
{
acknowledged: true,
insertedId: ObjectId('658eeaa5f9379d5e496cd759')
}
// 查看 tt 集合中的文档数量
mongodb [direct: primary] mydb> db.tt.countDocuments()
10
// 连接到 secondary 节点
[root@localhost ~]# mongosh --host 10.93.40.51 --port 31182 -u appuser -p 'app@12345' mydb
Current Mongosh Log ID: 658eead16f8283fb1a40a801
Connecting to: mongodb://<credentials>@10.93.40.51:31182/mydb?directConnection=true&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
Deprecation warnings:
- Using mongosh on the current operating system is deprecated, and support may be removed in a future release.
See https://www.mongodb.com/docs/mongodb-shell/install/#supported-operating-systems for documentation on supported platforms.
// 默认备节点不允许读取数据，关闭其限制
mongodb [direct: secondary] mydb> db.getMongo().setReadPref("secondaryPreferred")
// 可以看到备节点也可以正常访问到从主节点同步过来的数据（当然在写入十分频繁的场景下，也可能会存在一些同步延迟）
mongodb [direct: secondary] mydb> db.tt.find().sort({field: 1});
[
{ _id: ObjectId('658eeaa4f9379d5e496cd750'), field: 1 },
{ _id: ObjectId('658eeaa5f9379d5e496cd751'), field: 2 },
{ _id: ObjectId('658eeaa5f9379d5e496cd752'), field: 3 },
{ _id: ObjectId('658eeaa5f9379d5e496cd753'), field: 4 },
{ _id: ObjectId('658eeaa5f9379d5e496cd754'), field: 5 },
{ _id: ObjectId('658eeaa5f9379d5e496cd755'), field: 6 },
{ _id: ObjectId('658eeaa5f9379d5e496cd756'), field: 7 },
{ _id: ObjectId('658eeaa5f9379d5e496cd757'), field: 8 },
{ _id: ObjectId('658eeaa5f9379d5e496cd758'), field: 9 },
{ _id: ObjectId('658eeaa5f9379d5e496cd759'), field: 10 }
]
HA Testing
NodePort 测试环境的限制和问题：
当我们在 Kubernetes 集群中部署 MongoDB 集群时，并使用 MongoDB Operator 自动创建 Secret，遇到的主要问题是 MongoDB Operator 生成的连接字符串仅包含内部服务名，不支持 NodePort。
在生产环境中，我们更常见的是使用 LoadBalancer 服务，因为 LoadBalancer 能够将流量智能地路由到集群中的多个 MongoDB 节点。然而，在开发环境中，我们可能会更倾向于使用 NodePort。 对于在开发环境中解决服务名与 IP 映射的问题，一种可行的方法是在 hosts 文件中手动添加映射。然而这并不是一种理想的解决方案，因为它没有解决端口映射的问题，并且可能在大规模部署或动态环境中引入更多问题。
在我们的 MongoDB 客户端，无论是 MongoDB Shell 还是 Go 驱动程序，连接 MongoDB 服务器时，服务器会返回包含集群成员信息的元数据。在这个元数据中，成员节点使用内部服务名称表示，这在外部环境中是无法解析的，即便我们更改了 hosts 文件。并且，元数据中的端口默认是 MongoDB 的标准端口 27017，而在 NodePort 服务中，我们需要为每个节点使用不同的端口。
要在开发环境中成功使用 Go 驱动程序连接 MondgoDB，我们可能需要做出一些调整。首先，在 Operator 配置时，我们可能需要为每个 MongoDB 节点设置一个唯一的内部端口并使其与节点的 NodePort 对应。其次，我们需要在 hosts 文件中添加相应的映射，将内部服务名映射到正确的 IP 地址。
然而请注意，这只是理论上的解决方案，还没有得到实际验证，可能并不适用于所有的环境。我们总是推荐在生产环境中使用 LoadBalancer 类型服务或 Service Mesh 来处理这种问题，而不是进行复杂而可能引入新问题的手动配置操作。
我们原计划在开发环境中模拟真实的业务场景，使用 Go 程序通过连接字符串持续连接 MongoDB，并且不断地向数据库插入数据。我们观察并模拟了在这个过程中 MongoDB 主节点和备份节点可能会遇到的故障，以探索其高可用性 （HA） 和故障转移 （Failover） 的特性。整个实验的运行结果呈现在 Go 程序的日志中，我们从中观察到，在故障切换的过程中，有一段时间内数据无法被正常写入。在这个过程中，我们利用了 MongoDB 提供的服务发现机制，这让开发者可以很方便地建立数据库连接，而无需关心具体连接到哪个节点。尽管存在少数情况下数据无法正常写入的情形，但在大多数情况下，MongoDB 复制集所展现出的数据强一致性和主备节点切换的特性无疑对业务透明。也就是说，对于业务来说，几秒钟内无法正常写入的数据几乎不会对整体业务产生影响。虽然由于环境的限制，我们并未能完全实施所有的模拟实验，但我们依然可以通过模拟故障现场的方式——即删除 Kubernetes 中的 Pod 资源，来观察和体验 MongoDB 复制集强大的高可用性能力。
- 模拟备节点故障的情况（体现自愈能力）：
# 获取 mongo-repl 名称空间中所有 Pod 的详细信息，同时将各个 Pod 的 IP 地址，所在节点等信息展示出来
root@k8s-client:~# kubectl get pods -n mongo-repl -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-client-5b74bfbbc8-qmjzh 1/1 Running 0 8h 10.42.1.148 kube-dev02 <none> <none>
mongodb-0 2/2 Running 0 9h 10.42.1.103 kube-dev02 <none> <none>
mongodb-1 2/2 Running 0 3m46s 10.42.0.130 kube-dev01 <none> <none>
mongodb-arb-0 2/2 Running 0 9h 10.42.2.20 kube-dev03 <none> <none>
mongodb-kubernetes-operator-584c9fcb79-7t8wf 1/1 Running 0 10h 10.42.1.17 kube-dev02 <none> <none>
# 删除 mongo-repl 名称空间中名为 mongodb-1 的 Pod，以模拟故障发生
root@k8s-client:~# kubectl delete pod -n mongo-repl mongodb-1
pod "mongodb-1" deleted
# 在删除 Pod 后，再次执行获取 Pod 信息的命令，以查看Pod删除后Kubernetes集群中的变化
root@k8s-client:~# kubectl get pods -n mongo-repl -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-client-5b74bfbbc8-qmjzh 1/1 Running 0 8h 10.42.1.148 kube-dev02 <none> <none>
mongodb-0 2/2 Running 0 9h 10.42.1.103 kube-dev02 <none> <none>
mongodb-1 0/2 Init:0/2 0 3s <none> kube-dev01 <none> <none>
mongodb-arb-0 2/2 Running 0 9h 10.42.2.20 kube-dev03 <none> <none>
mongodb-kubernetes-operator-584c9fcb79-7t8wf 1/1 Running 0 10h 10.42.1.17 kube-dev02 <none> <none>
# 进入名为 mongo-client-5b74bfbbc8-qmjzh 的 Pod 中
root@k8s-client:~# kubectl -n mongo-repl exec -it mongo-client-5b74bfbbc8-qmjzh -- /bin/bash
# 在 Pod 内部，使用 mongosh 连接到 MongoDB 主节点
root@mongo-client-5b74bfbbc8-qmjzh:/# mongosh --host 10.42.1.103 --port 27017 -u root -p 'root@12345'
Current Mongosh Log ID: 658f0f202f5261240ea405c7
Connecting to: mongodb://<credentials>@10.42.1.103:27017/?directConnection=true&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
------
The server generated these startup warnings when booting
2023-12-29T08:54:52.230+00:00: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem
2023-12-29T08:54:52.230+00:00: The configured WiredTiger cache size is more than 80% of available RAM. See http://dochub.mongodb.org/core/faq-memory-diagnostics-wt
2023-12-29T08:54:52.927+00:00: You are running on a NUMA machine. We suggest launching mongod like this to avoid performance problems: numactl --interleave=all mongod [other options]
2023-12-29T08:54:52.927+00:00: /sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never'
2023-12-29T08:54:52.927+00:00: /sys/kernel/mm/transparent_hugepage/defrag is 'always'. We suggest setting it to 'never'
2023-12-29T08:54:52.927+00:00: vm.max_map_count is too low
------
# 查询复制集的的当前状态，可以看到 member1 的状态是 'not reachable' 的
mongodb [direct: primary] test> rs.status()
{
set: 'mongodb',
date: ISODate('2023-12-29T18:25:38.085Z'),
myState: 1,
term: Long('1'),
syncSourceHost: '',
syncSourceId: -1,
heartbeatIntervalMillis: Long('2000'),
majorityVoteCount: 2,
writeMajorityCount: 2,
votingMembersCount: 3,
writableVotingMembersCount: 2,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1703874316, i: 1 }), t: Long('1') },
lastCommittedWallTime: ISODate('2023-12-29T18:25:16.004Z'),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1703874316, i: 1 }), t: Long('1') },
appliedOpTime: { ts: Timestamp({ t: 1703874336, i: 1 }), t: Long('1') },
durableOpTime: { ts: Timestamp({ t: 1703874336, i: 1 }), t: Long('1') },
lastAppliedWallTime: ISODate('2023-12-29T18:25:36.005Z'),
lastDurableWallTime: ISODate('2023-12-29T18:25:36.005Z')
},
lastStableRecoveryTimestamp: Timestamp({ t: 1703874316, i: 1 }),
electionCandidateMetrics: {
lastElectionReason: 'electionTimeout',
lastElectionDate: ISODate('2023-12-29T08:56:04.371Z'),
electionTerm: Long('1'),
lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 1703840154, i: 1 }), t: Long('-1') },
lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1703840154, i: 1 }), t: Long('-1') },
numVotesNeeded: 2,
priorityAtElection: 1,
electionTimeoutMillis: Long('10000'),
numCatchUpOps: Long('0'),
newTermStartDate: ISODate('2023-12-29T08:56:04.488Z'),
wMajorityWriteAvailabilityDate: ISODate('2023-12-29T08:56:05.065Z')
},
members: [
{
_id: 0,
name: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 1,
stateStr: 'PRIMARY',
uptime: 34246,
optime: { ts: Timestamp({ t: 1703874336, i: 1 }), t: Long('1') },
optimeDate: ISODate('2023-12-29T18:25:36.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:25:36.005Z'),
lastDurableWallTime: ISODate('2023-12-29T18:25:36.005Z'),
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
electionTime: Timestamp({ t: 1703840164, i: 1 }),
electionDate: ISODate('2023-12-29T08:56:04.000Z'),
configVersion: 1,
configTerm: 1,
self: true,
lastHeartbeatMessage: ''
},
{
_id: 1,
name: 'mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 0,
state: 8,
stateStr: '(not reachable/healthy)',
uptime: 0,
optime: { ts: Timestamp({ t: 0, i: 0 }), t: Long('-1') },
optimeDurable: { ts: Timestamp({ t: 0, i: 0 }), t: Long('-1') },
optimeDate: ISODate('1970-01-01T00:00:00.000Z'),
optimeDurableDate: ISODate('1970-01-01T00:00:00.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:25:16.004Z'),
lastDurableWallTime: ISODate('2023-12-29T18:25:16.004Z'),
lastHeartbeat: ISODate('2023-12-29T18:25:29.478Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:25:18.632Z'),
pingMs: Long('0'),
lastHeartbeatMessage: "Couldn't get a connection within the time limit",
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 1
},
{
_id: 100,
name: 'mongodb-arb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 7,
stateStr: 'ARBITER',
uptime: 34183,
lastHeartbeat: ISODate('2023-12-29T18:25:37.435Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:25:37.479Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 1
}
],
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703874336, i: 1 }),
signature: {
hash: Binary.createFromBase64('3gp9QE+FspWGmMzg/zlU6PqhslM=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703874336, i: 1 })
}
# 等待一段时间，待 Pod 恢复，member1 自动加到集群成为 'SECONDARY'
mongodb [direct: primary] test> rs.status()
{
set: 'mongodb',
date: ISODate('2023-12-29T18:26:36.459Z'),
myState: 1,
term: Long('1'),
syncSourceHost: '',
syncSourceId: -1,
heartbeatIntervalMillis: Long('2000'),
majorityVoteCount: 2,
writeMajorityCount: 2,
votingMembersCount: 3,
writableVotingMembersCount: 2,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1703874396, i: 1 }), t: Long('1') },
lastCommittedWallTime: ISODate('2023-12-29T18:26:36.007Z'),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1703874396, i: 1 }), t: Long('1') },
appliedOpTime: { ts: Timestamp({ t: 1703874396, i: 1 }), t: Long('1') },
durableOpTime: { ts: Timestamp({ t: 1703874396, i: 1 }), t: Long('1') },
lastAppliedWallTime: ISODate('2023-12-29T18:26:36.007Z'),
lastDurableWallTime: ISODate('2023-12-29T18:26:36.007Z')
},
lastStableRecoveryTimestamp: Timestamp({ t: 1703874386, i: 1 }),
electionCandidateMetrics: {
lastElectionReason: 'electionTimeout',
lastElectionDate: ISODate('2023-12-29T08:56:04.371Z'),
electionTerm: Long('1'),
lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 1703840154, i: 1 }), t: Long('-1') },
lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1703840154, i: 1 }), t: Long('-1') },
numVotesNeeded: 2,
priorityAtElection: 1,
electionTimeoutMillis: Long('10000'),
numCatchUpOps: Long('0'),
newTermStartDate: ISODate('2023-12-29T08:56:04.488Z'),
wMajorityWriteAvailabilityDate: ISODate('2023-12-29T08:56:05.065Z')
},
members: [
{
_id: 0,
name: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 1,
stateStr: 'PRIMARY',
uptime: 34304,
optime: { ts: Timestamp({ t: 1703874396, i: 1 }), t: Long('1') },
optimeDate: ISODate('2023-12-29T18:26:36.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:26:36.007Z'),
lastDurableWallTime: ISODate('2023-12-29T18:26:36.007Z'),
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
electionTime: Timestamp({ t: 1703840164, i: 1 }),
electionDate: ISODate('2023-12-29T08:56:04.000Z'),
configVersion: 1,
configTerm: 1,
self: true,
lastHeartbeatMessage: ''
},
{
_id: 1,
name: 'mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 2,
stateStr: 'SECONDARY',
uptime: 56,
optime: { ts: Timestamp({ t: 1703874386, i: 1 }), t: Long('1') },
optimeDurable: { ts: Timestamp({ t: 1703874386, i: 1 }), t: Long('1') },
optimeDate: ISODate('2023-12-29T18:26:26.000Z'),
optimeDurableDate: ISODate('2023-12-29T18:26:26.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:26:36.007Z'),
lastDurableWallTime: ISODate('2023-12-29T18:26:36.007Z'),
lastHeartbeat: ISODate('2023-12-29T18:26:35.573Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:26:34.639Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
syncSourceId: 0,
infoMessage: '',
configVersion: 1,
configTerm: 1
},
{
_id: 100,
name: 'mongodb-arb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 7,
stateStr: 'ARBITER',
uptime: 34242,
lastHeartbeat: ISODate('2023-12-29T18:26:35.464Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:26:35.500Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 1
}
],
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703874396, i: 1 }),
signature: {
hash: Binary.createFromBase64('eHNIOSIkAiIELxznL4pAalGWOlU=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703874396, i: 1 })
}
- 模拟主节点故障的情况（会发生
Failover）：
# 查看 mongo-repl 命名空间下的所有 Pod，以及它们对应的 Node 和其他信息
root@k8s-client:~# kubectl get pods -n mongo-repl -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-client-5b74bfbbc8-qmjzh 1/1 Running 0 9h 10.42.1.148 kube-dev02 <none> <none>
mongodb-0 2/2 Running 0 9h 10.42.1.103 kube-dev02 <none> <none>
mongodb-1 2/2 Running 0 18m 10.42.0.131 kube-dev01 <none> <none>
mongodb-arb-0 2/2 Running 0 9h 10.42.2.20 kube-dev03 <none> <none>
mongodb-kubernetes-operator-584c9fcb79-7t8wf 1/1 Running 0 11h 10.42.1.17 kube-dev02 <none> <none>
# 删除 mongo-repl 名称空间中名为 mongodb-0 的 Pod，以模拟故障发生
root@k8s-client:~# kubectl delete pod -n mongo-repl mongodb-0
pod "mongodb-0" deleted
# 在删除 Pod 后，再次执行获取 Pod 信息的命令，以查看Pod删除后Kubernetes集群中的变化
root@k8s-client:~# kubectl get pods -n mongo-repl -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
mongo-client-5b74bfbbc8-qmjzh 1/1 Running 0 9h 10.42.1.148 kube-dev02 <none> <none>
mongodb-0 0/2 Init:0/2 0 3s <none> kube-dev02 <none> <none>
mongodb-1 2/2 Running 0 18m 10.42.0.131 kube-dev01 <none> <none>
mongodb-arb-0 2/2 Running 0 9h 10.42.2.20 kube-dev03 <none> <none>
mongodb-kubernetes-operator-584c9fcb79-7t8wf 1/1 Running 0 11h 10.42.1.17 kube-dev02 <none> <none>
# 进入名为 mongo-client-5b74bfbbc8-qmjzh 的 Pod 中
root@k8s-client:~# kubectl -n mongo-repl exec -it mongo-client-5b74bfbbc8-qmjzh -- /bin/bash
# 使用 mongosh 连接到备份节点（mongodb-1）的 MongoDB 服务
root@mongo-client-5b74bfbbc8-qmjzh:/# mongosh --host 10.42.0.131 --port 27017 -u root -p 'root@12345'
Current Mongosh Log ID: 658f138c523d8c4ee90b8ea6
Connecting to: mongodb://<credentials>@10.42.0.131:27017/?directConnection=true&appName=mongosh+2.1.1
Using MongoDB: 6.0.5
Using Mongosh: 2.1.1
For mongosh info see: https://docs.mongodb.com/mongodb-shell/
------
The server generated these startup warnings when booting
2023-12-29T18:25:37.256+00:00: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem
2023-12-29T18:25:37.256+00:00: The configured WiredTiger cache size is more than 80% of available RAM. See http://dochub.mongodb.org/core/faq-memory-diagnostics-wt
2023-12-29T18:25:38.397+00:00: You are running on a NUMA machine. We suggest launching mongod like this to avoid performance problems: numactl --interleave=all mongod [other options]
2023-12-29T18:25:38.397+00:00: /sys/kernel/mm/transparent_hugepage/enabled is 'always'. We suggest setting it to 'never'
2023-12-29T18:25:38.397+00:00: /sys/kernel/mm/transparent_hugepage/defrag is 'always'. We suggest setting it to 'never'
2023-12-29T18:25:38.397+00:00: vm.max_map_count is too low
------
# 查询 MongoDB 复制集的状态。可以看到，主节点已经被成功删除，并且备份节点 mongodb-1 已经变成了新的主节点
mongodb [direct: primary] test> rs.status()
{
set: 'mongodb',
date: ISODate('2023-12-29T18:44:31.634Z'),
myState: 1,
term: Long('3'),
syncSourceHost: '',
syncSourceId: -1,
heartbeatIntervalMillis: Long('2000'),
majorityVoteCount: 2,
writeMajorityCount: 2,
votingMembersCount: 3,
writableVotingMembersCount: 2,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1703875443, i: 2 }), t: Long('3') },
lastCommittedWallTime: ISODate('2023-12-29T18:44:03.052Z'),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1703875443, i: 2 }), t: Long('3') },
appliedOpTime: { ts: Timestamp({ t: 1703875463, i: 1 }), t: Long('3') },
durableOpTime: { ts: Timestamp({ t: 1703875463, i: 1 }), t: Long('3') },
lastAppliedWallTime: ISODate('2023-12-29T18:44:23.054Z'),
lastDurableWallTime: ISODate('2023-12-29T18:44:23.054Z')
},
lastStableRecoveryTimestamp: Timestamp({ t: 1703875416, i: 1 }),
electionCandidateMetrics: {
lastElectionReason: 'electionTimeout',
lastElectionDate: ISODate('2023-12-29T18:44:03.028Z'),
electionTerm: Long('3'),
lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 1703875426, i: 1 }), t: Long('1') },
lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1703875426, i: 1 }), t: Long('1') },
numVotesNeeded: 2,
priorityAtElection: 1,
electionTimeoutMillis: Long('10000'),
numCatchUpOps: Long('0'),
newTermStartDate: ISODate('2023-12-29T18:44:03.052Z'),
wMajorityWriteAvailabilityDate: ISODate('2023-12-29T18:44:03.592Z')
},
members: [
{
_id: 0,
name: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 0,
state: 8,
stateStr: '(not reachable/healthy)',
uptime: 0,
optime: { ts: Timestamp({ t: 0, i: 0 }), t: Long('-1') },
optimeDurable: { ts: Timestamp({ t: 0, i: 0 }), t: Long('-1') },
optimeDate: ISODate('1970-01-01T00:00:00.000Z'),
optimeDurableDate: ISODate('1970-01-01T00:00:00.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:44:03.052Z'),
lastDurableWallTime: ISODate('2023-12-29T18:44:03.052Z'),
lastHeartbeat: ISODate('2023-12-29T18:44:31.102Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:44:06.060Z'),
pingMs: Long('0'),
lastHeartbeatMessage: 'Error connecting to mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017 (10.42.1.230:27017) :: caused by :: Connection refused',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 3
},
{
_id: 1,
name: 'mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 1,
stateStr: 'PRIMARY',
uptime: 1134,
optime: { ts: Timestamp({ t: 1703875463, i: 1 }), t: Long('3') },
optimeDate: ISODate('2023-12-29T18:44:23.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:44:23.054Z'),
lastDurableWallTime: ISODate('2023-12-29T18:44:23.054Z'),
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
electionTime: Timestamp({ t: 1703875443, i: 1 }),
electionDate: ISODate('2023-12-29T18:44:03.000Z'),
configVersion: 1,
configTerm: 3,
self: true,
lastHeartbeatMessage: ''
},
{
_id: 100,
name: 'mongodb-arb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 7,
stateStr: 'ARBITER',
uptime: 1132,
lastHeartbeat: ISODate('2023-12-29T18:44:31.063Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:44:31.065Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 3
}
],
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703875463, i: 1 }),
signature: {
hash: Binary.createFromBase64('fq2GTCR9t8h0/DAHZendoFld5Vs=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703875463, i: 1 })
}
# 过段时间后，再次查询复制集的状态，可以看到原来主节点的 Pod 已经被 Kubernetes 自动重建，并以备份节点的身份加入到了复制集中。
- 执行手动重新选举操作（主备切换）：
// 当前集群状态是 mongodb-0 为 'SECONDARY'，mongodb-1 为 'PRIMARY'
mongodb [direct: primary] test> rs.status()
{
set: 'mongodb',
date: ISODate('2023-12-29T18:52:05.677Z'),
myState: 1,
term: Long('3'),
syncSourceHost: '',
syncSourceId: -1,
heartbeatIntervalMillis: Long('2000'),
majorityVoteCount: 2,
writeMajorityCount: 2,
votingMembersCount: 3,
writableVotingMembersCount: 2,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1703875923, i: 1 }), t: Long('3') },
lastCommittedWallTime: ISODate('2023-12-29T18:52:03.071Z'),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1703875923, i: 1 }), t: Long('3') },
appliedOpTime: { ts: Timestamp({ t: 1703875923, i: 1 }), t: Long('3') },
durableOpTime: { ts: Timestamp({ t: 1703875923, i: 1 }), t: Long('3') },
lastAppliedWallTime: ISODate('2023-12-29T18:52:03.071Z'),
lastDurableWallTime: ISODate('2023-12-29T18:52:03.071Z')
},
lastStableRecoveryTimestamp: Timestamp({ t: 1703875893, i: 1 }),
electionCandidateMetrics: {
lastElectionReason: 'electionTimeout',
lastElectionDate: ISODate('2023-12-29T18:44:03.028Z'),
electionTerm: Long('3'),
lastCommittedOpTimeAtElection: { ts: Timestamp({ t: 1703875426, i: 1 }), t: Long('1') },
lastSeenOpTimeAtElection: { ts: Timestamp({ t: 1703875426, i: 1 }), t: Long('1') },
numVotesNeeded: 2,
priorityAtElection: 1,
electionTimeoutMillis: Long('10000'),
numCatchUpOps: Long('0'),
newTermStartDate: ISODate('2023-12-29T18:44:03.052Z'),
wMajorityWriteAvailabilityDate: ISODate('2023-12-29T18:44:03.592Z')
},
members: [
{
_id: 0,
name: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 2,
stateStr: 'SECONDARY',
uptime: 452,
optime: { ts: Timestamp({ t: 1703875923, i: 1 }), t: Long('3') },
optimeDurable: { ts: Timestamp({ t: 1703875923, i: 1 }), t: Long('3') },
optimeDate: ISODate('2023-12-29T18:52:03.000Z'),
optimeDurableDate: ISODate('2023-12-29T18:52:03.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:52:03.071Z'),
lastDurableWallTime: ISODate('2023-12-29T18:52:03.071Z'),
lastHeartbeat: ISODate('2023-12-29T18:52:05.298Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:52:05.616Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: 'mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017',
syncSourceId: 1,
infoMessage: '',
configVersion: 1,
configTerm: 3
},
{
_id: 1,
name: 'mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 1,
stateStr: 'PRIMARY',
uptime: 1588,
optime: { ts: Timestamp({ t: 1703875923, i: 1 }), t: Long('3') },
optimeDate: ISODate('2023-12-29T18:52:03.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:52:03.071Z'),
lastDurableWallTime: ISODate('2023-12-29T18:52:03.071Z'),
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
electionTime: Timestamp({ t: 1703875443, i: 1 }),
electionDate: ISODate('2023-12-29T18:44:03.000Z'),
configVersion: 1,
configTerm: 3,
self: true,
lastHeartbeatMessage: ''
},
{
_id: 100,
name: 'mongodb-arb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 7,
stateStr: 'ARBITER',
uptime: 1586,
lastHeartbeat: ISODate('2023-12-29T18:52:05.154Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:52:05.153Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 3
}
],
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703875923, i: 1 }),
signature: {
hash: Binary.createFromBase64('mjPWiuEvF6rzP+MA1njx3pL5koU=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703875923, i: 1 })
}
// 强制重新竞选主节点（且当前主节点120s内降级且不允许参与竞选）
mongodb [direct: primary] test> db.adminCommand({ replSetStepDown: 120 })
{
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703876033, i: 1 }),
signature: {
hash: Binary.createFromBase64('40i0WrEIHFKFeLjKf+P3iMchFOc=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703876033, i: 1 })
}
// 由于总共就两个数据节点（1主1从1选举），因此强制竞选的结果就相当于主备切换，当然如果有更多的备节点，原先的主节点在恢复后仍具有重新参选的可能性，然而这并不是绝对的，因为选举结果取决于多种因素，如节点的优先级设定、系统的网络延迟等
mongodb [direct: secondary] test> rs.status()
{
set: 'mongodb',
date: ISODate('2023-12-29T18:54:20.087Z'),
myState: 2,
term: Long('4'),
syncSourceHost: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
syncSourceId: 0,
heartbeatIntervalMillis: Long('2000'),
majorityVoteCount: 2,
writeMajorityCount: 2,
votingMembersCount: 3,
writableVotingMembersCount: 2,
optimes: {
lastCommittedOpTime: { ts: Timestamp({ t: 1703876050, i: 1 }), t: Long('4') },
lastCommittedWallTime: ISODate('2023-12-29T18:54:10.591Z'),
readConcernMajorityOpTime: { ts: Timestamp({ t: 1703876050, i: 1 }), t: Long('4') },
appliedOpTime: { ts: Timestamp({ t: 1703876050, i: 1 }), t: Long('4') },
durableOpTime: { ts: Timestamp({ t: 1703876050, i: 1 }), t: Long('4') },
lastAppliedWallTime: ISODate('2023-12-29T18:54:10.591Z'),
lastDurableWallTime: ISODate('2023-12-29T18:54:10.591Z')
},
lastStableRecoveryTimestamp: Timestamp({ t: 1703876013, i: 1 }),
electionParticipantMetrics: {
votedForCandidate: true,
electionTerm: Long('4'),
lastVoteDate: ISODate('2023-12-29T18:54:00.576Z'),
electionCandidateMemberId: 0,
voteReason: '',
lastAppliedOpTimeAtElection: { ts: Timestamp({ t: 1703876033, i: 1 }), t: Long('3') },
maxAppliedOpTimeInSet: { ts: Timestamp({ t: 1703876033, i: 1 }), t: Long('3') },
priorityAtElection: 1,
newTermStartDate: ISODate('2023-12-29T18:54:00.590Z'),
newTermAppliedDate: ISODate('2023-12-29T18:54:01.582Z')
},
members: [
{
_id: 0,
name: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 1,
stateStr: 'PRIMARY',
uptime: 586,
optime: { ts: Timestamp({ t: 1703876050, i: 1 }), t: Long('4') },
optimeDurable: { ts: Timestamp({ t: 1703876050, i: 1 }), t: Long('4') },
optimeDate: ISODate('2023-12-29T18:54:10.000Z'),
optimeDurableDate: ISODate('2023-12-29T18:54:10.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:54:10.591Z'),
lastDurableWallTime: ISODate('2023-12-29T18:54:10.591Z'),
lastHeartbeat: ISODate('2023-12-29T18:54:19.604Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:54:18.595Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
electionTime: Timestamp({ t: 1703876040, i: 1 }),
electionDate: ISODate('2023-12-29T18:54:00.000Z'),
configVersion: 1,
configTerm: 4
},
{
_id: 1,
name: 'mongodb-1.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 2,
stateStr: 'SECONDARY',
uptime: 1723,
optime: { ts: Timestamp({ t: 1703876050, i: 1 }), t: Long('4') },
optimeDate: ISODate('2023-12-29T18:54:10.000Z'),
lastAppliedWallTime: ISODate('2023-12-29T18:54:10.591Z'),
lastDurableWallTime: ISODate('2023-12-29T18:54:10.591Z'),
syncSourceHost: 'mongodb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
syncSourceId: 0,
infoMessage: '',
configVersion: 1,
configTerm: 4,
self: true,
lastHeartbeatMessage: ''
},
{
_id: 100,
name: 'mongodb-arb-0.mongodb-svc.mongo-repl.svc.cluster.local:27017',
health: 1,
state: 7,
stateStr: 'ARBITER',
uptime: 1720,
lastHeartbeat: ISODate('2023-12-29T18:54:19.604Z'),
lastHeartbeatRecv: ISODate('2023-12-29T18:54:18.604Z'),
pingMs: Long('0'),
lastHeartbeatMessage: '',
syncSourceHost: '',
syncSourceId: -1,
infoMessage: '',
configVersion: 1,
configTerm: 4
}
],
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1703876050, i: 1 }),
signature: {
hash: Binary.createFromBase64('Pvp/6ay28lbUNlIX7yM73lnjc90=', 0),
keyId: Long('7317937781991276549')
}
},
operationTime: Timestamp({ t: 1703876050, i: 1 })
}
Backup & Restore
用于实现 MongoDB 数据库的备份以及将备份文件上传到 S3 对象存储服务中。
One-time Tasks
执行流程图：
+---------------+
| User ENV |
| (外部环境变量) |
+-------+-------+
|
|
+-------v-------+
| MongoDB Pod |
| (mongo-client)|
+-------+-------+
|
Backup operation using `mongodump`
|
+-------v-------+
| Volume |
| (backup) |
+-------+-------+
|
Check backup file & upload to S3 using aws-cli
|
+-------v-------+
| S3 Pod |
| (aws-client) |
+-------+-------+
|
Upload backup file to S3 bucket
|
+----------v-----------+
| S3 Bucket (in AWS) |
+----------------------+
编写mongodb-backup-to-s3-pod.yaml：
---
apiVersion: v1
kind: Pod
metadata:
name: mongodb-backup
namespace: mongo-repl
spec:
restartPolicy: OnFailure
volumes:
- name: backup
emptyDir: {}
containers:
- name: mongo-client
image: mongo
command: ["/bin/sh", "-c"]
args:
- >
mongodump --uri="${CONNECTION_STRING}" --db="${MONGO_DB}" --out=/backup/mdb-${MONGO_DB}-$(date +'%Y-%m-%d') &&
sleep 15 # 让aws-client容器有足够的时间来获取到备份文件
env:
- name: MONGO_DB
value: $MONGO_DB
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: mongodb-admin-root
key: connectionString.standardSrv
volumeMounts:
- name: backup
mountPath: /backup
- name: aws-client
image: garland/aws-cli-docker:1.16.133
command: ["/bin/sh", "-c"]
args:
- >
while true; do
if [ -f /backup/mdb-${MONGO_DB}-$(date +'%Y-%m-%d') ]; then
aws s3 cp /backup/mdb-${MONGO_DB}-$(date +'%Y-%m-%d') s3://mongo-backups/${USERNAME}/mdb-${MONGO_DB}-$(date +'%Y-%m-%d')
break
fi
sleep 10
done
env:
# 注意不是 $(VAR) 的形式，而是 $VAR 这并不是写错了，在下方进行解释！
- name: MONGO_DB
value: $MONGO_DB
- name: USERNAME
value: $USERNAME
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: aws-credentials
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: aws-credentials
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
configMapKeyRef:
name: aws-config
key: AWS_DEFAULT_REGION
- name: AWS_ENDPOINT_URL
valueFrom:
configMapKeyRef:
name: aws-config
key: AWS_ENDPOINT_URL
volumeMounts:
- name: backup
mountPath: /backup
---
apiVersion: v1
kind: ConfigMap
metadata:
name: aws-config
namespace: mongo-repl
data:
AWS_DEFAULT_REGION: us-east-1
AWS_ENDPOINT_URL: http://<your-minio-service-url>
---
apiVersion: v1
kind: Secret
metadata:
name: aws-credentials
namespace: mongo-repl
type: Opaque
data:
AWS_ACCESS_KEY_ID: bWluaW9hY2Nlc3NrZXk= # Base64编码的MinIO access key
AWS_SECRET_ACCESS_KEY: bWluaW9zZWNyZXRrZXk= # Base64编码的MinIO secret key
执行方案设计
在执行备份过程中，我们的解决方案设计了两个动态变量，以满足 “谁（用户名）需要备份哪个库（数据库名称）”。这种设计提供了高度的灵活性，为后续 DAAS 云数据库服务平台中 “备份恢复功能模块” 奠定了基础，但这样也带来了额外操作挑战：如何有效处理数据库名称和用户名的动态变化。一种常见的解决方案是创建 ConfigMap，但这也引入了另外的问题，例如在完成 Pod 任务后，我们需要删除 ConfigMap，这在执行其他备份任务时会导致任务必须串行执行。
利用envsubst工具，我们能妥善处理上述问题。我们首先将原来的$(MONGO_DB)形式改为${MONGO_DB}，这样便能让envsubst正确解析和替换相关变量。
测试方案的可行性：
# 设置 VAR 变量
root@k8s-client:~# export MONGO_DB="demo_db"
root@k8s-client:~# export USERNAME="demo_user"
# 替换掉原文本中的 VAR 变量
root@k8s-client:~# envsubst < mongodb-backup-to-s3-pod.yaml | egrep "MONGO_DB|USERNAME" -A 1
- name: MONGO_DB
value: demo_db
--
- name: MONGO_DB
value: demo_db
- name: USERNAME
value: demo_user
我们需要编写一个run.sh脚本作为执行器，通过该脚本，我们能实现环境变量的动态替换并执行备份任务，从而避免了额外的 ConfigMap 管理工作以及串行执行的限制。
#!/bin/bash
export MONGO_DB=$1
export USERNAME=$2
envsubst < mongodb-backup-to-s3-pod.yaml | kubectl apply -f -
# 提供执行的权限：
chmod +x run.sh
# 运行脚本并传入实际参数：
./run.sh "database" "@mystic"
这个方法使我们可以更高效地执行备份任务，而无需为额外的 ConfigMap 管理工作分心。此外，由于每个任务都在一个新的、具有独立环境变量的 Shell Session 中进行，因此我们可以同时处理多个备份任务，避免了串行执行的局限。这种方式极大地优化了工作流程，降低了心智负担，更方便于程序端的开发调用。
CronJob Backup
编写mongodb-backup-to-s3-cron.yaml 执行定时备份任务，方案逻辑同上：
apiVersion: batch/v1beta1
kind: CronJob
metadata:
name: mongodb-backup-cron
namespace: mongo-repl
spec:
schedule: "$SCHEDULE"
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 1
jobTemplate:
spec:
template:
spec:
restartPolicy: Never
containers:
- name: mongo-client
image: mongo
command: ["/bin/sh", "-c"]
args:
- >
mongodump --uri="${CONNECTION_STRING}" --db="$MONGO_DB" --out="/backup/mdb-$MONGO_DB-$(date +'%Y-%m-%d')" &&
sleep 15
env:
- name: MONGO_DB
value: $MONGO_DB
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: mongodb-admin-root
key: connectionString.standardSrv
volumeMounts:
- name: backup
mountPath: /backup
- name: aws-client
image: garland/aws-cli-docker:1.16.133
command: ["/bin/sh", "-c"]
args:
- >
while true; do
if [ -f "/backup/mdb-$MONGO_DB-$(date +'%Y-%m-%d')" ]; then
aws s3 cp "/backup/mdb-$MONGO_DB-$(date +'%Y-%m-%d')" "s3://mongo-backups/$USERNAME/mdb-$MONGO_DB-$(date +'%Y-%m-%d')"
break
fi
sleep 10
done
env:
- name: MONGO_DB
value: $MONGO_DB
- name: USERNAME
value: $USERNAME
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: aws-credentials
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: aws-credentials
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
configMapKeyRef:
name: aws-config
key: AWS_DEFAULT_REGION
- name: AWS_ENDPOINT_URL
valueFrom:
configMapKeyRef:
name: aws-config
key: AWS_ENDPOINT_URL
volumeMounts:
- name: backup
mountPath: /backup
volumes:
- name: backup
emptyDir: {}
使用以下脚本来调用这个 CronJob：
#!/bin/bash
export MONGO_DB=$1
export USERNAME=$2
export SCHEDULE=$3
envsubst < mongodb-backup-to-s3-cron.yaml | kubectl apply -f -
运行脚本并传入实际参数：
./run-cron.sh "database" "@mystic" "*/5 * * * *"
Restore Dump
编写mongodb-``restore``-``from``-s3.yaml
apiVersion: v1
kind: Pod
metadata:
name: mongodb-restore
namespace: mongo-repl
spec:
restartPolicy: OnFailure
volumes:
- name: restore
emptyDir: {}
containers:
- name: aws-client
image: garland/aws-cli-docker:1.16.133
command: ["/bin/sh", "-c"]
args:
- >
aws s3 cp s3://mongo-backups/${USERNAME}/${FILENAME} /restore/ &&
sleep 15
env:
- name: USERNAME
value: $USERNAME
- name: FILENAME
value: $FILENAME
- name: AWS_ACCESS_KEY_ID
valueFrom:
secretKeyRef:
name: aws-credentials
key: AWS_ACCESS_KEY_ID
- name: AWS_SECRET_ACCESS_KEY
valueFrom:
secretKeyRef:
name: aws-credentials
key: AWS_SECRET_ACCESS_KEY
- name: AWS_DEFAULT_REGION
valueFrom:
configMapKeyRef:
name: aws-config
key: AWS_DEFAULT_REGION
- name: AWS_ENDPOINT_URL
valueFrom:
configMapKeyRef:
name: aws-config
key: AWS_ENDPOINT_URL
volumeMounts:
- name: restore
mountPath: /restore
- name: mongo-client
image: mongo
command: ["/bin/sh", "-c"]
args:
- >
while true; do
if [ -f /restore/${FILENAME} ]; then
mongorestore --uri=${CONNECTION_STRING} --db=${TARGET_DBNAME} /restore/${FILENAME}
break
fi
sleep 10
done
env:
- name: TARGET_DBNAME
value: $TARGET_DBNAME
- name: FILENAME
value: $FILENAME
- name: CONNECTION_STRING
valueFrom:
secretKeyRef:
name: mongodb-admin-root
key: connectionString.standardSrv
volumeMounts:
- name: restore
mountPath: /restore
使用以下脚本来执行恢复任务：
#!/bin/bash
export USERNAME=$1
export FILENAME=$2
export TARGET_DBNAME=$3
envsubst < mongodb-restore-from-s3.yaml | kubectl apply -
运行脚本并传入实际参数：
./run-restore.sh "@mystic" "mdb-database-2023-12-30" "newdatabase"
