ceph提供底层存储功能，cephfs方式支持k8s的pv的3种访问模式ReadWriteOnce，ReadOnlyMany ，ReadWriteMany ，RBD支持ReadWriteOnce，ReadOnlyMany两种模式
下面，实验使用rbd作为statefulset多实例的存储方式：
先创建一个nginx，该nginx使用storageclass声明pvc：
apiVersion: apps/v1
kind: StatefulSet
#kind: Deployment
metadata:
name: my-nginx
spec:
selector:
matchLabels:
app: nginx
serviceName: ngx-service
replicas: 3
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:latest
ports:
- containerPort: 80
volumeMounts:
- name: rbd
mountPath: /usr/share/rbd
volumeClaimTemplates:
- metadata:
name: rbd
spec:
accessModes: [ "ReadWriteOnce" ]
# storageClassName: "rook-ceph-block"
resources:
requests:
storage: 1Gi
# volumes:
# - name: rbd
# persistentVolumeClaim: #指定pvc
# claimName: rbd-pvc
---
apiVersion: v1
kind: Service
metadata:
name: ngx-service
labels:
app: nginx
spec:
type: NodePort
selector:
app: nginx
ports:
- port: 80
targetPort: 80
nodePort: 32500
[root@node1 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-nginx-0 1/1 Running 0 19m
my-nginx-1 1/1 Running 0 18m
my-nginx-2 1/1 Running 0 18m
[root@node1 ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
rbd-my-nginx-0 Bound pvc-bff940d2-1b78-4224-8b62-7b3b05e1951c 1Gi RWO rook-ceph-block 35m
rbd-my-nginx-1 Bound pvc-749ab45c-c3a7-4ce1-8d06-2d7b976c97a0 1Gi RWO rook-ceph-block 33m
rbd-my-nginx-2 Bound pvc-b0d9f1ec-256a-465f-a34f-47f3f03c4051 1Gi RWO rook-ceph-block 31m
[root@node1 ~]# kubectl exec -it pod/my-nginx-1 -- /bin/bash
root@my-nginx-1:/# cd /usr/share/rbd
root@my-nginx-1:/usr/share/rbd# ls
lost+found
root@my-nginx-1:/usr/share/rbd# echo "caoyong" > caoyong.txt
root@my-nginx-1:/usr/share/rbd# cat caoyong.txt
caoyong
root@my-nginx-1:/usr/share/rbd# exit
[root@node1 ~]# kubectl get statefulset
NAME READY AGE
my-nginx 3/3 14m
[root@node1 ~]# kubectl delete statefulset my-nginx
statefulset.apps "my-nginx" deleted
#statefulset删除以后，pod均被删除，pvc没有被删除
[root@node1 ~]# kubectl get statefulset
No resources found in default namespace.
[root@node1 ~]# kubectl get pods
No resources found in default namespace.
[root@node1 ~]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
rbd-my-nginx-0 Bound pvc-bff940d2-1b78-4224-8b62-7b3b05e1951c 1Gi RWO rook-ceph-block 14m
rbd-my-nginx-1 Bound pvc-749ab45c-c3a7-4ce1-8d06-2d7b976c97a0 1Gi RWO rook-ceph-block 13m
rbd-my-nginx-2 Bound pvc-b0d9f1ec-256a-465f-a34f-47f3f03c4051 1Gi RWO rook-ceph-block 11m
#接下来，新建nginx statefulset
[root@node1 ~]# kubectl apply -f nginx.yaml
statefulset.apps/my-nginx created
service/ngx-service unchanged
[root@node1 ~]# kubectl get pods
NAME READY STATUS RESTARTS AGE
my-nginx-0 1/1 Running 0 39s
my-nginx-1 1/1 Running 0 23s
my-nginx-2 1/1 Running 0 9s
[root@node1 ~]# kubectl exec -it pod/my-nginx-1 -- /bin/bash
root@my-nginx-1:/# cd /usr/share/rbd
root@my-nginx-1:/usr/share/rbd# ls
caoyong.txt lost+found
root@my-nginx-1:/usr/share/rbd# cat
caoyong.txt lost+found/
root@my-nginx-1:/usr/share/rbd# cat caoyong.txt
caoyong
root@my-nginx-1:/usr/share/rbd# exit
exit
#可以看到，即便是stagefulset和pod均被删除，pvc中的内容仍然被保留下来
在ceph dashboard上面可以看到三个：
