kubernetes glusterfs动态存储

节点信息

[root@kube k8s]# kubectl get nodes
NAME    STATUS   ROLES                  AGE   VERSION
kube    Ready    control-plane,master   9h    v1.23.5
node1   Ready    <none>                 9h    v1.23.5

glusterfs 安装

• 修改hosts

192.168.137.99  kube
192.168.137.98  node1

• ssh免密登录

#设置heketi免密访问GlusterFS
[root@kube heketi]# ssh-keygen -t rsa -q -f /etc/heketi/heketi_key -N ""

# heketi服务由heketi用户启动，heketi用户需要有新生成key的读赋权，否则服务无法启动 [root@kube heketi]# chown heketi:heketi /etc/heketi/heketi_key

#分发公钥
[root@kube heketi]# ssh-copy-id -i /etc/heketi/heketi_key.pub root@node1

• 安装

#所有存储服务器下载安装glusterfs
[root@kube k8s]# yum install centos-release-gluster -y
[root@kube k8s]# yum install glusterfs-server -y

#启动
[root@kube k8s]# systemctl  start glusterd 
[root@kube k8s]# systemctl  enable glusterd 

#建立集群
[root@kube k8s]# gluster peer probe node1

#查看
[root@kube k8s]# gluster peer status

Number of Peers: 1
Hostname: node1
Uuid: fa63e152-382c-4134-9ec5-813a4b06d60b
State: Peer in Cluster (Connected)

• 安装heketi

[root@kube k8s]# yum install -y heketi heketi-client

• 配置heketi

[root@kube k8s]# cat /etc/heketi/heketi.json
{
  "_port_comment": "Heketi Server Port Number",
  "port": "8080",

  "_use_auth": "Enable JWT authorization. Please enable for deployment",
  "use_auth": false,

  "_jwt": "Private keys for access",
  "jwt": {
    "_admin": "Admin has access to all APIs",
    "admin": {
      "key": "feng"
    },
    "_user": "User only has access to /volumes endpoint",
    "user": {
      "key": "feng"
    }
  },

  "_glusterfs_comment": "GlusterFS Configuration",
  "glusterfs": {
    "_executor_comment": [
      "Execute plugin. Possible choices: mock, ssh",
      "mock: This setting is used for testing and development.",
      "      It will not send commands to any node.",
      "ssh:  This setting will notify Heketi to ssh to the nodes.",
      "      It will need the values in sshexec to be configured.",
      "kubernetes: Communicate with GlusterFS containers over",
      "            Kubernetes exec api."
    ],
    
    "executor": "ssh",
    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/etc/heketi/heketi_key",
      "user": "root"
      "port": "22",
      "fstab": "/etc/fstab"
    },

    "_kubeexec_comment": "Kubernetes configuration",
    "kubeexec": {
      "host" :"https://kubernetes.host:8443",
      "cert" : "/path/to/crt.file",
      "insecure": false,
      "user": "kubernetes username",
      "password": "password for kubernetes user",
      "namespace": "OpenShift project or Kubernetes namespace",
      "fstab": "Optional: Specify fstab file on node.  Default is /etc/fstab"
    },

    "_db_comment": "Database file name",
    "db": "/var/lib/heketi/heketi.db",

    "_loglevel_comment": [
      "Set log level. Choices are:",
      "  none, critical, error, warning, info, debug",
      "Default is warning"
    ],
    "loglevel" : "warning"
  }
}

# 修改这里
"executor": "ssh",
    "_sshexec_comment": "SSH username and private key file information",
    "sshexec": {
      "keyfile": "/etc/heketi/heketi_key",
      "user": "root"
      "port": "22",
      "fstab": "/etc/fstab"
    }

记住这里的admin密码key
"executor": "ssh"

• 启动heketi

[root@kube k8s]# systemctl enable heketi 
[root@kube k8s]# systemctl restart heketi 
[root@kube k8s]# systemctl status heketi

• 为 Heketi 创建拓扑配置文件

[root@kube k8s]# cat /etc/heketi/topology.json 
   {
    "clusters": [
       {
         "nodes": [
           {
             "node": {
               "hostnames": {
                 "manage": [
                   "192.168.137.99" 
                ],
                "storage": [
                  "192.168.137.99" 
                ]
              },
              "zone": 1
            },
            "devices": [
              "/dev/sda" 
            ]
          },
          {
            "node": {
              "hostnames": {
                "manage": [
                  "192.168.137.98" 
                ],
                "storage": [
                  "192.168.137.98"
                ]
              },
              "zone": 1
            },
            "devices": [
              "/dev/sda" 
            ]
          }
        ]
      }
    ]
  }

• 加载 Heketi JSON 文件

[root@kube k8s]# export HEKETI_CLI_SERVER=http://localhost:8080
[root@kube k8s]# heketi-cli topology load --json=/etc/heketi/topology.json
Creating cluster ... ID: 2d9e11adede04fe6d07cb81c5a1a7ea4
    Allowing file volumes on cluster.
    Allowing block volumes on cluster.
    Creating node 192.168.0.2 ... ID: 0a9f240ab6fd96ea014948c5605be675
        Adding device /dev/vdd ... OK
    Creating node 192.168.0.3 ... ID: 2468086cadfee8ef9f48bc15db81c88a
        Adding device /dev/vdd ... OK
    Creating node 192.168.0.4 ... ID: 4c21b33d5c32029f5b7dc6406977ec34
        Adding device /dev/vdd ... OK

记住 cluster id 2d9e11adede04fe6d07cb81c5a1a7ea4

k8s 应用glusterfs-secret.yaml

[root@kube k8s]# kubectl apply -f glusterfs-secret.yaml

apiVersion: v1
kind: Secret
metadata:
  name: heketi-secret
  namespace: default
data:
  # base64 encoded password. E.g.: echo -n "feng" | base64
  key: ZmVuZw==
type: kubernetes.io/glusterfs

k8s 应用glusterfs-storageclass.yaml

[root@kube k8s]# kubectl apply -f glusterfs-storageclass.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: gluster-storageclass
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: kubernetes.io/glusterfs
parameters:
  resturl: "http://kube:8080"
  clusterid: "0a1c043857fb331eb287d9615caa799e"
  restuser: "admin"
  secretNamespace: "default"
  secretName: "heketi-secret"
  gidMin: "40000"
  gidMax: "50000"
  volumetype: "replicate:2"
  volumeoptions: "client.ssl on, server.ssl on"
  volumenameprefix: "dept-dev"
  snapfactor: "10"
  customepnameprefix: "dbstorage"

参考

• csdn