@[TOC](Ubuntu20.04部署ceph16 pacific)
1 环境介绍与配置
1.1 ceph介绍
#ceph架构
#ceph支持的三种接口：
1 Object：有原生的API，而且也兼容Swift和S3的API。
2 Block：支持精简配置、快照、克隆。
3 File：Posix接口，支持快照。
#ceph的三种类型的优缺点：
#网络架构
1.2 环境介绍
Ubuntu 20.04.2 LTS (GNU/Linux 5.4.0-65-generic x86_64)
#ceph
nautilus 14.2.10-0.el7
#网络设计
192.168.3.0/24 #Cluster Network
192.168.1.0/24 #Public Network
#每台ceph节点下除系统盘外，挂2块硬盘
ceph009 eno3:192.168.3.9 eno5:192.168.1.9
ceph010 eno3:192.168.3.10 eno5:192.168.1.10
ceph011 eno3:192.168.3.11 eno5:192.168.1.11
1.3 基础环境准备
1.3.1 关闭selinux、防火墙
#关闭防火墙
sudo ufw status
sudo ufw disable
#关闭selinux,最小化安装是没有selinux
1.3.2 设置主机名，每台设置
hostnamectl set-hostname ceph009
su -
1.3.3 设置网卡对应网卡IP（自行更改IP）
1.3.4 在hosts里添加对应的ceph节点信息
#vim /etc/hosts
#[ceph16]
192.168.1.9 ceph009
192.168.1.10 ceph010
192.168.1.11 ceph011
1.3.5 添加Ceph pacific版本源
#系统源更换成ustc源，并更新文件缓存
wget -q -O- 'http://mirrors.ustc.edu.cn/ceph/keys/release.asc' | sudo apt-key add -
echo deb http://mirrors.ustc.edu.cn/ceph/debian-pacific/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
apt-get update
1.3.6 时间同步
#个人比较喜欢用以下方式来同步时间
#修改时区
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
ntpdate ntp3.aliyun.com
echo "*/3 * * * * ntpdate ntp3.aliyun.com &> /dev/null" > /tmp/crontab
crontab /tmp/crontab
##3# 1.3.7（可选）安装基础软件
apt install net-tools wget vim bash-completion lrzsz unzip zip ntpdate -y
2 ceph安装与配置
2.1 ceph-deploy工具部署
#如果是在16版本，支持使用cephadm工具部署，ceph-deploy在14版本前都支持 #Ceph-deploy必须作为具有无密码sudo特权的用户登录到Ceph节点，因为它需要在不提示密码的情况下安装软件和配置文件 #启用没有密码的ssh，在ceph009执行即可，三次回车！
root@ceph009:~# ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/root/.ssh/id_rsa):
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
Your identification has been saved in /root/.ssh/id_rsa
Your public key has been saved in /root/.ssh/id_rsa.pub
The key fingerprint is:
SHA256:cLdsgNQ2JFQFsFATaw/M6G8Sbs0voGJAsdybeUbg+ek root@ceph009
The key's randomart image is:
+---[RSA 3072]----+
| .oO*+o. |
| . . * *+ |
|. = + X.o.. |
| + = o = + . |
|. O . S + |
|. =.X . |
|. .B.= |
|.... E.. |
|.. .. |
+----[SHA256]-----+
#将密钥复制到每个Ceph节点
ssh-copy-id -o StrictHostKeyChecking=no root@ceph009
ssh-copy-id -o StrictHostKeyChecking=no root@ceph010
ssh-copy-id -o StrictHostKeyChecking=no root@ceph011
#验证，不需要输入密码即为成功
ssh root@ceph010
2.2 创建并配置ceph集群
2.2.1 创建ceph配置目录和集群
#创建集群目录，用于维护ceph-deploy为集群生成的配置文件和密钥。
mkdir ~/cephcluster && cd ~/cephcluster
root@ceph009:~/cephcluster# pwd
/root/cephcluster
#==注意点== #ceph-deploy会将文件输出到当前目录。如果在==执行ceph-deploy时，一定要确保您在这个目录中==。
==#附：节点清理 #如果在任何时候你遇到麻烦，你想重新开始，执行以下清除Ceph包，并擦除其所有数据和配置==:
ceph-deploy forgetkeys
ceph-deploy purge ceph009 ceph010 ceph011
ceph-deploy purgedata ceph009 ceph010 ceph011
rm ceph.*
#安装ceph-deploy
apt-get install python3 python3-pip -y
#二选一，第一个国内加速下载地址，第二个为原地址慢
git clone https://github.com.cnpmjs.org/ceph/ceph-deploy.git
git clone https://github.com/ceph/ceph-deploy.git
cd ceph-deploy
#如果要使用ceph-deploy直接部署ceph那么需要修改以下这个文件
#vim /root/cephcluster/ceph-deploy/build/lib/ceph_deploy/install.py
#将args.release = 'nautilus' 修改成 args.release = 'pacific'
pip3 install setuptools
python3 setup.py install
#在cephcluster目录下执行创建集群
root@ceph009:~/cephcluster# ceph-deploy new ceph009 ceph010 ceph011
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.1.0): /usr/local/bin/ceph-deploy new ceph009 ceph010 ceph011
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] mon : ['ceph009', 'ceph010', 'ceph011']
[ceph_deploy.cli][INFO ] ssh_copykey : True
[ceph_deploy.cli][INFO ] fsid : None
[ceph_deploy.cli][INFO ] cluster_network : None
[ceph_deploy.cli][INFO ] public_network : None
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7f8934e6cd30>
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] func : <function new at 0x7f8934e63430>
[ceph_deploy.new][DEBUG ] Creating new cluster named ceph
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph009][DEBUG ] connected to host: ceph009
[ceph009][INFO ] Running command: /bin/ip link show
[ceph009][INFO ] Running command: /bin/ip addr show
[ceph009][DEBUG ] IP addresses found: ['192.168.1.9', '192.168.3.9']
[ceph_deploy.new][DEBUG ] Resolving host ceph009
[ceph_deploy.new][DEBUG ] Monitor ceph009 at 192.168.1.9
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph010][DEBUG ] connected to host: ceph009
[ceph010][INFO ] Running command: ssh -CT -o BatchMode=yes ceph010 true
[ceph010][DEBUG ] connected to host: ceph010
[ceph010][INFO ] Running command: /bin/ip link show
[ceph010][INFO ] Running command: /bin/ip addr show
[ceph010][DEBUG ] IP addresses found: ['192.168.3.10', '192.168.1.10']
[ceph_deploy.new][DEBUG ] Resolving host ceph010
[ceph_deploy.new][DEBUG ] Monitor ceph010 at 192.168.1.10
[ceph_deploy.new][INFO ] making sure passwordless SSH succeeds
[ceph011][DEBUG ] connected to host: ceph009
[ceph011][INFO ] Running command: ssh -CT -o BatchMode=yes ceph011 true
[ceph011][DEBUG ] connected to host: ceph011
[ceph011][INFO ] Running command: /bin/ip link show
[ceph011][INFO ] Running command: /bin/ip addr show
[ceph011][DEBUG ] IP addresses found: ['192.168.1.11', '192.168.3.11']
[ceph_deploy.new][DEBUG ] Resolving host ceph011
[ceph_deploy.new][DEBUG ] Monitor ceph011 at 192.168.1.11
[ceph_deploy.new][DEBUG ] Monitor initial members are ['ceph009', 'ceph010', 'ceph011']
[ceph_deploy.new][DEBUG ] Monitor addrs are ['192.168.1.9', '192.168.1.10', '192.168.1.11']
[ceph_deploy.new][DEBUG ] Creating a random mon key...
[ceph_deploy.new][DEBUG ] Writing monitor keyring to ceph.mon.keyring...
[ceph_deploy.new][DEBUG ] Writing initial config to ceph.conf...
2.2.2 ceph配置文件修改
#创建公共访问的网络 #vim /root/cephcluster/ceph.conf
[global]
fsid = 76235629-6feb-4f0c-a106-4be33d485535
mon_initial_members = ceph131, ceph132, ceph133
mon_host = 172.16.1.131,172.16.1.132,172.16.1.133
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public_network = 192.168.1.0/24
cluster_network = 192.168.3.0/24
#设置副本数
osd_pool_default_size = 1
#设置最小副本数
osd_pool_default_min_size = 2
#设置时钟偏移0.5s
mon_clock_drift_allowed = .50
2.2.3 各节点安装基础包
#安装ceph相关包，每台执行
apt-get install -y ceph ceph-osd ceph-mds ceph-mon radosgw
2.2.4 部署初始mon并生成密钥:
#初始化mon，执行过程无报错即可
ceph-deploy mon create-initial
#这时候目录底下应该会多以下的key
root@ceph009:~/cephcluster# ll
total 308
drwxr-xr-x 3 root root 4096 Apr 15 15:55 ./
drwx------ 6 root root 4096 Apr 15 15:53 ../
-rw------- 1 root root 113 Apr 15 15:55 ceph.bootstrap-mds.keyring
-rw------- 1 root root 113 Apr 15 15:55 ceph.bootstrap-mgr.keyring
-rw------- 1 root root 113 Apr 15 15:55 ceph.bootstrap-osd.keyring
-rw------- 1 root root 113 Apr 15 15:55 ceph.bootstrap-rgw.keyring
-rw------- 1 root root 151 Apr 15 15:55 ceph.client.admin.keyring
-rw-r--r-- 1 root root 454 Apr 15 15:53 ceph.conf
-rw-r--r-- 1 root root 264849 Apr 15 15:55 ceph-deploy-ceph.log
-rw------- 1 root root 73 Apr 15 15:52 ceph.mon.keyring
#使用Ceph -deploy将配置文件和管理密钥复制到您的管理节点和Ceph节点
root@ceph009:~/cephcluster# ceph-deploy admin ceph009 ceph010 ceph011
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO ] Invoked (2.1.0): /usr/local/bin/ceph-deploy admin ceph009 ceph010 ceph011
[ceph_deploy.cli][INFO ] ceph-deploy options:
[ceph_deploy.cli][INFO ] verbose : False
[ceph_deploy.cli][INFO ] quiet : False
[ceph_deploy.cli][INFO ] username : None
[ceph_deploy.cli][INFO ] overwrite_conf : False
[ceph_deploy.cli][INFO ] ceph_conf : None
[ceph_deploy.cli][INFO ] cluster : ceph
[ceph_deploy.cli][INFO ] client : ['ceph009', 'ceph010', 'ceph011']
[ceph_deploy.cli][INFO ] cd_conf : <ceph_deploy.conf.cephdeploy.Conf object at 0x7fb850c66b20>
[ceph_deploy.cli][INFO ] default_release : False
[ceph_deploy.cli][INFO ] func : <function admin at 0x7fb8512a6dc0>
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph009
[ceph009][DEBUG ] connected to host: ceph009
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph010
[ceph010][DEBUG ] connected to host: ceph010
[ceph_deploy.admin][DEBUG ] Pushing admin keys and conf to ceph011
[ceph011][DEBUG ] connected to host: ceph011
2.2.5 部署mgr服务
#创建mgr，执行过程无报错即可
ceph-deploy mgr create ceph009 ceph010 ceph011
#检查状态
root@ceph009:~/cephcluster# systemctl status ceph-mgr@ceph009
● ceph-mgr@ceph009.service - Ceph cluster manager daemon
Loaded: loaded (/lib/systemd/system/ceph-mgr@.service; enabled; vendor preset: enabled)
Active: active (running) since Thu 2021-04-15 16:01:39 CST; 38s ago
Main PID: 30585 (ceph-mgr)
Tasks: 60 (limit: 76600)
Memory: 264.5M
CGroup: /system.slice/system-ceph\x2dmgr.slice/ceph-mgr@ceph009.service
└─30585 /usr/bin/ceph-mgr -f --cluster ceph --id ceph009 --setuser ceph --setgroup ceph
Apr 15 16:01:39 ceph009 systemd[1]: Started Ceph cluster manager daemon.
2.2.6 添加OSD
#创建osd，执行过程无报错即可
ceph-deploy osd create --data /dev/sdb ceph009
ceph-deploy osd create --data /dev/sdb ceph010
ceph-deploy osd create --data /dev/sdb ceph011
2.2.7 验证集群状态
root@ceph009:~/cephcluster# ceph osd tree
ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF
-1 5.58893 root default
-3 1.86298 host ceph009
0 ssd 0.93149 osd.0 up 1.00000 1.00000
4 ssd 0.93149 osd.4 up 1.00000 1.00000
-7 1.86298 host ceph010
2 ssd 0.93149 osd.2 up 1.00000 1.00000
3 ssd 0.93149 osd.3 up 1.00000 1.00000
-5 1.86298 host ceph011
1 ssd 0.93149 osd.1 up 1.00000 1.00000
5 ssd 0.93149 osd.5 up 1.00000 1.00000
3 扩展集群的服务 ==- 请根据需求开启==
==#所有的ceph-deploy操作都必须是使用cephdeploy用户并在配置文件目录底下执行！！！==
[cephdeploy@ceph131 cephcluster]$ pwd
/home/cephdeploy/cephcluster
3.1 添加元数据服务器（mds）
ceph-deploy mds create ceph009
3.2 添加监控端（mon）
ceph-deploy mon add ceph010 ceph011
#一旦您添加了新的Ceph监视器，Ceph将开始同步监视器并形成一个quorum。您可以通过执行以下操作来检查仲裁状态:
ceph quorum_status --format json-pretty
3.3 增加守护进程（mgr）
#Ceph Manager守护进程以活动/备用模式操作。部署其他管理器守护进程可以确保，如果一个守护进程或主机失败，另一个守护进程可以接管，而不会中断服务。
ceph-deploy mgr create ceph011
#验证
ceph -s
3.4 增加对象存储网关（rgw）
ceph-deploy rgw create ceph009 ceph010 ceph011
#默认情况下，RGW实例将监听端口7480。这可以通过在运行RGW的节点上编辑ceph.conf来改变，如下所示:
[client]
rgw frontends = civetweb port=80
#修改完端口需要重启服务
systemctl restart ceph-radosgw.service
4 存储部署
4.1 启用CephFS
4.1.1 请确认至少有一台节点启用了mds服务，==pg数是有算法的，可以使用官网计算器去计算！==
==PG数量的预估 集群中单个池的PG数计算公式如下：PG 总数 = (OSD 数 * 100) / 最大副本数 / 池数 (结果必须舍入到最接近2的N次幂的值)== #在ceph集群，其中一台执行命令即可，这里用ceph132
root@ceph009:~/cephcluster# ceph osd pool create cephfs_data 16
pool 'cephfs_data' created
root@ceph009:~/cephcluster# ceph osd pool create cephfs_metadata 16
pool 'cephfs_metadata' created
root@ceph009:~/cephcluster# ceph fs new cephfs_storage cephfs_metadata cephfs_data
new fs with metadata pool 7 and data pool 6
root@ceph009:~/cephcluster# ceph df
--- RAW STORAGE ---
CLASS SIZE AVAIL USED RAW USED %RAW USED
ssd 5.6 TiB 5.6 TiB 42 MiB 42 MiB 0
TOTAL 5.6 TiB 5.6 TiB 42 MiB 42 MiB 0
--- POOLS ---
POOL ID PGS STORED OBJECTS USED %USED MAX AVAIL
device_health_metrics 1 1 0 B 0 0 B 0 5.3 TiB
.rgw.root 2 32 1.3 KiB 4 16 KiB 0 5.3 TiB
default.rgw.log 3 32 23 KiB 303 640 KiB 0 5.3 TiB
default.rgw.control 4 32 0 B 8 0 B 0 5.3 TiB
default.rgw.meta 5 23 0 B 0 0 B 0 5.3 TiB
cephfs_data 6 16 0 B 0 0 B 0 5.3 TiB
cephfs_metadata 7 16 2.3 KiB 21 32 KiB 0 5.3 TiB
4.1.2 挂载cephfs
#在客户端创建密码文件
[root@ceph133 ~]# cat /etc/ceph/ceph.client.admin.keyring
[client.admin]
key = AQC5e/1eXm9WExAAmlD9aZoc2dZO6jbU8UXSqg==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
[root@ceph133 ~]# vim admin.secret
[root@ceph133 ~]# ll
total 8
-rw-r--r-- 1 root root 41 Jul 2 15:17 admin.secret
#挂载cephfs文件夹，并验证
root@ceph009:~/cephcluster# cat /etc/ceph/ceph.client.admin.keyring
[client.admin]
key = AQB/8XdgXQeoARAAHPkpeH1ep5Myjugm+J52Og==
caps mds = "allow *"
caps mgr = "allow *"
caps mon = "allow *"
caps osd = "allow *"
root@ceph009:~/cephcluster# vim admin.secret
root@ceph009:~/cephcluster# cat admin.secret
AQB/8XdgXQeoARAAHPkpeH1ep5Myjugm+J52Og==
root@ceph009:~/cephcluster# mkdir /mnt/cephfs_storage
root@ceph009:~/cephcluster# mount -t ceph 192.168.1.11:6789:/ /mnt/cephfs_storage -o name=admin,secretfile=admin.secret
root@ceph009:~/cephcluster# df -h |grep mnt
192.168.1.11:6789:/ 5.4T 0 5.4T 0% /mnt/cephfs_storage
4.2 启用块存储
4.2.1 在ceph集群，其中一台执行命令即可，这里用ceph133
#初始化rbd池 ==PG数量的预估 集群中单个池的PG数计算公式如下：PG 总数 = (OSD 数 * 100) / 最大副本数 / 池数 (结果必须舍入到最接近2的N次幂的值)==
root@ceph009:~/cephcluster# ceph osd pool create rbd_storage 16 16 replicated
pool 'rbd_storage' created
#创建一个块设备
root@ceph009:~/cephcluster# rbd create --size 1024 rbd_image -p rbd_storage
root@ceph009:~/cephcluster# rbd ls rbd_storage
rbd_image
#删除命令
rbd rm rbd_storage/rbd_image
4.2.2 挂载rbd块设备
#将块设备映射到系统内核
root@ceph009:~/cephcluster# rbd map rbd_storage/rbd_image
/dev/rbd0
root@ceph009:~/cephcluster# lsblk |grep rbd
rbd0 252:0 0 1G 0 disk
#格式化rbd设备
root@ceph009:~/cephcluster# mkfs.ext4 -m0 /dev/rbd/rbd_storage/rbd_image
mke2fs 1.45.5 (07-Jan-2020)
Discarding device blocks: done
Creating filesystem with 262144 4k blocks and 65536 inodes
Filesystem UUID: c7689555-c39b-43fd-89ed-452f7b679587
Superblock backups stored on blocks:
32768, 98304, 163840, 229376
Allocating group tables: done
Writing inode tables: done
Creating journal (8192 blocks): done
Writing superblocks and filesystem accounting information: done
#挂载rbd设备
root@ceph009:~/cephcluster# mkdir /mnt/rbd_storage
root@ceph009:~/cephcluster# df -h |grep rbd
/dev/rbd0 976M 2.6M 958M 1% /mnt/rbd_storage
#取消内核挂载
rbd unmap /dev/rbd0
4.3 启用RGW对象存储
4.3.1 请确认至少有一台节点启用了rgw服务
#启用rgw见3.4
#使用浏览器，查看 http://192.168.1.9:7480/，说明已经启用成功
5 启用ceph dashboard
5.1 启用dashboard
#Ceph仪表板是一个内置的基于web的Ceph管理和监视应用程序，用于管理集群。 #节点默认没有安装mgr-dashboard程序，因此要先安装
apt install ceph-mgr-dashboard -y
#任一节点启用dashboard，没有提示
root@ceph009:~/cephcluster# ceph mgr module enable dashboard
#配置登陆认证
root@ceph009:~/cephcluster# ceph dashboard create-self-signed-cert
Self-signed certificate created
#配置登陆账户 #由于16版本创建用户有变化，需要先创建密码文件，这里密码用admin.123 #vim passwd
admin.123
#用户名为admin，加入administrator组，-i 后面跟着前面创建的密码文件
root@ceph009:~/cephcluster# ceph dashboard ac-user-create admin administrator -i passwd
{"username": "admin", "password": "$2b$12$DnfJoDkorvLZRMl730oX3eCEZ1FCwVce8Kfk4F.30LxnBX9jmoZES", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1618482185, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": false}
#测试登陆，浏览器查看：https://192.168.1.9:8443/ 用户名：admin 密码：admin.123
root@ceph009:~/cephcluster# ceph mgr services
{
"dashboard": "https://ceph010:8443/"
}
5.2 Dashboard中启用RGW
#创建rgw的用户
root@ceph009:~/cephcluster# radosgw-admin user create --uid=rgwadmin --display-name=rgwadmin --system
{
"user_id": "rgwadmin",
"display_name": "rgwadmin",
"email": "",
"suspended": 0,
"max_buckets": 1000,
"subusers": [],
"keys": [
{
"user": "rgwadmin",
"access_key": "QXVBBHJA5IC1UFLMN9KC",
"secret_key": "qmMjZbhOI1mV7zSLN4UcplnbJQ4BnRPNVv9gR6Vo"
}
],
"swift_keys": [],
"caps": [],
"op_mask": "read, write, delete",
"system": "true",
"default_placement": "",
"default_storage_class": "",
"placement_tags": [],
"bucket_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"user_quota": {
"enabled": false,
"check_on_raw": false,
"max_size": -1,
"max_size_kb": 0,
"max_objects": -1
},
"temp_url_keys": [],
"type": "rgw",
"mfa_ids": []
}
#设置凭证，取创建用户生成的key #先创建key文件，再更新
root@ceph009:~/cephcluster# cat rgw-api-access-key
QXVBBHJA5IC1UFLMN9KC
root@ceph009:~/cephcluster# cat rgw-api-secret-key
qmMjZbhOI1mV7zSLN4UcplnbJQ4BnRPNVv9gR6Vo
root@ceph009:~/cephcluster# ceph dashboard set-rgw-api-access-key -i rgw-api-access-key
Option RGW_API_ACCESS_KEY updated
root@ceph009:~/cephcluster# ceph dashboard set-rgw-api-secret-key -i rgw-api-secret-key
Option RGW_API_SECRET_KEY updated
#禁用SSL
root@ceph009:~/cephcluster# ceph dashboard set-rgw-api-ssl-verify False
Option RGW_API_SSL_VERIFY updated
#启用rgw的dashboard
root@ceph010:~# ceph dashboard set-rgw-api-host 192.168.1.10
Option RGW_API_HOST updated
root@ceph010:~# ceph dashboard set-rgw-api-port 7480
Option RGW_API_PORT updated
root@ceph010:~# ceph dashboard set-rgw-api-scheme http
Option RGW_API_SCHEME updated
root@ceph010:~# ceph dashboard set-rgw-api-admin-resource admin
Option RGW_API_ADMIN_RESOURCE updated
root@ceph010:~# ceph dashboard set-rgw-api-user-id rgwadmin
Option RGW_API_USER_ID updated
root@ceph010:~# systemctl restart ceph-radosgw.target
root@ceph010:~# systemctl status ceph-radosgw.target
#至此结束
X.部署过程遇到的问题
eg1.[ceph010][ERROR ] RuntimeError: command returned non-zero exit status: 1
[ceph_deploy.osd][ERROR ] Failed to execute command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
[ceph_deploy][ERROR ] GenericError: Failed to create 1 OSDs
解决方案：
#取消挂载
umount /dev/sdb1 or /dev/sdb2
#取消开机挂载
vim /etc/fstab
common /dev/sdb uuid mount
#格式化
parted -s /dev/sdb mklabel gpt mkpart primary xfs 0% 100%
#重启
reboot
#
mkfs.xfs /dev/sdb -f
#重新执行命令
ceph-deploy osd create --data /dev/sdb ceph010
eg2.bluestore(/var/lib/ceph/osd/ceph-4/block) _read_bdev_label failed to open /var/lib/ceph/osd/ceph-4/block: (1) Operation not permitted
现象：OSD全部启动失败
原因是：Ceph 16.2.0 引入了新的限制
解决方案：
vim /lib/systemd/system/ceph-osd@.service
去掉ProtectClock=true，保存退出
systemctl daemon-reload
systemctl restart ceph-osd@*
即可解决
eg2.[root@ceph131 cephcluster]# ceph-deploy mon create-initial
[ceph131][INFO ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 5
[ceph_deploy.mon][WARNIN] waiting 5 seconds before retrying
[ceph131][INFO ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 4
[ceph_deploy.mon][WARNIN] waiting 10 seconds before retrying
[ceph131][INFO ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 3
[ceph_deploy.mon][WARNIN] waiting 10 seconds before retrying
[ceph131][INFO ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 2
[ceph_deploy.mon][WARNIN] waiting 15 seconds before retrying
[ceph131][INFO ] Running command: ceph --cluster=ceph --admin-daemon /var/run/ceph/ceph-mon.ceph131.asok mon_status
[ceph_deploy.mon][WARNIN] mon.ceph131 monitor is not yet in quorum, tries left: 1
[ceph_deploy.mon][WARNIN] waiting 20 seconds before retrying
[ceph_deploy.mon][ERROR ] Some monitors have still not reached quorum:
[ceph_deploy.mon][ERROR ] ceph131
解决方案：
1 查看ceph.com的配置，public_network 或者mon_host 是否配置正确,
2 查看hosts配置是不是有增加了ipv6的解析，直接去掉即可再重新部署
eg2.[ceph_deploy][ERROR ] IOError: [Errno 13] Permission denied: '/root/cephcluster/ceph-deploy-ceph.log'
解决方案：
[root@ceph131 cephcluster]# echo "cephdeploy ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephdeploy
cephdeploy ALL = (root) NOPASSWD:ALL
[root@ceph131 cephcluster]# chmod 0440 /etc/sudoers.d/cephdeploy
[root@ceph131 cephcluster]# cat /etc/sudoers.d/cephdeploy
cephdeploy ALL = (root) NOPASSWD:ALL
[root@ceph131 cephcluster]# usermod -G root cephdeploy
eg3.[root@ceph131 ~]# ceph -s
cluster:
id: 76235629-6feb-4f0c-a106-4be33d485535
health: HEALTH_WARN
clock skew detected on mon.ceph133
原因是：时钟偏移问题
解决方案：
1 在deploy节点的部署目录底下修改ceph.conf文件
[cephdeploy@ceph131 cephcluster]$ pwd
/home/cephdeploy/cephcluster
2 编辑ceph.conf，并添加以下字段保存退出
#设置时钟偏移
mon clock drift allowed = 2
mon clock drift warn backoff = 30
3 重新下发配置
[cephdeploy@ceph131 cephcluster]$ ceph-deploy --overwrite-conf config push ceph{131..133}
4 至对应节点重启mon
[root@ceph133 ~]# systemctl restart ceph-mon@ceph133.service
eg4.[root@ceph132 ~]# ceph mgr module enable dashboard
Error ENOENT: all mgr daemons do not support module 'dashboard', pass --force to force enablement
原因是：节点没有安装ceph-mgr-dashboard，在mgr的节点上安装。
yum install ceph-mgr-dashboard
eg5.[root@ceph133 ~]# ceph -s
cluster:
id: 76235629-6feb-4f0c-a106-4be33d485535
health: HEALTH_WARN
application not enabled on 1 pool(s)
[root@ceph133 ~]# ceph health detail
HEALTH_WARN application not enabled on 1 pool(s)
POOL_APP_NOT_ENABLED application not enabled on 1 pool(s)
application not enabled on pool 'rbd_storage'
use 'ceph osd pool application enable <pool-name> <app-name>', where <app-name> is 'cephfs', 'rbd', 'rgw', or freeform for custom applications.
解决方案：
ceph osd pool application enable rbd_storage rdb
