配置网卡

root@deploy:~# cat /etc/netplan/00-installer-config.yaml 
# This is the network config written by 'subiquity'
network:
  renderer: networkd
  ethernets:
    ens33:
            #dhcp4: true
      dhcp4: no
      addresses: [172.31.6.109/24]
      gateway4: 172.31.6.2
      nameservers:
              addresses: [114.114.114.114]
      optional: true
    ens37:
            #dhcp4: true
      dhcp4: no
      addresses: [192.168.209.109/24]
      optional: true
  version: 2

时间同步

apt install ntpdate

sudo timedatectl set-timezone Asia/Shanghai
sudo ntpdate time1.cloud.tencent.com
sudo vim /etc/systemd/timesyncd.conf
[Time]
NTP=time1.cloud.tencent.com

timedatectl set-ntp true
sudo systemctl enable systemd-timesyncd
sudo systemctl restart systemd-timesyncd

支持https镜像仓库源

apt install -y apt-transport-https ca-certificates curl software-properties-common

配置清华镜像仓库（所有节点）

wget -q -O- 'https://download.ceph.com/keys/release.asc' |sudo apt-key add -
apt-add-repository 'deb https://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific/ bionic main'
sudo apt update

groupadd cephadmin -g 2088 && useradd -r -m -s /bin/bash -u 2088 -g 2088 cephadmin && echo cephadmin:123456|chpasswd

安装ceph-deploy（deploy节点操作）

wget https://bootstrap.pypa.io/pip/2.7/get-pip.py --no-check-certificate
python2 get-pip.py
pip2 install ceph-deploy

初始化mon节点（deploy节点操作）

ceph-deploy new --cluster-network 192.168.209.0/24 --public-network 172.31.6.0/24 mon1

初始化osd节点（deploy节点操作）

ceph-deploy install --no-adjust-repos --nogpgcheck osd1 osd2 osd3

安装mon服务(在mon节点操作)

apt-cache madison ceph-mon
apt install ceph-mon

ceph集群添加ceph-mon服务（在deploy节点操作）

pwd 
./ceph-cluster
ceph-deploy mon create-initial

安装ceph管理命令（deploy和mon节点操作）

apt install ceph-common -y

发布集群配置文件（osd和deploy节点操作）

ceph-deploy admin osd1 osd2 osd3 deploy
setfacl -m u:cephadmin:rw /etc/ceph/ceph.client.admin.keyring

安装mgr服务(在mgr节点操作)

apt-cache madison ceph-mgr
apt install ceph-mgr

部署ceph-mgr节点(deploy节点操作)

ceph-deploy mgr create mgr1

去掉ceph告警mon is allowing insecure global_id reclaim

ceph config set mon auth_allow_insecure_global_id_reclaim false

初始化存储节点

#osd节点安装运行环境
ceph-deploy install --release pacific osd1
ceph-deploy install --release pacific osd2
ceph-deploy install --release pacific osd3

#列出ceph node节点磁盘
ceph-deploy disk list osd1
ceph-deploy disk list osd2
ceph-deploy disk list osd3

#擦除ceph数据盘
ceph-deploy disk zap osd1 /dev/nvme0n1
ceph-deploy disk zap osd1 /dev/nvme0n2
ceph-deploy disk zap osd1 /dev/nvme0n3
ceph-deploy disk zap osd1 /dev/nvme0n4

ceph-deploy disk zap osd2 /dev/nvme0n1
ceph-deploy disk zap osd2 /dev/nvme0n2
ceph-deploy disk zap osd2 /dev/nvme0n3
ceph-deploy disk zap osd2 /dev/nvme0n4

ceph-deploy disk zap osd3 /dev/nvme0n1
ceph-deploy disk zap osd3 /dev/nvme0n2
ceph-deploy disk zap osd3 /dev/nvme0n3
ceph-deploy disk zap osd3 /dev/nvme0n4

添加osd

ceph-deploy osd create osd1 --data /dev/nvme0n1
ceph-deploy osd create osd1 --data /dev/nvme0n2
ceph-deploy osd create osd1 --data /dev/nvme0n3
ceph-deploy osd create osd1 --data /dev/nvme0n4

ceph-deploy osd create osd2 --data /dev/nvme0n1
ceph-deploy osd create osd2 --data /dev/nvme0n2
ceph-deploy osd create osd2 --data /dev/nvme0n3
ceph-deploy osd create osd2 --data /dev/nvme0n4

ceph-deploy osd create osd3 --data /dev/nvme0n1
ceph-deploy osd create osd3 --data /dev/nvme0n2
ceph-deploy osd create osd3 --data /dev/nvme0n3
ceph-deploy osd create osd3 --data /dev/nvme0n4

移除osd

1.停用设备
ceph osd out {osd-num}
2.停止进程
systemctl stop ceph-osd@{osd-num}
3.移除设备
ceph osd purge {id} --yes-i-really-mean-it

基本命令

查看osd树
ceph osd tree
创建pool
ceph osd pool create mypool 32 32
查看创建的pool的详细信息，pg数量和组合明细
ceph pg ls-by-pool mypool |awk '{print $1,$2,$15}'
查看pool
ceph osd pool ls
将文件放入pool中
rados put msg1 /var/log/syslog --pool=mypool
查看pool中的内容
rados ls --pool=mypool
查看pool中文件msg1存储的位置
ceph osd map mypool msg1
将pool中文件下载到电脑
rados get msg1 --pool=mypool /tmp/a.txt
删除pool中文件
rados rm msg1 --pool=mypool

ceph集群高可用

#mon2,mon3节点
apt install ceph-mon

#deploy节点
ceph-deploy mon add mon2
ceph-deploy mon add mon3

#验证mon节点状态
ceph quorum_status --format json-pretty

#时钟不对齐报错解决方法
vi ceph.conf
mon clock drift allowed = 2
mon clock drift warn backoff = 30
#推送配置
ceph-deploy --overwrite-conf config push mon1
ceph-deploy --overwrite-conf config push mon2
ceph-deploy --overwrite-conf config push mon3
#重启服务
systemctl restart ceph-mon@mon1.service
systemctl restart ceph-mon@mon2.service
systemctl restart ceph-mon@mon3.service

#扩展mgr节点
#mgr2节点
apt install ceph-mgr

#deploy节点
ceph-deploy mgr create mgr2
ceph-deploy admin ceph-mgr2

块存储（RBD）

创建RBD

1.创建存储池
ceph osd pool create poolname pg_num pgp_num
ceph osd pool create myrbd1 32 32
2.对存储池启用RBD
ceph osd pool application enable myrbd1 rbd
3.通过rbd命令对存储池初始化
rbd pool init -p myrbd1
4.创建并验证img
rbd create myimg1 --size 5G --pool myrbd1
rbd create myimg2 --size 3G --pool myrbd1 --image-format 2 --image-feature layering
#列出指定pool中所有img
rbd ls --pool myrbd1
#查看指定rbd的信息
rbd --image myimg1 --pool myrbd1 info
#当前ceph状态
ceph df

客户端使用块存储

1.安装ceph-common
apt install ceph-common -y
2.从deploy服务器同步认证文件
scp ceph.conf ceph.client.admin.keyring root@172.31.6.110:/etc/ceph
3.客户端映射img
rbd -p myrbd1 map myimg1
#验证rbd
lsblk
fdisk -l
4.格式化磁盘并挂载
mkfs.xfs /dev/xxx
mkdir /data
mount /dev/xxx /data
#验证数据写入
dd if=/dev/zero of=/data/ceph-test-file bs=1MB count=100
#ceph验证数据
ceph df
#删除数据并验证
rm -rf ceph-test-file
cdph df
#立即回收空间
fstrim -v /data
#或者挂载添加选项
mount -t xfs -o discard /dev/xxx /data

对象存储（RGW）

部署radosgw服务

#mgr节点操作
apt-cache madison radosgw
apt install radosgw

#deploy节点操作
ceph-deploy --overwrite-conf rgw create mgr1

#验证
ps -ef|grep radosgw
curl xxx.xx.xx.xx:7480
ceph -s
ceph osd pool ls

ceph-fs文件存储

部署MDS服务

#mgr节点操作
apt-cache madison ceph-mds
apt install ceph-mds=16.x.x-1bionic

#deploy节点操作
ceph-deploy mds create mgr1

#验证mds服务
ceph mds stat

创建cephfs metadata和data存储池

#保存matadata的pool
ceph osd pool create cephfs-metadata 32 32
#保存数据的pool
ceph osd pool create cephfs-data 64 64

创建cephfs并验证

ceph fs new mycephfs cephfs-metadata cephfs-data
ceph fs ls
ceph fs status mycephfs
ceph mds stat

客户端挂载cephfs

1.查看keyring
cat ceph.client.admin.keyring
2.挂载
mount -t ceph x.x.x.x:6789:/ /mnt -o name=admin,secret=xxxxxxxxxxx==
3.验证
df -Th
dd if=/dev/zero of=/mnt/ceph-test-file bs=1MB count=100
ceph df

命令总结

1.显示存储池
ceph osd pool ls
ceph osd lspools

2.查看pg状态
ceph pg stat

3.查看指定pool或所有的pool的状态
ceph osd pool stats poolname

4.查看集群存储状态
ceph df
ceph df detail

5.查看osd状态
ceph osd stat

6.显示osd的底层详细信息
ceph osd dump

7.显示osd和节点的对应关系
ceph osd tree
#查看磁盘序号
ll /var/lib/ceph/osd/ceph-13/block
#找出对应的磁盘
lsblk

8.查看mon节点状态
ceph mon stat
ceph mon dump