drbd84-NFS

大猫喵小猫
220 阅读1分钟

#安装drbd

drbd9 支持更多节点，84对内核有最低依赖3.10.0-1160.25.1.el7.x86_64，非裸盘需要清零

  • 主从模式

  • 独立分区元数据 meta-disk external

  • 导入库与安装

rpm -ivh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm
rpm --import /etc/pki/rpm-gpg/*
yum install -y drbd84-utils kmod-drbd84
  • 注意更新切换内核
#当前内核
$ uname -r
#默认启动
$ grub2-editenv list
#如果不是最小依赖，需要重启切换内核，否则之后 初始会 hang 让后
$ reboot -h now
  • 内核过低
read from drbdsetup
: Success
read from drbdsetup
: Success
Command 'drbdmeta 0 v08 /dev/sdc1 external create-md' terminated with exit code 20
  • 驱动
modprobe drbd
echo "modprobe drbd">>/etc/rc.local
chmod +x /etc/rc.d/rc.local
  • 磁盘
#fdisk 的非交互模式 parted if disk >2T using gpt
parted /dev/sdd- mklabel gpt
parted -a optimal /dev/sdd- mkpart primary 0% 2G
#2T磁盘初始同步用了近10h，这里100% 可用适当调整
parted -a optimal /dev/sdd- mkpart primary 2G 100%
parted /dev/sdd p

#notice kernel
partprobe /dev/sdd

#you may need to zero out the first part
#dd if=/dev/zero of=/dev/sdd1-  bs=1M oflag=direct status=progress
#格式化数据分区
mkfs.ext4 /dev/sdd2-
  • 实例配置
#global_common
mv /etc/drbd.d/global_common.conf /etc/drbd.d/global_common.conf.bak
cat  > /etc/drbd.d/global_common.conf <<-EOF
 global {
   usage-count no;
 }
 common {
   net {
     protocol C;
   }
 }
EOF

#drbd.res
cat  > /etc/drbd.d/drbd.res <<-EOF
resource drbd{
  disk /dev/sdd2;
  device /dev/drbd0;
  meta-disk /dev/sdd1[0];
    on node05 {
      address     172.16.11.1:7788;
    }
    on node04 {
      address     172.16.11.2:7788;
    }
}
EOF
  • 始化设备
#初始化设备元文件
drbdadm create-md drbd
#启动服务
iptables -I INPUT -p tcp --dport 7788 -m comment --comment "drbd" -j ACCEPT
systemctl start drbd 
#激活-主节点执行即可
drbdadm up all  或者 drbdadm up drbd

#忽略systemctl start drbd  已自动激活
Device '0' is configured!
Command 'drbdmeta 0 v08 /dev/sdb internal apply-al' terminated with exit code 20

#首次，数据不一致，主覆盖从令数据一致（自动设为主盘）
drbdadm -- --overwrite-data-of-peer primary drbd
#或 强制设为主盘
drbdadm primary all --force 或者 drbdadm primary mysql_data --force 
#擦看同步状态-速度比较慢
cat /proc/drbd
  • 挂载-验证
#mount提示错误”mount: block device /dev/drbd0 is write-protected, mounting read-only mount: Wrong medium type“,是因为mount只能在Primary一端使用。

mkdir /ha-data
mount /dev/drbd0 /ha-data
touch /ha-data/file{1..3}
#主节点切换从节点
umount /ha-data
drbdadm secondary drbd
#从节点升级主节点
drbdadm primary drbd
mount /dev/drbd0 /ha-data
ls -l  /ha-data
  • NFS-Keepalive
yum install -y nfs-utils nfs4-acl-tools

#To allow for faster failover, you may decrease the grace period

# permanent change
vi /etc/nfs.conf
#uncomment
[nfsd]
 grace-time=10
 lease-time=10

# ephemeral change
# service nfs stop
# echo 10 > /proc/sys/fs/nfs/nlm_grace_period
# echo 10 > /proc/fs/nfsd/nfsv4gracetime
# echo 10 > /proc/fs/nfsd/nfsv4leasetime
# service nfs start

systemctl start rpcbind

mkdir -p /srv/nfs/ha-data
mount /dev/drbd0 /srv/nfs/ha-data

#NFSv4 virtual filesystem root
/srv/nfs 172.16.1.0/24(rw,fsid=0,crossmnt,insecure,no_subtree_check,async,no_root_squash)
/srv/nfs/ha-data/zookeeper1 172.16.1.0/24(rw,nohide,insecure,no_subtree_check,async,no_root_squash)
/srv/nfs/ha-data/zookeeper2 172.16.1.0/24(rw,nohide,insecure,no_subtree_check,async,no_root_squash)
/srv/nfs/ha-data/zookeeper3 172.16.1.0/24(rw,nohide,insecure,no_subtree_check,async,no_root_squash)

 systemctl start nfs

#挂载
mount -t nfs4 172.16.1.155:/  /swarm-store
mount -t nfs 172.16.1.155:/srv/nfs/ha-data  /swarm-store

#自动挂载
echo '172.16.1.155:/   /swarm-store      nfs4     auto     0       0' >> /etc/fstab
#To force systemd to reload the changed /etc/fstab file content, run
$ sudo systemctl daemon-reload
#To, further, make systemd auto remount any new entries, do
$ sudo systemctl restart remote-fs.target local-fs.target 

#卸载
umount -l /ha-swarm
 
#检查
ps -C nfsd --no-header | wc -l # 查看nfs服务状态
ip a |grep 1.155 # 查看是否vip
ps -C keepalived --no-header | wc -l # 查看keepalived状态
drbdadm role drbd # 查看drbd主备状态
mount -l |grep drbd0  # 查看挂载情况

#vrrp 监听 - 正常只有主节点发生vrrp消息 tcpdump -i eth0 vrrp -n

#开启vrrp 协议 #主备都运行下面的命令 firewall-cmd --direct --permanent --add-rule ipv4 filter INPUT 0 --in-interface em1 --destination 224.0.0.18 --protocol vrrp -j ACCEPT firewall-cmd --reload

#如果使用的防火墙是iptables  #master iptables  -A INPUT -i em1 -p vrrp -s 192.168.1.32  -j ACCEPT  #(192.168.1.32 从机IP) #backup iptables  -A INPUT -i em1 -p vrrp -s 192.168.1.33 -j ACCEPT  #(192.168.1.33 主机IP)

  • 计算元数据分区大小脚本 metacalc.sh

Eg. metacalc.sh /dev/vg1/root

#!/bin/bash

which bc >/dev/null 2>&1
if [ ! $? -eq 0 ]; then
    echo "Error: bc is not installed"
    exit 1
fi

if [ $# -lt 1 ]; then
    echo "Error: Please supply block device path"
    echo "Eg. /dev/vg1/backups"
    exit 1
fi

DEVICE=$1

SECTOR_SIZE=$( blockdev --getss $DEVICE )
SECTORS=$( blockdev --getsz $DEVICE )
MD_SIZE=$( echo "((($SECTORS + (2^18)-1) / 262144 * 8) + 72)" | bc )
FS_SIZE=$( echo "$SECTORS - $MD_SIZE" | bc )

MD_SIZE_MB=$( echo "($MD_SIZE / 4 / $SECTOR_SIZE) + 1" | bc )
FS_SIZE_MB=$( echo "($FS_SIZE / 4 / $SECTOR_SIZE)" | bc )

echo "Filesystem: $FS_SIZE_MB MiB"
echo "Filesystem: $FS_SIZE Sectors"
echo "Meta Data:  $MD_SIZE_MB MiB"
echo "Meta Data:  $MD_SIZE Sectors"
echo "--"
echo "Resize commands: resize2fs -p "$DEVICE $FS_SIZE_MB"M; drbdadm create-md res"
avatar
文章
阅读
粉丝