@[TOC](第三十一章高可用集群KEEPALIVED)

实验：双主Keepalived

实验环境

主机名称网卡 IP/GW
client 桥接 1.16/24 GW1.7
route NAT、桥接 37.7和1.7
ka1 NAT 37.30/24 GW37.7
ka2 NAT 37.40/24 GW37.7
rs1 NAT 37.10/24 GW37.7
rs2 NAT 37.20/24 GW37.7

主机名称	网卡	IP/GW
client	桥接	1.16/24 GW1.7
route	NAT、桥接	37.7和1.7
ka1	NAT	37.30/24 GW37.7
ka2	NAT	37.40/24 GW37.7
rs1	NAT	37.10/24 GW37.7
rs2	NAT	37.20/24 GW37.7

ka1和ka2

[root@ka1 ~]# yum install keepalived httpd -y
[root@ka1 ~]# echo Sorry Server > /var/www/html/index.html
[root@ka1 ~]# systemctl start httpd

[root@ka1 ~]# ssh-keygen

相互ke验证

[root@ka1 ~]# ssh-copy-id 192.168.37.40
和
[root@ka2 ~]# ssh-copy-id 192.168.37.30

ka1

[root@ka1 ~]# vim /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.37.30 ka1    <--
192.168.37.40 ka2    <--

[root@ka1 ~]# scp /etc/hosts ka2:/etc/

[root@ka1 ~]# cd /etc/keepalived/
[root@ka1 keepalived]# ls
keepalived.conf
[root@ka1 keepalived]# cp keepalived.conf{,.bak}
[root@ka1 keepalived]# ls
keepalived.conf  keepalived.conf.bak

[root@ka1 keepalived]# vim keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id ka1
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.37.100/24 dev eth0 label eth0:1
    }
#出了故障触发相应的脚本执行
    notify_master "/etc/keepalived/notify.sh master"   <--
    notify_backup "/etc/keepalived/notify.sh backup"   <--
    notify_fault "/etc/keepalived/notify.sh fault"     <--
}

vrrp_instance VI_2 {
    state BACKUP
    interface eth0
    virtual_router_id 88
    priority 80
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 654321
    }
    virtual_ipaddress {
        192.168.37.200/24 dev eth0 label eth0:2
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

[root@ka1 keepalived]# scp keepalived.conf ka2:/etc/keepalived/

[root@ka1 keepalived]# vim notify.sh
#!/bin/bash
#
contact='root@localhost'
notify() {
	mailsubject="$(hostname) to be $1, vip floating"
	mailbody="$(date +'%F %T'): vrrp transition, $(hostname) changed to be $1"
	echo "$mailbody" | mail -s "$mailsubject" $contact
}
case $1 in
master)
	notify master
	;;
backup)
	notify backup
	;;
fault)
	notify fault
	;;
*)
	echo "Usage: $(basename $0) {master|backup|fault}"
	exit 1
	;;
esac

[root@ka1 keepalived]# chmod +x notify.sh 
[root@ka1 keepalived]# scp notify.sh ka2:/etc/keepalived/
[root@ka1 keepalived]# systemctl start keepalived


#测试一下、看看是否收到邮件
[root@ka1 keepalived]# ./notify.sh master
[root@ka1 keepalived]# mail
Heirloom Mail version 12.5 7/5/10.  Type ? for help.
"/var/spool/mail/root": 1 message 1 new
>N  1 root                  Sun Jul 31 14:37  18/671   "ka1 to be master, vip floating"
& 1
Message  1:
From root@ka1.localdomain  Sun Jul 31 14:37:30 2022
Return-Path: <root@ka1.localdomain>
X-Original-To: root@localhost
Delivered-To: root@localhost.localdomain
Date: Sun, 31 Jul 2022 14:37:30 -0400
To: root@localhost.localdomain
Subject: ka1 to be master, vip floating
User-Agent: Heirloom mailx 12.5 7/5/10
Content-Type: text/plain; charset=us-ascii
From: root@ka1.localdomain (root)
Status: R

2022-07-31 14:37:30: vrrp transition, ka1 changed to be master

& q
Held 1 message in /var/spool/mail/root
You have mail in /var/spool/mail/root

ka2

[root@ka2 ~]# vim /etc/keepalived/keepalived.conf 

! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id ka2    <--名称
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state BACKUP    <--备份
    interface eth0
    virtual_router_id 66
    priority 80    <--优先级
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.37.100/24 dev eth0 label echo0:1
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

vrrp_instance VI_2 {
    state MASTER    <--主
    interface eth0
    virtual_router_id 88        <--
    priority 100    <--优先级
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 654321    <--
    }
    virtual_ipaddress {
        192.168.37.200/24 dev eth0 label eth0:2    <--
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}


[root@ka2 ~]# systemctl start keepalived

ka1

[root@ka1 keepalived]# vim /etc/sysconfig/keepalived 
# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp               -P    Only run with VRRP subsystem.
# --check              -C    Only run with Health-checker subsystem.
# --dont-release-vrrp  -V    Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs  -I    Dont remove IPVS topology on daemon stop.
# --dump-conf          -d    Dump the configuration data.
# --log-detail         -D    Detailed log messages.
# --log-facility       -S    0-7 Set local syslog facility (default=LOG_DAEMON)
#
KEEPALIVED_OPTIONS="-D -S 6"    <--


[root@ka1 keepalived]# scp /etc/sysconfig/keepalived ka2:/etc/sysconfig/

[root@ka1 keepalived]# vim /etc/rsyslog.conf 
...
# Save boot messages also to boot.log
local7.*                                                /var/log/boot.log
local6.*                                                /var/log/keepalived.log    <--
...

[root@ka1 keepalived]# scp /etc/rsyslog.conf ka2:/etc/
[root@ka1 keepalived]# systemctl restart keepalived rsyslog

ka2

[root@ka2 ~]# systemctl restart keepalived rsyslog

邮件设置

配置发送邮件的邮箱设置：

vim ~/.mailrc 或 /etc/mail.rc

set from=clark_0932@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=clark_0932@qq.com
set smtp-auth-password=llrtrpiosdttbiha
set smtp-auth=login
set ssl-verify=ignore

rs2

[root@rs2 ~]# vim .mailrc
set from=clark_0932@qq.com
set smtp=smtp.qq.com
set smtp-auth-user=clark_0932@qq.com
set smtp-auth-password=llrtrpiosdttbiha    <--登陆qq邮箱设置--账户--POP3/IMAP/SMTP/Exchange/CardDAV/CalDAV服务（'POP3/SMTP服务'开启）--生成授权码
set smtp-auth=login
set ssl-verify=ignore

#测试、发邮件后qq邮箱会收到邮件
[root@rs2 ~]# echo test | mail -s linux clark_0932@qq.com

KeepAlived支持IPVS

虚拟服务器：

配置参数：
virtual_server IP port | virtual_server fwmark int
{
   ...
   real_server {
   ...
   }
   ...
}

常用参数

delay_loop <INT>：检查后端服务器的时间间隔

lb_algo rr|wrr|lc|wlc|lblc|sh|dh：定义调度方法

lb_kind NAT|DR|TUN：集群的类型

persistence_timeout <INT>：持久连接时长

protocol TCP：服务协议，仅支持TCP

sorry_server <IPADDR> <PORT>：所有RS故障时，备用服务器地址

real_server <IPADDR> <PORT>
{
   weight <INT> RS权重
   notify_up <STRING>|<QUOTED-STRING> RS上线通知脚本
   notify_down <STRING>|<QUOTED-STRING> RS下线通知脚本
   HTTP_GET|SSL_GET|TCP_CHECK|SMTP_CHECK|MISC_CHEC K { ... }：定义当前主机的健康状态检测方法
}

KeepAlived配置检测

HTTP_GET|SSL_GET：应用层检测

HTTP_GET|SSL_GET {
   url {
       path <URL_PATH>：定义要监控的URL
       status_code <INT>：判断上述检测机制为健康状态的响应码
       digest <STRING>：判断为健康状态的响应的内容的校验码
       }
connect_timeout <INTEGER>：连接请求的超时时长
nb_get_retry <INT>：重试次数
delay_before_retry <INT>：重试之前的延迟时长
connect_ip <IP ADDRESS>：向当前RS哪个IP地址发起健康状态检测请求
connect_port <PORT>：向当前RS的哪个PORT发起健康状态检测请求
bindto <IP ADDRESS>：发出健康状态检测请求时使用的源地址
bind_port <PORT>：发出健康状态检测请求时使用的源端口
}

传输层检测 TCP_CHECK

TCP_CHECK {
   connect_ip <IP ADDRESS>：向当前RS的哪个IP地址发起健康状态检测请求
   connect_port <PORT>：向当前RS的哪个PORT发起健康状态检测请求
   bindto <IP ADDRESS>：发出健康状态检测请求时使用的源地址
   bind_port <PORT>：发出健康状态检测请求时使用的源端口
   connect_timeout <INTEGER>：连接请求的超时时长
}

实验：单主LVS

ka1

[root@ka1 keepalived]# vim keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id ka1
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 66
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.37.100/24 dev eth0 label eth0:1
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

virtual_server 192.168.37.100 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80

    real_server 192.168.37.10 80 {
        weight 1
        HTTP_GET {
            url {
              path /
              status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
   real_server 192.168.37.20 80 {
        weight 1 
        HTTP_GET {
            url {
              path /
              status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        } 
    }
}

    
[root@ka1 keepalived]# scp keepalived.conf ka2:/etc/keepalived/
[root@ka1 keepalived]# systemctl restart keepalived

ka2

[root@ka2 ~]# cat /etc/keepalived/keepalived.conf 
! Configuration File for keepalived

global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from keepalived@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id ka2    <--
   vrrp_mcast_group4 224.100.100.100
}

vrrp_instance VI_1 {
    state BACKUP    <--
    interface eth0
    virtual_router_id 66
    priority 80    <--
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.37.100/24 dev eth0 label eth0:1
    }
    notify_master "/etc/keepalived/notify.sh master"
    notify_backup "/etc/keepalived/notify.sh backup"
    notify_fault "/etc/keepalived/notify.sh fault"
}

virtual_server 192.168.37.100 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    protocol TCP
    sorry_server 127.0.0.1 80

    real_server 192.168.37.10 80 {
        weight 1
        HTTP_GET {
            url {
              path /
              status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
   real_server 192.168.37.20 80 {
        weight 1 
        HTTP_GET {
            url {
              path /
              status_code 200
            }
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        } 
    }
}    
                        
[root@ka2 ~]# systemctl restart keepalived

rs1

[root@rs1 ~]# vim lvs_dr_rs.sh 
#!/bin/bash
vip=192.168.37.100    <--
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up
    #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*)   
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
    esac
    
        
    
[root@rs1 ~]# bash lvs_dr_rs.sh start
[root@rs1 ~]# scp lvs_dr_rs.sh 192.168.37.20:

rs2

[root@rs2 ~]# bash lvs_dr_rs.sh start

ka1和ka2

[root@ka1 ~]# yum install -y ipvsadm 
[root@ka1 ~]# systemctl restart keepalived

route

#配置路由转发功能 
[root@route ~]# vim /etc/sysctl.conf 
net.ipv4.ip_forward = 1 
#启动路由转发 
[root@route ~]# sysctl -p 
net.ipv4.ip_forward = 1


[root@route ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
192.168.1.0     0.0.0.0         255.255.255.0   U     101    0        0 eth1
192.168.37.0    0.0.0.0         255.255.255.0   U     100    0        0 eth0

client

[root@client ~]$ while : ;do curl 192.168.37.100 ;sleep 0.5;done
<h1>rs1</h1>
<h1>rs2</h1>
<h1>rs1</h1>
<h1>rs2</h1>
^C

ka1

#停止ka1主机的keepalived服务
[root@ka1 ~]# systemctl stop keepalived

#策略没有了
[root@ka1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

ka2

#ka2接管ka1规则、策略等···
[root@ka2 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.37.100:80 rr
  -> 192.168.37.10:80             Route   1      0          15        
  -> 192.168.37.20:80             Route   1      0          15  

[root@ka2 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:42:ff:ea brd ff:ff:ff:ff:ff:ff
    inet 192.168.37.40/24 brd 192.168.37.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet 192.168.37.100/24 scope global secondary eth0:1    <--
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe42:ffea/64 scope link 
       valid_lft forever preferred_lft forever

测试

ka1

[root@ka1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.37.100:80 rr
  -> 192.168.37.10:80             Route   1      0          27        
  -> 192.168.37.20:80             Route   1      0          28

rs1

[root@rs1 ~]# systemctl stop httpd

ka1

[root@ka1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.37.100:80 rr       
  -> 192.168.37.20:80             Route   1      0          28

rs2

[root@rs2 ~]# systemctl stop httpd

ka1

'Sorry Server'上线

[root@ka1 ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.37.100:80 rr
  -> 127.0.0.1:80                 Route   1      0          4

第三十一章 高可用集群KEEPALIVED

实验：双主Keepalived

邮件设置

KeepAlived支持IPVS

常用参数

KeepAlived配置检测

实验：单主LVS

第三十一章高可用集群KEEPALIVED