@[TOC](第三十章 LVS调度器小节2)

实验：LVS的实现DR模型

图片.png

主机名称	IP地址
client	192.168.1.16\GW192.168.1.7
route	37.7
LVS	37.30\GW192.168.37.7
Web1	37.10\GW192.168.37.7
Web2	37.20\GW192.168.37.7

route

#配置路由转发功能
[root@route ~]# vim /etc/sysctl.conf
net.ipv4.ip_forward = 1

#启动路由转发
[root@route ~]# sysctl -p
net.ipv4.ip_forward = 1

client

路由配置好后、内网可ping通

[root@client ~]$ ping 192.168.37.10 -c 2
PING 192.168.37.10 (192.168.37.10) 56(84) bytes of data.
64 bytes from 192.168.37.10: icmp_seq=1 ttl=63 time=1.25 ms
64 bytes from 192.168.37.10: icmp_seq=2 ttl=63 time=1.98 ms

--- 192.168.37.10 ping statistics ---
2 packets transmitted, 2 received, 0% packet loss, time 1003ms
rtt min/avg/max/mdev = 1.254/1.617/1.980/0.363 ms


[root@client ~]$ curl 192.168.37.10/index.html
R10
[root@client ~]$ curl 192.168.37.20/index.html
R20

web1

[root@web1 ~]# cat lvs_dr_rs.sh 
#!/bin/bash
#Author:wang
#Date:2017-08-13
vip=192.168.37.100
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up
    #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*) 
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac


[root@web1 ~]# bash lvs_dr_rs.sh start
The httpd Server is Ready!
The RS Server is Ready!
[root@web1 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.37.100/32 scope global lo:1
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:27:de:e1 brd ff:ff:ff:ff:ff:ff
    inet 192.168.37.10/24 brd 192.168.37.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe27:dee1/64 scope link 
       valid_lft forever preferred_lft forever


[root@web1 ~]# scp lvs_dr_rs.sh 192.168.37.20:

web2

[root@web2 ~]# bash lvs_dr_rs.sh start
The httpd Server is Ready!
The RS Server is Ready!

lvs

[root@lvs ~]# ip a a 192.168.37.100/24 dev eth0
[root@lvs ~]# yum install ipvsadm -y
[root@lvs ~]# ipvsadm -A -t 192.168.37.100:80 -s rr
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.37.100:80 rr
[root@lvs ~]# ipvsadm -a -t 192.168.37.100:80 -r 192.168.37.10
[root@lvs ~]# ipvsadm -a -t 192.168.37.100:80 -r 192.168.37.20
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.37.100:80 rr
  -> 192.168.37.10:80             Route   1      0          0         
  -> 192.168.37.20:80             Route   1      0          0

client

[root@client ~]$ while true; do curl http://192.168.37.100/index.html ;sleep 0.5;done
<h1>web2</h1>
<h1>web1</h1>
<h1>web2</h1>
<h1>web1</h1>
<h1>web2</h1>

实验：基于跨网段通讯DR模型

route

#添加路由转发功能
[root@route ~]# vim /etc/sysctl.conf 
net.ipv4.ip_forward = 1    <--
#使路由转发生效
[root@route ~]# sysctl -p
#添加网卡在'192.168.37.7'配置文件中
[root@route ~]# nmcli connection modify eth0 +ipv4.addresses 10.0.0.200/8
[root@route ~]# nmcli connection up eth0

[root@route network-scripts]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
10.0.0.0        0.0.0.0         255.0.0.0       U     100    0        0 eth0
192.168.1.0     0.0.0.0         255.255.255.0   U     101    0        0 eth1
192.168.37.0    0.0.0.0         255.255.255.0   U     100    0        0 eth0

rs1

[root@rip10 ~]# systemctl start httpd
[root@rip10 ~]# vim lvs_dr_rs.sh 
#!/bin/bash
vip=10.0.0.100    <--修改IP地址
mask='255.255.255.255'
dev=lo:1
rpm -q httpd &> /dev/null || yum -y install httpd &>/dev/null
service httpd start &> /dev/null && echo "The httpd Server is Ready!"
echo "<h1>`hostname`</h1>" > /var/www/html/index.html

case $1 in
start)
    echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
    ifconfig $dev $vip netmask $mask #broadcast $vip up
    #route add -host $vip dev $dev
    echo "The RS Server is Ready!"
    ;;
stop)
    ifconfig $dev down
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
    echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
    echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
    echo "The RS Server is Canceled!"
    ;;
*)
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac

#运行脚本
[root@rip10 ~]# bash lvs_dr_rs.sh start

[root@rip10 ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 10.0.0.100/32 scope global lo:1    <--可以看到地址已经绑定
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 00:0c:29:27:de:e1 brd ff:ff:ff:ff:ff:ff
    inet 192.168.37.10/24 brd 192.168.37.255 scope global noprefixroute eth0
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe27:dee1/64 scope link 
       valid_lft forever preferred_lft forever

[root@rip10 ~]# scp lvs_dr_rs.sh 192.168.37.20:

rs2

#启动服务
[root@rip20 ~]# systemctl start httpd
#运行脚本
[root@rip20 ~]# bash lvs_dr_rs.sh start

lvs

[root@lvs ~]# cat lvs_dr_vs.sh 
#!/bin/bash
vip='10.0.0.100'    <--VIP地址
iface='lo:1'
mask='255.255.255.255'
port='80'
rs1='192.168.37.10'    <--RS1地址
rs2='192.168.37.20'    <--RS2地址
scheduler='rr'    <--模式
type='-g'
rpm -q ipvsadm &> /dev/null || yum -y install ipvsadm &> /dev/null

case $1 in
start)
    ifconfig $iface $vip netmask $mask #broadcast $vip up
    iptables -F
 
    ipvsadm -A -t ${vip}:${port} -s $scheduler
    ipvsadm -a -t ${vip}:${port} -r ${rs1} $type -w 1
    ipvsadm -a -t ${vip}:${port} -r ${rs2} $type -w 1
    echo "The VS Server is Ready!"
    ;;
stop)
    ipvsadm -C
    ifconfig $iface down
    echo "The VS Server is Canceled!"
    ;;
*)
    echo "Usage: $(basename $0) start|stop"
    exit 1
    ;;
esac

#运行脚本
[root@lvs ~]# bash lvs_dr_vs.sh start

client

[root@client ~]$ curl 10.0.0.100
<h1>rip10</h1>
[root@client ~]$ curl 10.0.0.100
<h1>rip20</h1>
[root@client ~]$ curl 10.0.0.100
<h1>rip10</h1>
[root@client ~]$ curl 10.0.0.100
<h1>rip20</h1>

实现自签名证书

rs1和Rs2

[root@rip10 ~]# yum install -y mod_ssl
[root@rip10 ~]# systemctl restart httpd
和
[root@rip20 ~]# yum install -y mod_ssl
[root@rip20 ~]# systemctl restart httpd

lvs

[root@lvs ~]# ipvsadm -A -t 10.0.0.100:443 -s rr 
[root@lvs ~]# ipvsadm -a -t 10.0.0.100:443 -r 192.168.37.10
[root@lvs ~]# ipvsadm -a -t 10.0.0.100:443 -r 192.168.37.20
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.100:80 rr
  -> 192.168.37.10:80             Route   1      0          0         
  -> 192.168.37.20:80             Route   1      0          0         
TCP  10.0.0.100:443 rr
  -> 192.168.37.10:443            Route   1      0          0         
  -> 192.168.37.20:443            Route   1      0          0

client

可以访问

[root@client ~]$ curl -k https://10.0.0.100/
<h1>rip10</h1>
[root@client ~]$ curl -k https://10.0.0.100/
<h1>rip20</h1>

FireWall Mark(防火墙标签)

FWM：FireWall Mark

MARK target 可用于给特定的报文打标记

--set-mark value

其中：value 可为0xffff格式，表示十六进制数字

借助于防火墙标记来分类报文，而后基于标记定义集群服务；可将多个不同的应用使用同一个集群服务进行调度

实现方法：
#在Director主机打标记：
iptables -t mangle -A PREROUTING -d $vip -p $proto -m multiport --dports $port1,$port2,... -j MARK --set-mark NUMBER
#在Director主机基于标记定义集群服务：
ipvsadm -A -f NUMBER [options]

lvs

开始给80、443端口合并成一条命令并贴标签

[root@lvs ~]# iptables -t mangle -A PREROUTING -d 10.0.0.100 -p tcp -m multiport --dports 80,443 -j MARK --set-mark 10
[root@lvs ~]# iptables -t mangle -vnL
Chain PREROUTING (policy ACCEPT 63 packets, 4184 bytes)
 pkts bytes target     prot opt in     out     source               destination         
    0     0 MARK       tcp  --  *      *       0.0.0.0/0            10.0.0.100           multiport dports 80,443 MARK set 0xa

Chain INPUT (policy ACCEPT 63 packets, 4184 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain FORWARD (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain OUTPUT (policy ACCEPT 33 packets, 3052 bytes)
 pkts bytes target     prot opt in     out     source               destination         

Chain POSTROUTING (policy ACCEPT 33 packets, 3052 bytes)
 pkts bytes target     prot opt in     out     source               destination

[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  10.0.0.100:80 rr
  -> 192.168.37.10:80             Route   1      0          0         
  -> 192.168.37.20:80             Route   1      0          0         
TCP  10.0.0.100:443 rr
  -> 192.168.37.10:443            Route   1      0          0         
  -> 192.168.37.20:443            Route   1      0          0         

#清空
[root@lvs ~]# ipvsadm -C
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
[root@lvs ~]# ipvsadm -A -f 10 -s rr
[root@lvs ~]# ipvsadm -a -f 10 -r 192.168.37.10 -g
[root@lvs ~]# ipvsadm -a -f 10 -r 192.168.37.20 -g
#此时、只要访问80或443统一调度
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
FWM  10 rr
  -> 192.168.37.10:0              Route   1      0          0         
  -> 192.168.37.20:0              Route   1      0          0

client

[root@client ~]$ while : ;do curl 10.0.0.100 ;curl -k https://10.0.0.100/ ;sleep 0.5;done
<h1>rip20</h1>
<h1>rip10</h1>
<h1>rip20</h1>
<h1>rip10</h1>
^C

持久连接

session 绑定：对共享同一组RS的多个集群服务，需要统一进行绑定，lvs sh算法无法实现

持久连接（lvs persistence ）模板：实现无论使用任何调度算法，在一段时间内（默认360s ），能够实现将来自同一个地址的请求始终发往同一个RS
ipvsadm -A|E -t|u|f service-address [-s scheduler] [-p [timeout]]
持久连接实现方式：

每端口持久（PPC）：每个端口定义为一个集群服务，每集群服务单独调度

每防火墙标记持久（PFWMC）：基于防火墙标记定义集群服务；可实现将多个端口上的应用统一调度，即所谓的port Affinity

每客户端持久（PCC）：基于0端口（表示所有服务）定义集群服务，即将客户端对所有应用的请求都调度至后端主机，必须定义为持久模式

#持久连接、'-p'默认360秒
[root@lvs ~]# ipvsadm -E -f 10 -s rr -p
[root@lvs ~]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
FWM  10 rr persistent 360
  -> 192.168.37.10:0              Route   1      0          0         
  -> 192.168.37.20:0              Route   1      0          0

LVS高可用性

Director不可用，整个系统将不可用；SPoF Single Point of Failure

解决方案：高可用

keepalived heartbeat/corosync

某RS不可用时，Director依然会调度请求至此RS

解决方案：由Director对各RS健康状态进行检查，失败时禁用，成功时启用

keepalived heartbeat/corosync ldirectord

检测方式：

(a) 网络层检测，icmp

(b) 传输层检测，端口探测

(c) 应用层检测，请求某关键资源

RS全不用时：backup server, sorry server

第三十章 LVS调度器 小节2

实验：LVS的实现DR模型

实验：基于跨网段通讯DR模型

FireWall Mark(防火墙标签)

持久连接

LVS高可用性

第三十章 LVS调度器小节2