目录
一.两台设备(2.130和2.133)作为调度器,前主后备
1.部署keepalived
2.修改配置文件准备启动
3.配置keepalived的系统日志并启动
二.模拟调度器掉点和web服务进程丢失
1.调度器掉点
2.当类似于httpd这种网站服务掉点
三.以三种健康检查方式引入演示LVS+keepalived
1.TCP_CHECK
2.HTTP_GET|SSL_GET
3.MISC
一.两台设备(2.130和2.133)作为调度器,前主后备
1.部署keepalived
链接:百度网盘 请输入提取码百度网盘为您提供文件的网络备份、同步和分享服务。空间大、速度快、安全稳固,支持教育网加速,支持手机端。注册使用百度网盘即可享受免费存储空间https://pan.baidu.com/s/1T0JmFUrKHe0I4htpniGYeg 提取码:dp1j
如下是两台设备都要做的,所以设备hostname有所不同但不影响
[root@localhost ~ ]# tar xvf keepalived-2.2.8.tar.gz -C /usr/local/src/
[root@localhost ~ ]# yum install -y openssl-devel
[root@localhost ~ ]# cd /usr/local/src/keepalived-2.2.8/
[root@localhost keepalived-2.2.8]# yum install -y gcc gcc-c++ make openssl-devel
[root@localhost keepalived-2.2.8]# ./configure --prefix=/usr/local/keepalived \
> --sysconfdir=/etc --sbindir=/usr/sbin --bindir=/usr/bin
#指定安装、系统配置目录等,有需要的可以自己修改
[root@localhost keepalived-2.2.8]# make && make install #编译安装
[root@main keepalived-2.2.8]# tree /etc/keepalived/
/etc/keepalived/
├── keepalived.conf.sample
└── samples├── keepalived.conf.conditional_conf├── keepalived.conf.fwmark├── keepalived.conf.HTTP_GET.port├── keepalived.conf.inhibit├── keepalived.conf.IPv6├── keepalived.conf.misc_check├── keepalived.conf.misc_check_arg├── keepalived.conf.PING_CHECK├── keepalived.conf.quorum├── keepalived.conf.sample├── keepalived.conf.SMTP_CHECK├── keepalived.conf.SSL_GET├── keepalived.conf.status_code├── keepalived.conf.track_interface├── keepalived.conf.UDP_CHECK├── keepalived.conf.virtualhost├── keepalived.conf.virtual_server_group├── keepalived.conf.vrrp├── keepalived.conf.vrrp.localcheck├── keepalived.conf.vrrp.lvs_syncd├── keepalived.conf.vrrp.routes├── keepalived.conf.vrrp.rules├── keepalived.conf.vrrp.scripts├── keepalived.conf.vrrp.static_ipaddress├── keepalived.conf.vrrp.sync├── sample.misccheck.smbcheck.sh└── sample_notify_fifo.sh
1 directory, 28 files
[root@main keepalived-2.2.8]# tree /usr/local/keepalived/
/usr/local/keepalived/
└── share├── doc│ └── keepalived│ └── README├── man│ ├── man1│ │ └── genhash.1│ ├── man5│ │ └── keepalived.conf.5│ └── man8│ └── keepalived.8└── snmp└── mibs
9 directories, 4 files
[root@main keepalived-2.2.8]# cat /usr/lib/systemd/system/keepalived.service
[Unit]
Description=LVS and VRRP High Availability Monitor
After=network-online.target syslog.target
Wants=network-online.target
Documentation=man:keepalived(8)
Documentation=man:keepalived.conf(5)
Documentation=man:genhash(1)
Documentation=https://keepalived.org
[Service]
Type=forking
PIDFile=/run/keepalived.pid
KillMode=process
EnvironmentFile=-/etc/sysconfig/keepalived
ExecStart=/usr/sbin/keepalived $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID
[Install]
WantedBy=multi-user.target
[root@main keepalived-2.2.8]# vim /etc/sysconfig/keepalived
[root@main keepalived-2.2.8]# tail -1 /etc/sysconfig/keepalived
KEEPALIVED_OPTIONS="-D -d -S 0"
[root@main keepalived-2.2.8]# vim /etc/rsyslog.conf
[root@main keepalived-2.2.8]# systemctl restart rsyslog.service
#服务脚本,但是启动还无法正常完成,继续往下看
2.修改配置文件准备启动
(1)这是主设备
[root@main keepalived]# pwd
/etc/keepalived
[root@main keepalived]# cp keepalived.conf.sample keepalived.conf #修改此文件,这里只放了修改了的部分
[root@main keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {#notification_email {# acassen@firewall.loc# failover@firewall.loc# sysadmin@firewall.loc #这些觉得暂时用不上可以先不管#}#notification_email_from Alexandre.Cassen@firewall.loc#smtp_server 192.168.2.130#smtp_connect_timeout 30router_id 1 #router_id,主备两个双设备需要不一致vrrp_skip_check_adv_addrvrrp_strictvrrp_garp_interval 0vrrp_gna_interval 0
}
vrrp_instance VI_1 {state MASTER #设定类型为masterinterface ens33 #通信网卡名称virtual_router_id 1 #虚拟router组id,主备需要一致priority 100 #优先级值,主高于备advert_int 1authentication {auth_type PASS #身份验证,密码也需要主备一致auth_pass 1111}virtual_ipaddress {192.168.2.100 #要设定的VIP,主备一致}
}
(2)备设备
[root@serverc keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {#notification_email {# acassen@firewall.loc# failover@firewall.loc# sysadmin@firewall.loc#}#notification_email_from Alexandre.Cassen@firewall.loc#smtp_server 192.168.200.1#smtp_connect_timeout 30router_id 2 #主备不一致vrrp_skip_check_adv_addrvrrp_strictvrrp_garp_interval 0vrrp_gna_interval 0
}
vrrp_instance VI_1 {state BACKUP #设定为BACKUPinterface ens33 #通信网卡名称virtual_router_id 1 #组id一致priority 80 #优先级要低于主advert_int 1nopreempt #非抢占模式authentication {auth_type PASS #主备一致auth_pass 1111}virtual_ipaddress {192.168.2.100 #VIP,主备一致}
}
3.配置keepalived的系统日志并启动
[root@main ~]# vim /etc/sysconfig/keepalived #此文件是自动生成的,修改内容如下
KEEPALIVED_OPTIONS="-D -d -S 0"
[root@main ~]# vim /etc/rsyslog.conf
.# Save boot messages also to boot.log
local7.* /var/log/boot.log
# Save keepalived messages also to keepalived.log
local0.* /var/log/keepalived.log
#找准位置添加local0这行
#将这几行取消注释
$ModLoad imudp
$UDPServerRun 514
$ModLoad imtcp
$InputTCPServerRun 514
[root@main ~]# systemctl restart rsyslog.service keepalived.service
[root@main ~]# tail -5 /var/log/keepalived.log #日志已经产生内容
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
Mar 27 20:01:48 main Keepalived_vrrp[33409]: Sending gratuitous ARP on ens33 for 192.168.2.100
[root@main ~]# ip a| grep ens33 -A3 #并且主设备上的VIP已经生成
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:5d:7f:b7 brd ff:ff:ff:ff:ff:ffinet 192.168.2.130/24 brd 192.168.2.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.2.100/32 scope global ens33valid_lft forever preferred_lft foreverinet6 fe80::bf1e:b2a3:a943:8a6d/64 scope link noprefixroute valid_lft forever preferred_lft forever
二.模拟调度器掉点和web服务进程丢失
1.调度器掉点
(1)VIP分配在主设备时,访问到hell
(2)主设备服务器断开、keepalived服务失效
此时VIP备绑定去备设备上了,访问到的内容也变为nihao
[root@main ~]# systemctl stop keepalived.service
[root@serverc keepalived]# ip a | grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:2b:95:b3 brd ff:ff:ff:ff:ff:ffinet 192.168.2.133/24 brd 192.168.2.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.2.100/32 scope global ens33valid_lft forever preferred_lft forever
2.当类似于httpd这种网站服务掉点
使用脚本后台运行来保障httpd和keepalived持续运转
[root@main keepalived]# cat testhttpd.sh
#!/bin/bash
while true; dohttpdpid=$(ps -C httpd --no-header | wc -l)if [ ${httpdpid} -eq 0 ]; thensystemctl start httpdsleep 10httpdpid=$(ps -C httpd --no-header | wc -l)if [ ${httpdpid} -eq 0 ]; thensystemctl stop keepalivedfielsesystemctl restart httpdfisleep 10
done
[root@main keepalived]# nohup ./testhttpd.sh &
[2] 49373
[root@main keepalived]# nohup: ignoring input and appending output to ‘nohup.out’
^C
[root@main keepalived]# jobs
[1]+ Stopped (wd: ~)
[2]- Running nohup ./testhttpd.sh &
[root@main keepalived]# systemctl stop httpd #手动停掉后过几秒又将其启动起来了
[root@main keepalived]# ps -C httpdPID TTY TIME CMD
[root@main keepalived]# ps -C httpdPID TTY TIME CMD51258 ? 00:00:00 httpd51259 ? 00:00:00 httpd51261 ? 00:00:00 httpd51262 ? 00:00:00 httpd51263 ? 00:00:00 httpd51264 ? 00:00:00 httpd51288 ? 00:00:00 httpd
三.以三种健康检查方式引入演示LVS+keepalived
主设备-192.168.2.130
备设备-192.168.2.133
VIP-192.168.2.100
RS1-192.168.2.131
RS2-192.168.2.132
1.TCP_CHECK
(1)主设备配置
[root@main keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {#notification_email {# acassen@firewall.loc# failover@firewall.loc# sysadmin@firewall.loc#}#notification_email_from Alexandre.Cassen@firewall.loc#smtp_server 192.168.2.130#smtp_connect_timeout 30router_id 1vrrp_skip_check_adv_addrvrrp_strictvrrp_garp_interval 0vrrp_gna_interval 0
}
vrrp_instance VI_1 {state MASTERinterface ens33virtual_router_id 1mcast_src_ip 192.168.2.130priority 100advert_int 1nopreemptauthentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.2.100}
}
virtual_server 192.168.2.100 80 { #VIPdelay_loop 6 #健康检查间隔时间lb_algo rr #调度方式为rrlb_kind DR #LVS模式为DRprotocol TCP #TCP协议
real_server 192.168.2.131 80 { #RIP1weight 1 #权重值TCP_CHECK { #TCP_CHECK方式connect_timeout 3 #连接超时时间nb_get_retry 3 #重连次数connection_port 80 #检查端口delay_before_retry 3 #重连间隔时间}}real_server 192.168.2.132 80 { #RIP2weight 1TCP_CHECK {connect_timeout 3nb_get_retry 3connection_port 80delay_before_retry 3}}
}
[root@main keepalived]# systemctl restart keepalived.service
(2)备设备配置
[root@serverc keepalived]# cat keepalived.conf
! Configuration File for keepalived
global_defs {#notification_email {# acassen@firewall.loc# failover@firewall.loc# sysadmin@firewall.loc#}#notification_email_from Alexandre.Cassen@firewall.loc#smtp_server 192.168.200.1#smtp_connect_timeout 30router_id 2vrrp_skip_check_adv_addrvrrp_strictvrrp_garp_interval 0vrrp_gna_interval 0
}
vrrp_instance VI_1 {state BACKUPinterface ens33virtual_router_id 1priority 80advert_int 1nopreemptauthentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.2.100}
}
virtual_server 192.168.2.100 80 {delay_loop 6lb_algo rrlb_kind DRprotocol TCP
real_server 192.168.2.131 80 {weight 1TCP_CHECK {connect_timeout 3nb_get_retry 3connection_port 80delay_before_retry 3}}real_server 192.168.2.132 80 {weight 1TCP_CHECK {connect_timeout 3nb_get_retry 3connection_port 80delay_before_retry 3}}
}
[root@serverc keepalived]# systemctl restart keepalived.service
(3)节点执行lvs-dr脚本服务,来进行绑定VIP和添加通信路由,这步可以手动做,参考前面lvs-dr集群的文章
[root@servera ~]# vim /etc/init.d/lvs-dr
[root@servera ~]# cat /etc/init.d/lvs-dr #VIP等需要自己更改
#!/bin/bash
LOCK=/var/lock/ipvsadm.lock
VIP=192.168.2.100
. /etc/rc.d/init.d/functions
start() {PID=`ifconfig | grep lo:130 | wc -l`if [ $PID -ne 0 ];thenecho "The LVS-DR-RIP Server is already running !"else/sbin/ifconfig lo:130 $VIP netmask 255.255.255.255 broadcast $VIP up/sbin/route add -host $VIP dev lo:130echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignoreecho "2" >/proc/sys/net/ipv4/conf/lo/arp_announceecho "1" >/proc/sys/net/ipv4/conf/ens33/arp_ignoreecho "2" >/proc/sys/net/ipv4/conf/ens33/arp_announceecho "1" >/proc/sys/net/ipv4/conf/all/arp_ignoreecho "2" >/proc/sys/net/ipv4/conf/all/arp_announce/bin/touch $LOCKecho "starting LVS-DR-RIP server is ok !"fi
}
stop() {/sbin/route del -host $VIP dev lo:130/sbin/ifconfig lo:130 down >/dev/nullecho "0" >/proc/sys/net/ipv4/conf/lo/arp_ignoreecho "0" >/proc/sys/net/ipv4/conf/lo/arp_announceecho "0" >/proc/sys/net/ipv4/conf/ens33/arp_ignoreecho "0" >/proc/sys/net/ipv4/conf/ens33/arp_announceecho "0" >/proc/sys/net/ipv4/conf/all/arp_ignoreecho "0" >/proc/sys/net/ipv4/conf/all/arp_announcerm -rf $LOCKecho "stopping LVS-DR-RIP server is ok !"
}
status() {if [ -e $LOCK ];thenecho "The LVS-DR-RIP Server is already running !"elseecho "The LVS-DR-RIP Server is not running !"fi
}
case "$1" instart)start;;stop)stop;;restart)stopstart;;status)status;;*)echo "Usage: $1 {start|stop|restart|status}"exit 1
esac
exit 0
[root@servera ~]# systemctl daemon-reload
[root@servera ~]# service lvs-dr start
[root@servera ~]# route -n #通信路由添加成功
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
0.0.0.0 192.168.2.1 0.0.0.0 UG 100 0 0 ens33
192.168.2.0 0.0.0.0 255.255.255.0 U 100 0 0 ens33
192.168.2.100 0.0.0.0 255.255.255.255 UH 0 0 0 lo
[root@servera ~]# ip a| grep lo #环回创建成功
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00inet 127.0.0.1/8 scope host loinet 192.168.2.100/32 brd 192.168.2.100 scope global lo:130inet 192.168.2.131/24 brd 192.168.2.255 scope global noprefixroute ens33
(4)主设备上查看VIP是否创建成功
[root@main keepalived]# ip a | grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:5d:7f:b7 brd ff:ff:ff:ff:ff:ffinet 192.168.2.130/24 brd 192.168.2.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.2.100/32 scope global ens33valid_lft forever preferred_lft forever
[root@main keepalived]# ipvsadm -Ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 192.168.2.100:80 rr-> 192.168.2.131:80 Route 1 0 0 -> 192.168.2.132:80 Route 1 0 0
(5)进行测试
负载均衡测试
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
节点131上httpd服务掉点,具体可以通过watch ipvsadm -Ln来查看节点剔除和恢复过程
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
主调度器设备130掉点
[root@main keepalived]# systemctl stop keepalived.service
[root@serverc keepalived]# ip a | grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:2b:95:b3 brd ff:ff:ff:ff:ff:ffinet 192.168.2.133/24 brd 192.168.2.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.2.100/32 scope global ens33valid_lft forever preferred_lft forever
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
#负载均衡正常
2.HTTP_GET|SSL_GET
以genhash来生成检查摘要信息
[root@main keepalived]# genhash -s 192.168.2.131 -p 80 -u /index.html
db1dd528b0e0c9a347eda778aec00559
[root@main keepalived]# genhash -s 192.168.2.132 -p 80 -u /index.html
27d4c8a485f28559e9b1737702b40225
#如下配置
virtual_server 192.168.2.100 80 {delay_loop 6lb_algo rrlb_kind DRprotocol TCP
real_server 192.168.2.131 80 {weight 1
# TCP_CHECK {
# connect_timeout 3
# nb_get_retry 3
# connection_port 80
# delay_before_retry 3
# }HTTP_GET {url {path /index.htmldigset 2d4074c5771f087dd468d1960185f1f5}connect_port 80connect_timeout 3nb_get_retry 3delay_before_retry 3} }real_server 192.168.2.132 80 {weight 1
# TCP_CHECK {
# connect_timeout 3
# nb_get_retry 3
# connection_port 80
# delay_before_retry 3
# }HTTP_GET {url {path /index.htmldigset 2d4074c5771f087dd468d1960185f1f5 #基于页面后端hash值#status 200 #基于返回状态码} connect_port 80connect_timeout 3nb_get_retry 3delay_before_retry 3}}
}
#重启keepalived后测试
测试
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done #负载均衡
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done #节点1掉点
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
10.0.0.12
[root@main keepalived]# systemctl stop keepalived.service #主设备掉点,负载均衡正常
[root@serverc keepalived]# ip a |grep ens33 -A1
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:2b:95:b3 brd ff:ff:ff:ff:ff:ffinet 192.168.2.133/24 brd 192.168.2.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.2.100/32 scope global ens33valid_lft forever preferred_lft forever
[root@localhost ~]# for ((i=1;i<=10;i++));do curl 192.168.2.100;done
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
10.0.0.11
10.0.0.12
3.MISC
利用健康测试脚本来测试httpd服务
real_server 192.168.2.131 80 {weight 1MISC_CHECK {misc_path "/etc/keepalived/test.sh 192.168.2.131"misc_timeout 3
}
# TCP_CHECK {
# connect_timeout 3
# nb_get_retry 3
# connection_port 80
# delay_before_retry 3
# }
# HTTP_GET {
# url {
# path /index.html
# digset 2d4074c5771f087dd468d1960185f1f5
# status 200
# }
# connect_port 80
# connect_timeout 3
# nb_get_retry 3
# delay_before_retry 3
# }}
[root@main keepalived]# cat test.sh #只针对200状态码
#!/bin/bash
if [ $# -ne 1 ]
thenecho "You should supply an url parameter."exit 1
elsen=`curl -I "$1" 2> /dev/null | grep "200 OK" | wc -l`if [ $n -eq 1 ]thenexit 0elseexit 1fi
fi
[root@main keepalived]# cat test.sh #可以使用nmap来做,yum install -y nmap
#!/bin/bash
if [ $# -ne 1 ]
thenecho "You should supply an url parameter."exit 1
elseip_and_path=$1ip=$(echo "$ip_and_path" | sed 's/.*\/\/\([0-9.]*\).*/\1/')n=$(nmap -p80 "$ip" | awk '/^80\/tcp/ {print $2}')if [ $n == 'open' ]thenexit 0elseexit 1fi
fi