1：拉去镜像osixia/keepalived
[root@localhost ~]# docker pull osixia/keepalived
Using default tag: latest
latest: Pulling from osixia/keepalived
89d9c30c1d48: Pull complete
c9ada908efbe: Pull complete
4a9101e4ada9: Pull complete
b428f77d2dc4: Pull complete
c1f2db3054e3: Pull complete
8ef2fa7e2034: Pull complete
c641dd822a23: Pull complete
Digest: sha256:74179ae03efe2a975bfbd411181315d6eb8052d91c10627b03ad60bceac71890
Status: Downloaded newer image for osixia/keepalived:latest
docker.io/osixia/keepalived:latest
[root@localhost ~]#
2： 启动容器keepalived做为MASTER
[root@localhost ~]# docker run -d --name keepalived-master --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -e KEEPALIVED_INTERFACE="ens33" -e KEEPALIVED_PRIORITY=100 -e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['192.168.11.43', '192.168.11.47']" -e KEEPALIVED_VIRTUAL_IPS="192.168.11.40" -e KEEPALIVED_STATE="MASTER" --privileged=true osixia/keepalived --loglevel debug
5bd11fe03cdcd2ebf35722066b030fd55b30b46fe23e6ae8caca5a3952f2ea72
[root@localhost ~]#
启动容器keepalived做为BACKUP
[root@localhost opt]# docker run -d --name keepalived-backup --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -e KEEPALIVED_INTERFACE="ens33" -e KEEPALIVED_PRIORITY=50 -e KEEPALIVED_UNICAST_PEERS="#PYTHON2BASH:['192.168.11.43', '192.168.11.47']" -e KEEPALIVED_VIRTUAL_IPS="192.168.11.40" -e KEEPALIVED_STATE="BACKUP" --privileged=true osixia/keepalived --loglevel debug
e3de9120daaffea32c1ffc72424622b47bababef80e9da6dd0a99f8459788cfc
[root@localhost opt]#
如果MASTER和BACKUP的KEEPALIVED_PRIORITY一样，需要手动才能切换
2-1：–net=host这是共享宿主机网络，KEEPALIVED_INTERFACE="ens33"中的ens33需要确定宿主机是否有这个网卡，通过命令ip add查看
[root@localhost ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00inet 127.0.0.1/8 scope host lovalid_lft forever preferred_lft foreverinet6 ::1/128 scope host valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:46:90:db brd ff:ff:ff:ff:ff:ffinet 192.168.11.43/24 brd 192.168.11.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.11.40/32 scope global ens33valid_lft forever preferred_lft foreverinet6 fd0d:3fd3:f1d::907/128 scope global noprefixroute valid_lft forever preferred_lft foreverinet6 fd0d:3fd3:f1d:0:b544:377c:c1d2:978f/64 scope global noprefixroute valid_lft forever preferred_lft foreverinet6 fe80::64d6:f397:92fd:1c0f/64 scope link noprefixroute valid_lft forever preferred_lft forever2-2：查看容器情况
[root@localhost ~]# docker ps -a
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
5bd11fe03cdc osixia/keepalived "/container/tool/run…" 12 minutes ago Up 11 minutes keepalived-master
73e19612fe5f mysql:8.0.33 "docker-entrypoint.s…" 31 hours ago Up 4 hours 0.0.0.0:3306->3306/tcp, :::3306->3306/tcp, 33060/tcp mysql2-3：通过IP-add查看是否有这个KEEPALIVED_VIRTUAL_IPS=“192.168.11.40”，有就代表正常
[root@localhost ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00inet 127.0.0.1/8 scope host lovalid_lft forever preferred_lft foreverinet6 ::1/128 scope host valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:46:90:db brd ff:ff:ff:ff:ff:ffinet 192.168.11.43/24 brd 192.168.11.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.11.40/32 scope global ens33valid_lft forever preferred_lft foreverinet6 fd0d:3fd3:f1d::907/128 scope global noprefixroute valid_lft forever preferred_lft foreverinet6 fd0d:3fd3:f1d:0:b544:377c:c1d2:978f/64 scope global noprefixroute valid_lft forever preferred_lft foreverinet6 fe80::64d6:f397:92fd:1c0f/64 scope link noprefixroute valid_lft forever preferred_lft forever2-4：通过容器日记查看信息
[root@localhost ~]# docker logs keepalived-master
2-5：出现错误关于keepalived执行后日志狂刷IPVS: Can’t initialize ipvs: Protocol not available的问题
安装了keepalived+lvs,达到了高可用的负载均衡，但是今天再启用的时候发现keepalived不正常，通过 /var/log/messages 查看系统日志发现狂刷
2-6： 查看：ip_vs模块系统默认没有自动加载
lsmod | grep ip_vs
2-7： 命令查看一下，如果没有任何输出则表示ip_vs模块并没有被内核加载，那必须手动加载一下
modprobe ip_vs
modprobe ip_vs_wrr
2-8： 过程如下，如果要让系统开机加载此模块，在/etc/rc.local文件中添加上面两句，这样开机就能自动加载了
[root@localhost ~]# modprobe ip_vs
[root@localhost ~]# modprobe ip_vs_wrr
[root@localhost ~]# lsmod | grep ip_vs
ip_vs_wrr 12697 0
ip_vs 145458 2 ip_vs_wrr
nf_conntrack 139264 7 ip_vs,nf_nat,nf_nat_ipv4,xt_conntrack,nf_nat_masquerade_ipv4,nf_conntrack_netlink,nf_conntrack_ipv4
libcrc32c 12644 4 xfs,ip_vs,nf_nat,nf_conntrack
[root@localhost ~]#
3：docker启动的配置文件目录在容器路径/container/service/keepalived/assets下
/container/service/keepalived/assets # ls
README.md keepalived.conf notify.sh
/container/service/keepalived/assets # cat keepalived.conf
global_defs {default_interface ens33
}vrrp_instance VI_1 {interface ens33state BACKUPvirtual_router_id 51priority 100nopreempt #非抢占模式，如果设置降权重方式，必须要去掉该值或者注释，不然不会抢占unicast_peer {192.168.11.43192.168.11.47}virtual_ipaddress {192.168.11.40}authentication {auth_type PASSauth_pass d0cker}notify "/container/service/keepalived/assets/notify.sh"
}3-1：MASTER的keepalived挂载配置文件keepalived.conf启动容器方式（重要主要：文件的权限必须设置为644，配置文件其他权限挂载keepalived启动不起来）
docker run -d --name keepalived-master --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -v /opt/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf --privileged=true osixia/keepalived --loglevel debug --copy-service
global_defs {default_interface ens33
}
vrrp_script check_mysql {script "/container/service/keepalived/assets/check_mysql.sh" #检测脚本文件interval 5 #检测时间间隔，单位是秒weight -10 #权重#检测连续2次失败才算确定是真失败。会用weight减少优先级（1-255之间）fall 2 #脚本结果导致的优先级变更，检测失败（脚本返回非0）则优先级 - 10 rise 1 #检测1次成功就算成功。但不修改优先级
}
vrrp_instance VI_1 {interface ens33 #设置实例绑定的网卡state BACKUP #设置实例初始状态，实际的MASTER和BACKUP是选举决定的virtual_router_id 51 #同一实例下virtual_router_id必须相同priority 100 #设置优先级，优先级高的会被竞选为Masternopreempt #非抢占模式，如果设置降权重方式，必须要去掉该值或者注释，不然不会抢占unicast_peer { #单播模式，设置对端ip192.168.11.43192.168.11.47}virtual_ipaddress { #设置VIP，可以设置多个192.168.11.40}authentication { #设置认证auth_type PASS #认证方式，支持PASS和AHauth_pass d0cker #认证密码}track_script { #设置追踪脚本check_mysql}notify "/container/service/keepalived/assets/notify.sh"
}
3-2：BACKUP的keepalived挂载配置文件keepalived.conf启动容器方式（重要主要：文件的权限必须设置为644，配置文件其他权限挂载keepalived启动不起来）
docker run -d --name keepalived-backup --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -v /opt/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf --privileged=true osixia/keepalived --loglevel debug --copy-service
global_defs {default_interface ens33
}
vrrp_script check_mysql {script "/container/service/keepalived/assets/check_mysql.sh" #检测脚本文件interval 5 #检测时间间隔，注意这个时间必须比脚本执行的耗时时间长，不然运行都人为有问题，就会降权重，单位是秒weight -10 #权重，priority值减去此值要小于备服务的priority值#检测连续2次失败才算确定是真失败。会用weight减少优先级（1-255之间）fall 2 #脚本结果导致的优先级变更，检测失败（脚本返回非0） rise 1 #检测1次成功就算成功。但不修改优先级
}vrrp_instance VI_1 {interface ens33 #设置实例绑定的网卡state BACKUP #设置实例初始状态，实际的MASTER和BACKUP是选举决定的virtual_router_id 51 #同一实例下virtual_router_id必须相同priority 50 #设置优先级，优先级高的会被竞选为Masternopreempt #非抢占模式，如果设置降权重方式，必须要去掉该值或者注释，不然不会抢占unicast_peer { #单播模式，设置对端ip192.168.11.43192.168.11.47}virtual_ipaddress { #设置VIP，可以设置多个192.168.11.40}authentication { #设置认证auth_type PASS #认证方式，支持PASS和AHauth_pass d0cker #认证密码}track_script { #设置追踪脚本check_mysql}notify "/container/service/keepalived/assets/notify.sh"
}
需要特别注意：nopreempt这个参数只能用于state为BACKUP的情况，所以在配置的时候要把master和backup的state都设置成BACKUP，这样才会实现keepalived的非抢占模式！
3-3：当MASTER出现故障自动切换BACKUP，当MASTER恢复，自动切换回MASTER，如果MASTER和BACKUP的KEEPALIVED_PRIORITY一样，需要手动才能切换
：vip 不释放问题，如果停止keepalived还不释放vip可使用命令删除如下
ip addr del 192.168.11.40/32 dev ens33
3-4：mysql状态 check_mysql监听脚本，这个脚本是直接关闭keepalived（注意权限需要改为744）
#!/bin/bash#检测mysql是否启动了
if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #如果mysql没有启动就启动mysqldocker restart mysql #重启mysqlsleep 3 #注意设置这个时间一定要比interval小，不然一直认为执行脚本失败，单位是秒if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #mysql重启失败，则停掉keepalived服务，进行VIP转移systemctl stop keepalived #非容器部署kill 1 #容器部署 fi
fi
3-5：mysql状态 check_mysql监听脚本，这个脚本是返回值，可以检测几次，和动态减去权重（注意权限需要改为744）
#!/bin/bash
#检测mysql是否启动了
if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #如果mysql没有启动就启动mysqldocker restart mysql #重启mysqlsleep 3 #注意设置这个时间一定要比interval小，不然一直认为执行脚本失败，单位是秒if [ "$(ps -ef | grep "mysql"| grep -v grep )" == "" ];then #mysql重启失败，返回 0exit 0elseexit 1 #mysql重启存在，返回 1 fi
fi
3-6：mysql状态 chk_mysql_port监听脚本，这个脚本是直接关闭keepalived（注意权限需要改为744）
#!/bin/bash
counter=$(netstat -na|grep "LISTEN"|grep "3306"|wc -l)
if [ "${counter}" -eq 0 ]; thensystemctl stop keepalived #非容器部署kill 1 #容器部署
fi
3-7：挂载配置文件和挂载脚本的docker启动方式
docker run -d --name keepalived-master --net=host -v /etc/localtime:/etc/localtime:ro -v /etc/timezone:/etc/timezone:ro -v /opt/keepalived/chk_mysql_port.sh:/container/service/keepalived/assets/chk_mysql_port.sh -v /opt/keepalived/keepalived.conf:/container/service/keepalived/assets/keepalived.conf --privileged=true osixia/keepalived --loglevel debug --copy-service
3-8：nginx状态监听脚本
#!/bin/bash
#检测nginx是否启动了
if [ "$(ps -ef | grep "nginx: master process"| grep -v grep )" == "" ];then #如果nginx没有启动就启动nginx docker restart nginx #重启nginxsleep 3 #单位是秒if [ "$(ps -ef | grep "nginx: master process"| grep -v grep )" == "" ];then #nginx重启失败，则停掉keepalived服务，进行VIP转移systemctl stop keepalived #非容器部署kill 1 #容器部署 fi
fi
3-9：可以通过docker logs -f keepalived-master实时查看日记，也可以看到降权和复权的过程
Thu Mar 14 11:48:22 2024: Script `check_mysql_port` now returning 1
Thu Mar 14 11:48:22 2024: VRRP_Script(check_mysql_port) failed (exited with status 1)
Thu Mar 14 11:48:22 2024: (VI_1) Changing effective priority from 100 to 40
Thu Mar 14 11:48:25 2024: (VI_1) Receive advertisement timeout
Thu Mar 14 11:48:25 2024: (VI_1) Entering MASTER STATE
Thu Mar 14 11:48:25 2024: (VI_1) setting VIPs.
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: (VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:25 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
I'm the MASTER! Whup whup.
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: (VI_1) Sending/queueing gratuitous ARPs on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:30 2024: Sending gratuitous ARP on ens33 for 192.168.11.40
Thu Mar 14 11:48:43 2024: Interface veth3a6fe0a added
Thu Mar 14 11:48:43 2024: Interface veth8c4218d added
Thu Mar 14 11:48:43 2024: Interface veth3a6fe0a deleted
Thu Mar 14 11:48:46 2024: Script `check_mysql_port` now returning 0
Thu Mar 14 11:48:46 2024: VRRP_Script(check_mysql_port) succeeded
Thu Mar 14 11:48:46 2024: (VI_1) Changing effective priority from 40 to 100
Thu Mar 14 11:49:45 2024: Interface veth3a6fe0a added
Thu Mar 14 11:49:45 2024: Interface veth3a6fe0a deleted
Thu Mar 14 11:49:45 2024: Interface veth8c4218d deleted
Thu Mar 14 11:49:45 2024: Unknown interface deleted
Thu Mar 14 11:49:50 2024: Script `check_mysql_port` now returning 1
Thu Mar 14 11:49:54 2024: VRRP_Script(check_mysql_port) failed (exited with status 1)
Thu Mar 14 11:49:54 2024: (VI_1) Changing effective priority from 100 to 404：负载均衡的keepalived.conf配置文件讲解
global_defs { #全局定义部分notification_email { #设置报警邮件地址，可设置多个acassen@firewall.loc #接收通知的邮件地址}notification_email_from test0@163.com #设置 发送邮件通知的地址smtp_server smtp.163.com #设置 smtp server 地址，可是ip或域名.可选端口号 （默认25）smtp_connect_timeout 30 #设置 连接 smtp server的超时时间router_id LVS_DEVEL #主机标识，用于邮件通知vrrp_skip_check_adv_addrvrrp_strict #严格执行VRRP协议规范，此模式不支持节点单播vrrp_garp_interval 0vrrp_gna_interval 0script_user keepalived_script #指定运行脚本的用户名和组。默认使用用户的默认组。如未指定，默认为keepalived_script 用户，如无此用户，则使用rootenable_script_security #如果路径为非root可写，不要配置脚本为root用户执行。
}vrrp_script chk_nginx_service { #VRRP 脚本声明script "/etc/keepalived/chk_nginx.sh" #周期性执行的脚本interval 3 #运行脚本的间隔时间，秒weight -20 #权重，priority值减去此值要小于备服务的priority值fall 3 #检测几次失败才为失败，整数rise 2 #检测几次状态为正常的，才确认正常，整数user keepalived_script #执行脚本的用户或组
}vrrp_instance VI_1 { #vrrp 实例部分定义，VI_1自定义名称state MASTER #指定 keepalived 的角色，必须大写 可选值：MASTER|BACKUPinterface ens33 #网卡设置，lvs需要绑定在网卡上，realserver绑定在回环口。区别：lvs对访问为外，realserver为内不易暴露本机信息virtual_router_id 51 #虚拟路由标识，是一个数字，同一个vrrp 实例使用唯一的标识，MASTER和BACKUP 的 同一个 vrrp_instance 下 这个标识必须保持一致priority 100 #定义优先级，数字越大，优先级越高。advert_int 1 #设定 MASTER 与 BACKUP 负载均衡之间同步检查的时间间隔，单位为秒，两个节点设置必须一样authentication { #设置验证类型和密码，两个节点必须一致auth_type PASSauth_pass 1111}virtual_ipaddress { #设置虚拟IP地址，可以设置多个虚拟IP地址，每行一个192.168.119.130}track_script { #脚本监控状态chk_nginx_service #可加权重，但会覆盖声明的脚本权重值。chk_nginx_service weight -20}notify_master "/etc/keepalived/start_haproxy.sh start" #当前节点成为master时，通知脚本执行任务notify_backup "/etc/keepalived/start_haproxy.sh stop" #当前节点成为backup时，通知脚本执行任务notify_fault "/etc/keepalived/start_haproxy.sh stop" #当当前节点出现故障，执行的任务;
}## 虚拟服务器virtual_server定义块 ，虚拟服务器定义是keepalived框架最重要的项目了，是keepalived.conf必不可少的部分。 该部分是用来管理LVS的，是实现keepalive和LVS相结合的模块。
virtual_server 192.168.119.130 80 { #定义RealServer对应的VIP及服务端口，IP和端口之间用空格隔开delay_loop 6 #每隔6秒查询realserver状态lb_algo rr #后端调试算法（load balancing algorithm）lb_kind DR #LVS调度类型NAT/DR/TUN#persistence_timeout 60 同一IP的连接60秒内被分配到同一台realserverprotocol TCP #用TCP协议检查realserver状态real_server 192.168.119.120 80 {weight 1 #权重，最大越高，lvs就越优先访问TCP_CHECK { #keepalived的健康检查方式HTTP_GET | SSL_GET | TCP_CHECK | SMTP_CHECK | MISCconnect_timeout 10 #10秒无响应超时retry 3 #重连次数3次delay_before_retry 3 #重连间隔时间connect_port 80 #健康检查realserver的端口}}real_server 192.168.119.121 80 {weight 1 #权重，最大越高，lvs就越优先访问TCP_CHECK { #keepalived的健康检查方式HTTP_GET | SSL_GET | TCP_CHECK | SMTP_CHECK | MISCconnect_timeout 10 #10秒无响应超时retry 3 #重连次数3次delay_before_retry 3 #重连间隔时间connect_port 80 #健康检查realserver的端口}}
}
4-1：虚拟服务器转发
! Configuration File for keepalivedglobal_defs {router_id db01
}vrrp_instance VI_1 {state BACKUPinterface ens33virtual_router_id 51priority 100nopreemptadvert_int 1authentication {auth_type PASSauth_pass 1111}virtual_ipaddress {192.168.132.40}
}
virtual_server 192.168.11.40 3306 { #定义了一个虚拟服务器，监听 IP 地址为 192.168.11.40，端口为 3306（MySQL 默认端口）delay_loop 6 #设置检查间隔为 6 秒persistence_timeout 50 #设置持久性失败计数器超时时间为 50protocol TCPreal_server 192.168.11.43 3306 { #定义了一个真实服务器，即要监控的 MySQL 服务所在的服务器notify_down /etc/keepalived/kill_keepalived.sh #当检测到真实服务器不可用时，执行 /etc/keepalived/kill_keepalived.sh 脚本，通常用于触发切换操作TCP_CHECK {connect_timeout 3 #连接超时时间为 3 秒nb_get_retry 3 #最大重试次数为 3 次connect_port 3306 #连接的端口为 3306delay_before_retry 3 #重试前等待 3 秒}}
}
