keepalived2.2.8+drbd9+nfs高可用存储部署

一.本文基于上一篇文章keepalived环境来做的，主机信息如下

二.为两台虚拟机准备添加一块新硬盘设备

三.安装drbd9

1.使用扩展源的rpm包来下载

2.创建资源并挂载到新增的硬盘

3.主设备升级身份

4.主备两个设备手动切换身份演示

四.安装配置nfs

五.安装keeepalived

六.编辑keepalived.conf配置文件

1.主设备配置文件如下

2.备设备配置文件如下

3.脚本文件

七.测试

1.主设备nfs掉点，查看keepalived调用探测脚本是否能够将nfs重新启动

2.主设备keepalived掉点，查看VIP和挂载目录是否能够飘逸到备设备

一.本文基于上一篇文章keepalived环境来做的，主机信息如下

主keepalived+drbd：192.168.2.130（main）

备keepalived+drbd：192.168.2.133（serverc）

VIP：192.168.2.100

二.为两台虚拟机准备添加一块新硬盘设备

关机状态下进行（开机状态下新增后需要重启）如果没有特殊名称要求，直接都点下一步即可，两台设备都做

[root@main ~]# lsblk 
NAME            MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda               8:0    0   20G  0 disk 
├─sda1            8:1    0    1G  0 part /boot
└─sda2            8:2    0   19G  0 part ├─centos-root 253:0    0   17G  0 lvm  /└─centos-swap 253:1    0    2G  0 lvm  
sdb               8:16   0   20G  0 disk    #刚才新增的
sr0              11:0    1  4.4G  0 rom  /cdrom

三.安装drbd9

1.使用扩展源的rpm包来下载

主备都做

[root@main ~]# rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org && rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm && yum install gcc gcc-c++ make glibc flex kernel* -y && yum install -y drbd90-utils kmod-drbd90 && modprobe drbd && echo drbd > /etc/modules-load.d/drbd.conf && lsmod | grep drbd
drbd                  654854  0 
libcrc32c              12644  5 xfs,drbd,ip_vs,nf_nat,nf_conntrack
[root@main drbd.d]# systemctl start drbd   #启动drbd

[root@main ~]# tree /etc/drbd.d  #配置文件和资源存放路径如下
/etc/drbd.d
├── global_common.conf
└── r0.res

0 directories, 2 files
[root@main ~]# cat /etc/drbd.conf 
# You can find an example in  /usr/share/doc/drbd.../drbd.conf.example

include "drbd.d/global_common.conf";
include "drbd.d/*.res";

2.创建资源并挂载到新增的硬盘

[root@main drbd.d]# cat r0.res #创建r0资源，主备都做，可以拷贝进行
resource r0 {disk /dev/sdb;device /dev/drbd0;meta-disk internal;on main {address 192.168.2.130:7789;}on serverc {address 192.168.2.133:7789;}
}
[root@main drbd.d]# drbdadm create-md r0
initializing activity log
initializing bitmap (640 KB) to all zero
Writing meta data...
New drbd meta data block successfully created.
[root@main drbd.d]# drbdadm up r0

3.主设备升级身份

刚安装完成两台设备应该都是secondary，主备手动升级身份

初始化过程在9版本用watch drbdadm status 资源名称看

[root@main drbd.d]# drbdadm primary r0 --force
[root@main drbd.d]# mkfs.xfs /dev/drbd0  #主做备不做
[root@main drbd.d]# mkdir /data   
[root@main drbd.d]# mount /dev/drbd0 /data   
[root@main drbd.d]# drbdadm status r0    #查看状态，只查看角色可以使用drbdadm role r0
r0 role:Primarydisk:UpToDateserverc role:Secondarypeer-disk:UpToDate
[root@serverc ~]# drbdadm status r0   #备设备的状态
r0 role:Secondarydisk:UpToDatemain role:Primarypeer-disk:UpToDate[root@main ~]# mkfs.xfs /dev/drbd0
meta-data=/dev/drbd0             isize=512    agcount=4, agsize=1310678 blks=                       sectsz=512   attr=2, projid32bit=1=                       crc=1        finobt=0, sparse=0
data     =                       bsize=4096   blocks=5242711, imaxpct=25=                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal log           bsize=4096   blocks=2560, version=2=                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0
[root@main ~]# mount /dev/drbd0 /data   #主设备查看挂载情况，写入测试文件
[root@main ~]# df -h | grep data
/dev/drbd0                20G   33M   20G   1% /data
[root@main drbd.d]# ll /data
total 0
[root@main drbd.d]# echo hello > /data/a.txt
[root@main drbd.d]# ll /data/
total 4
-rw-r--r-- 1 root root 6 Mar 29 21:53 a.txt
[root@main drbd.d]# cat /data/a.txt 
hello

4.主备两个设备手动切换身份演示

先在主备降级并卸载后到备设备升级挂载

[root@main ~]# drbdadm role r0
Primary
[root@main ~]# umount /data
[root@main ~]# drbdadm secondary r0
[root@main ~]# drbdadm role r0
Secondary

[root@serverc ~]# drbdadm role r0
Secondary
[root@serverc ~]# drbdadm primary r0
[root@serverc ~]# mount /dev/drbd0 /data
[root@serverc ~]# drbdadm role r0
Primary
#在日志中是可以清除查看到这个身份转换过程的
#tail -f /var/log/messages
Mar 30 13:38:53 main kernel: XFS (drbd0): Unmounting Filesystem
Mar 30 13:39:05 main kernel: drbd r0: Preparing cluster-wide state change 1582556808 (1->-1 3/2)
Mar 30 13:39:05 main kernel: drbd r0: State change 1582556808: primary_nodes=0, weak_nodes=0
Mar 30 13:39:05 main kernel: drbd r0: Committing cluster-wide state change 1582556808 (1ms)
Mar 30 13:39:05 main kernel: drbd r0: role( Primary -> Secondary ) [secondary]
Mar 30 13:39:35 main kernel: drbd r0 serverc: Preparing remote state change 1834794712
Mar 30 13:39:35 main kernel: drbd r0 serverc: Committing remote state change 1834794712 (primary_nodes=1)
Mar 30 13:39:35 main kernel: drbd r0 serverc: peer( Secondary -> Primary ) [remote]

四.安装配置nfs

主备都做

[root@main drbd.d]# yum install -y rpcbind nfs-utils
[root@main drbd.d]# vim /etc/exports
[root@main ~]# cat /etc/exports
/data 192.168.2.0/24(rw,sync,no_root_squash)
[root@serverc ~]# cat /etc/exports
/data 192.168.2.0/24(rw,sync,no_root_squash)

systemctl start rpcbind nfs

五.安装keeepalived

参考上一篇文章部署

六.编辑keepalived.conf配置文件

1.主设备配置文件如下

[root@main keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {notification_email {acassen@firewall.locfailover@firewall.locsysadmin@firewall.loc}notification_email_from Alexandre.Cassen@firewall.locsmtp_server 192.168.2.130smtp_connect_timeout 30router_id LVS_DEVEL1vrrp_skip_check_adv_addrvrrp_strictvrrp_garp_interval 0vrrp_gna_interval 0
}

vrrp_script chk_nfs {    #这个要写在引用部分前，不然引用不上这个脚本，在这还是耽误了挺长时间，还有后面track_script及后面三个部分的顺序好像也是有影响，可以自己具体实验一下script "/etc/keepalived/chk_nfs.sh"interval 2  #每隔多少秒检测weight -40   
}

vrrp_instance VI_1 {state MASTER   #主interface ens33virtual_router_id 51mcast_src_ip 192.168.2.130priority 100advert_int 1nopreempt   #非抢占模式，优先级生效authentication {auth_type PASSauth_pass 123456}track_script {   #引入验证块chk_nfs}notify_master /etc/keepalived/notify_master.sh   #以下是几个日志脚本，稍后写notify_backup /etc/keepalived/notify_backup.shnotify_fault /etc/keepalived/notify_fault.shvirtual_ipaddress {   #VIP192.168.2.100}
}

2.备设备配置文件如下

[root@serverc keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {notification_email {acassen@firewall.locfailover@firewall.locsysadmin@firewall.loc}notification_email_from Alexandre.Cassen@firewall.locsmtp_server 192.168.200.1smtp_connect_timeout 30router_id LVS_DEVEL2vrrp_skip_check_adv_addrvrrp_strictvrrp_garp_interval 0vrrp_gna_interval 0
}
vrrp_script chk_nfs { script "/etc/keepalived/chk_nfs.sh"interval 2weught 40
}

vrrp_instance VI_1 {state BACKUP   #从interface ens33virtual_router_id 51priority 80   #优先级低于主advert_int 1nopreemptauthentication {auth_type PASSauth_pass 123456}track_script {chk_nfs}notify_master /etc/keepalived/notify_master.shnotify_backup /etc/keepalived/notify_backup.shnotify_fault /etc/keepalived/notify_fault.shvirtual_ipaddress {192.168.2.100}
}

3.脚本文件

主备都做

[root@main keepalived]# cat chk_nfs.sh    #nfs检测脚本
#!/bin/bash
/usr/bin/systemctl status nfs
if [ $? -ne 0 ]     #校测到没有启动就尝试启动nfs
thensystemctl start nfsif [ $? -ne 0 ]then    #如果仍然启动不了就卸载，降级，停止keealived服务umount -v /dev/drbd0drbdadm secondary r0systemctl stop keepalivedfi
fi[root@main keepalived]# cat notify_master.sh 
#!/bin/bash
time=`date "+%F  %H:%M:%S"`   #日志时间格式
log=/etc/keepalived/logs   
echo -e "$time    ------notify_master------\n" >> $log/notify_master.log
drbdadm primary r0 &>> $log/notify_master.log    #当为主设备时先升级，挂载，重启nfs
mount -v /dev/drbd0 /data &>> $log/notify_master.log
systemctl restart nfs &>> $log/notify_master.log
echo -e "\n" >> $log/notify_master.log[root@main keepalived]# cat notify_backup.sh 
#!/bin/bash
time=`date "+%F %H:%M:%S"`
log=/etc/keepalived/logs
echo -e "$time ------notify_backup------\n" >> $log/notify_backup.log
systemctl stop nfs &>> $log/notify_backup.log   #当为备设备时，先停止nfs，卸载，降级
umount -v /dev/drbd0 &>> $log/notify_backup.log
drbdadm secondary r0 &>> $log/notify_backup.log
echo -e "\n" >> $log/notify_backup.log[root@main keepalived]# cat notify_fault.sh      
#!/bin/bash
time=`date "+%F  %H:%M:%S"`
echo -e "$time  ------notify_fault------\n" >> /etc/keepalived/logs/notify_fault.log
systemctl stop nfs &>> /etc/keepalived/logs/notify_fault.log
umount -v /dev/drbd0 &>> /etc/keepalived/logs/notify_fault.log
drbdadm secondary r0 &>> /etc/keepalived/logs/notify_fault.log
echo -e "\n" >> /etc/keepalived/logs/notify_fault.logchmod +x *.sh

七.测试

1.主设备nfs掉点，查看keepalived调用探测脚本是否能够将nfs重新启动

[root@main keepalived]# systemctl stop nfs
[root@main keepalived]# systemctl status nfs
● nfs-server.service - NFS server and servicesLoaded: loaded (/usr/lib/systemd/system/nfs-server.service; disabled; vendor preset: disabled)Drop-In: /run/systemd/generator/nfs-server.service.d└─order-with-mounts.confActive: active (exited) since Sat 2024-03-30 14:15:02 CST; 5s agoProcess: 82157 ExecStopPost=/usr/sbin/exportfs -f (code=exited, status=0/SUCCESS)Process: 82153 ExecStopPost=/usr/sbin/exportfs -au (code=exited, status=0/SUCCESS)Process: 82151 ExecStop=/usr/sbin/rpc.nfsd 0 (code=exited, status=0/SUCCESS)Process: 82283 ExecStartPost=/bin/sh -c if systemctl -q is-active gssproxy; then systemctl reload gssproxy ; fi (code=exited, status=0/SUCCESS)Process: 82266 ExecStart=/usr/sbin/rpc.nfsd $RPCNFSDARGS (code=exited, status=0/SUCCESS)Process: 82264 ExecStartPre=/usr/sbin/exportfs -r (code=exited, status=0/SUCCESS)Main PID: 82266 (code=exited, status=0/SUCCESS)Tasks: 0Memory: 0BCGroup: /system.slice/nfs-server.serviceMar 30 14:15:02 main systemd[1]: Starting NFS server and services...
Mar 30 14:15:02 main systemd[1]: Started NFS server and services.
[root@main keepalived]# df -h |grep data
/dev/drbd0                20G   33M   20G   1% /data#具体日志文件如下
Mar 30 14:15:00 main systemd: Stopping NFS server and services...
Mar 30 14:15:00 main systemd: Stopped NFS server and services.
Mar 30 14:15:00 main systemd: Stopping NFSv4 ID-name mapping service...
Mar 30 14:15:00 main systemd: Stopping NFS Mount Daemon...
Mar 30 14:15:00 main rpc.mountd[65956]: Caught signal 15, un-registering and exiting.
Mar 30 14:15:00 main systemd: Stopped NFSv4 ID-name mapping service.
Mar 30 14:15:00 main kernel: nfsd: last server has exited, flushing export cache
Mar 30 14:15:00 main systemd: Stopped NFS Mount Daemon.
Mar 30 14:15:02 main systemd: Starting Preprocess NFS configuration...
Mar 30 14:15:02 main systemd: Started Preprocess NFS configuration.
Mar 30 14:15:02 main systemd: Starting NFSv4 ID-name mapping service...
Mar 30 14:15:02 main systemd: Starting NFS Mount Daemon...
Mar 30 14:15:02 main systemd: Started NFSv4 ID-name mapping service.
Mar 30 14:15:02 main rpc.mountd[82263]: Version 1.3.0 starting
Mar 30 14:15:02 main systemd: Started NFS Mount Daemon.
Mar 30 14:15:02 main systemd: Starting NFS server and services...
Mar 30 14:15:02 main kernel: NFSD: starting 90-second grace period (net ffffffffb0b16200)
Mar 30 14:15:02 main systemd: Reloading GSSAPI Proxy Daemon.
Mar 30 14:15:02 main systemd: Reloaded GSSAPI Proxy Daemon.
Mar 30 14:15:02 main systemd: Started NFS server and services.
Mar 30 14:15:02 main systemd: Starting Notify NFS peers of a restart...
Mar 30 14:15:02 main sm-notify[82289]: Version 1.3.0 starting
Mar 30 14:15:02 main sm-notify[82289]: Already notifying clients; Exiting!
Mar 30 14:15:02 main systemd: Started Notify NFS peers of a restart.

2.主设备keepalived掉点，查看VIP和挂载目录是否能够飘逸到备设备

[root@main keepalived]# systemctl stop keepalived.service
[root@serverc keepalived]# ip a |grep ens33 -A1   #VIP已漂移
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000link/ether 00:0c:29:2b:95:b3 brd ff:ff:ff:ff:ff:ffinet 192.168.2.133/24 brd 192.168.2.255 scope global noprefixroute ens33valid_lft forever preferred_lft foreverinet 192.168.2.100/32 scope global ens33valid_lft foreve preferred_lft forever
[root@serverc keepalived]# df -h   #挂载漂移
Filesystem               Size  Used Avail Use% Mounted on
devtmpfs                 2.9G     0  2.9G   0% /dev
tmpfs                    2.9G     0  2.9G   0% /dev/shm
tmpfs                    2.9G   20M  2.9G   1% /run
tmpfs                    2.9G     0  2.9G   0% /sys/fs/cgroup
/dev/mapper/centos-root   17G  4.1G   13G  25% /
/dev/sda1               1014M  272M  743M  27% /boot
/dev/sr0                 4.4G  4.4G     0 100% /cdrom
tmpfs                    585M     0  585M   0% /run/user/0
/dev/drbd0                20G   33M   20G   1% /data
#日志情况如下
Mar 30 15:08:13 main Keepalived_vrrp[116817]:   Reset promote_secondaries counter 0
Mar 30 15:08:13 main Keepalived_vrrp[116817]: (VI_1) Entering BACKUP STATE (init)
Mar 30 15:08:13 main Keepalived_vrrp[116817]: VRRP sockpool: [ifindex(  2), family(IPv4), proto(112), fd(12,13) multicast, address(224.0.0.18)]
Mar 30 15:08:13 main Keepalived_vrrp[116817]: VRRP_Script(chk_nfs) succeeded
Mar 30 15:08:13 main rpc.mountd[114695]: Caught signal 15, un-registering and exiting.
Mar 30 15:08:13 main kernel: drbd r0 serverc: Preparing remote state change 1490261391
Mar 30 15:08:13 main kernel: drbd r0 serverc: Committing remote state change 1490261391 (primary_nodes=1)
Mar 30 15:08:13 main kernel: drbd r0 serverc: peer( Secondary -> Primary ) [remote]
Mar 30 15:08:13 main kernel: drbd r0/0 drbd0: Disabling local AL-updates ([root@serverc keepalived]# drbdadm status r0   #serverc此时变为主，#main变为备
r0 role:Primarydisk:UpToDatemain role:Secondarypeer-disk:UpToDate