二阶段测试：
架构：
| 服务器类型 | 部署组件 | ip地址 |
|---|---|---|
| DR1调度服务器 主（ha01） | Keepalived+LVS-DR | 192.168.60.30 |
| DR2调度服务器 备 (ha02) | Keepalived+LVS-DR | 192.168.60.40 |
| web1节点服务器 (slave01) | Nginx+Tomcat+MySQL 备+MHA manager+MHA node | 192.168.60.91 |
| web2节点服务器 (slave02) | Nginx+Tomcat +MySQL 备+MHA node | 192.168.60.92 |
| NFS存储服务器（master01） | MySQL 主+NFS+MHA node | 192.168.60.93 |
| vip | 虚拟ip | 192.168.60.100 |
一、关闭防护墙并修改主机名
[root@localhost ~]# systemctl stop firewalld.service && setenforce 0 [root@ha01 ~]# hostnamectl set-hostname ha01 [root@ha02 ~]# hostnamectl set-hostname ha02 [root@slave1 ~]# hostnamectl set-hostname slave1 [root@slave2 ~]# hostnamectl set-hostname slave2 [root@master ~]# hostnamectl set-hostname master [root@localhost ~]# su
二、调度器ha01和ha02安装ipvsadm
[root@ha01 net]# yum -y install ipvsadm
三、部署lvs_DR
1、在ha01和ha02上配置虚拟网卡
[root@ha01 net]# cd /etc/sysconfig/network-scripts/ [root@ha01 network-scripts]# vim ifcfg-ens33:0 DEVICE=ens33:0 ONBOOT=yes IPADDR=192.168.60.100 NETMASK=255.255.255.255 [root@ha01 network-scripts]# ifup ifcfg-ens33:0
2、更改调度器的响应参数
[root@ha01 network-scripts]# vim /etc/sysctl.conf net.ipv4.ip_forward=0 #关闭数据包转发功能 net.ipv4.conf.all.send_redirects=0 #禁止系统发送icmp重定向的消息。 net.ipv4.conf.default.send_redirects=0 #禁止默认网络接口发送ICMP重定向的消息 net.ipv4.conf.ens33.send_redirects=0 #针对ens33设备，禁止发送ICMP重定向消息 [root@ha01 network-scripts]# sysctl -p net.ipv4.ip_forward = 0 net.ipv4.conf.all.send_redirects = 0 net.ipv4.conf.default.send_redirects = 0 net.ipv4.conf.ens33.send_redirects = 0
3、创建多个逻辑网络接口
[root@ha01 network-scripts]# route add -host 192.168.60.100 dev ens33:0 [root@ha01 network-scripts]# vim /etc/rc.local /usr/sbin/route add -host 192.168.60.100 dev ens33:0
4、配置负载均衡分配策略
[root@ha01 network-scripts]# ipvsadm-save > /etc/sysconfig/ipvsadm #保持策略 [root@ha01 network-scripts]# systemctl start ipvsadm.service #重启ipvsadm服务 [root@ha01 network-scripts]# ipvsadm -C #清空策略 [root@ha01 network-scripts]# ipvsadm -A -t 192.168.60.100:80 -s rr [root@ha01 network-scripts]# ipvsadm -a -t 192.168.60.100:80 -r 192.168.60.92:80 -g [root@ha01 network-scripts]# ipvsadm -a -t 192.168.60.100:80 -r 192.168.60.93:80 -g [root@ha01 network-scripts]# ipvsadm IP Virtual Server version 1.2.1 (size=4096) Prot LocalAddress:Port Scheduler Flags-> RemoteAddress:Port Forward Weight ActiveConn InActConn TCP ha01:http rr-> 192.168.60.92:http Route 1 0 0 -> 192.168.60.93:http Route 1 0 0
5、配置两台slave的web节点
5.1、配置虚拟ip地址
[root@slave1 opt]# cd /etc/sysconfig/network-scripts/ [root@slave1 network-scripts]# cp ifcfg-ens33 ifcfg-lo:0 [root@slave1 network-scripts]# vim ifcfg-lo:0 DEVICE=lo:0 ONBOOT=yes IPADDR=192.168.60.100 NETMASK=255.255.255.255 [root@slave1 network-scripts]# ifup ifcfg-lo:0
5.2、配置添加路由
[root@slave2 network-scripts]# route add -host 192.168.60.100 dev lo:0 [root@slave2 network-scripts]# vim /etc/rc.local route add -host 192.168.60.100 dev lo:0
5.3、配置ARP内核响应参数防止更新VIP中的MAC地址，避免发生冲突
[root@slave1 network-scripts]# vim /etc/sysctl.conf net.ipv4.conf.lo.arp_ignore = 1 net.ipv4.conf.lo.arp_announce = 2 net.ipv4.conf.all.arp_ignore = 1 net.ipv4.conf.all.arp_announce = 2 [root@slave1 network-scripts]# sysctl -p
5.4、在master上部署NFS存储服务器
[root@master opt]# rpm -q rpcbind nfs-utils #检查是否安装nfs rpcbind-0.2.0-47.el7.x86_64 nfs-utils-1.3.0-0.61.el7.x86_64 [root@master opt]# systemctl start nfs [root@master opt]# systemctl start rpcbind #开启服务 [root@master opt]# systemctl enable nfs #设置开机自启 [root@master opt]# systemctl enable rpcbind [root@master opt]# mkdir /opt/web1 /opt/web2 [root@master opt]# echo '<h1>This is node web1</h1>' > /opt/web1/index.html [root@master opt]# echo '<h1>This is node web2</h1>' > /opt/web2/index.html #添加网页内容
[root@master opt]# vim /etc/exports /opt/web1 192.168.60.0/24(ro) /opt/web2 192.168.60.0/24(ro) [root@master opt]# exportfs -rv exporting 192.168.60.0/24:/opt/web2 exporting 192.168.60.0/24:/opt/web1
6、创建软连接并开启nginx服务
[root@slave1 nginx]# ln -s /usr/local/nginx/sbin/nginx /usr/local/sbin/ [root@slave1 nginx]# nginx [root@slave1 nginx]# netstat -antp | grep 80 tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 25085/nginx: master
7、节点服务器挂载共享目录（slave1 slave2分别挂载/opt/web1 /opt/web2）
[root@slave1 nginx]# cd / [root@slave1 /]# showmount -e 192.168.60.91 Export list for 192.168.60.91: /opt/web2 192.168.60.0/24 /opt/web1 192.168.60.0/24
slave1：
[root@slave1 /]# mount.nfs 192.168.60.91:/opt/web1 /usr/local/nginx/html/ [root@slave1 /]# cd /usr/local/nginx/html/ [root@slave1 html]# cat index.html <h1>This is node web1</h1>
slave2：
[root@slave2 /]# mount.nfs 192.168.60.91:/opt/web2 /usr/local/nginx/html/ [root@slave2 /]# cd /usr/local/nginx/html/ [root@slave2 html]# cat index.html <h1>This is node web2</h1>
8、测试lvs_DR模式
[root@master opt]# curl 192.168.60.100 <h1>This is node web2</h1> [root@master opt]# curl 192.168.60.100 <h1>This is node web1</h1> [root@master opt]# curl 192.168.60.100 <h1>This is node web2</h1> [root@master opt]# curl 192.168.60.100 <h1>This is node web1</h1> [root@master opt]# curl 192.168.60.100 <h1>This is node web2</h1>
四、部署Nginx+Tomcat的动静分离（两个slave都装）
1、安装tomcat
[root@slave1 html]# cd /opt/ [root@slave1 opt]# rz -E rz waiting to receive. [root@slave1 opt]# ls apache-tomcat-9.0.16.tar.gz
#写一个安装tomcat的脚本 [root@slave1 opt]# vim tomcat.sh #!/bin/bash#安装部署tomcat systemctl stop firewalld setenforce 0 #安装JDK cd /opt rpm -ivh jdk-8u201-linux-x64.rpm &> /dev/null java -version #设置JDK环境变量 cat > /etc/profile.d/java.sh <<EOF export JAVA_HOME=/usr/java/jdk1.8.0_201-amd64 export CLASSPATH=.:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar export PATH=$JAVA_HOME/bin:$PATH EOF source /etc/profile.d/java.sh if [ $? -eq 0 ];then echo -e "\033[34;1m JDK安装完成！ \033[0m" fi java -version #安装启动Tomcat cd /opt tar zxvf apache-tomcat-9.0.16.tar.gz &> /dev/null mv apache-tomcat-9.0.16 /usr/local/tomcat ##启动tomcat /usr/local/tomcat/bin/startup.sh if [ $? -eq 0 ];then echo -e "\033[34;1m tomcat安装完成！ \033[0m" fi
[root@slave1 opt]# chmod +x tomcat.sh #给脚本赋执行权 [root@slave2 opt]# ./tomcat.sh #执行脚本
2、动静分离配置
slave1：
[root@slave1 opt]# mkdir /usr/local/tomcat/webapps/test
[root@slave1 opt]# cd /usr/local/tomcat/webapps/
[root@slave1 webapps]# cd test/
[root@slave1 test]# vim /usr/local/tomcat/webapps/test/index.jsp
<%@ page language="java" import="java.util.*" pageEncoding="UTF-8" %>
<html>
<head>
<title>tomcat1</title>
</head>
<body>
<% out.println("This is tomcat1 server");%>
<div>动态页面1</div><br/>
</body>
</html>
[root@slave1 test]# cd /usr/local/tomcat/conf/
slave2：
[root@slave2 opt]# mkdir /usr/local/tomcat/webapps/test
[root@slave2 opt]# cd /usr/local/tomcat/webapps/
[root@slave2 webapps]# cd test/
[root@slave2 test]# vim /usr/local/tomcat/webapps/test/index.jsp
<%@ page language="java" import="java.util.*" pageEncoding="UTF-8" %>
<html>
<head>
<title>tomcat2</title>
</head>
<body>
<% out.println("This is tomcat2 server");%>
<div>动态页面2</div><br/>
</body>
</html>
[root@slave2 test]# cd /usr/local/tomcat/conf/
3、配置tomcatweb界面配置
[root@slave2 conf]# cd /usr/local/tomcat/conf/
[root@slave2 conf]# cp server.xml{,.bak}
[root@slave2 conf]# vim server.xml
#修改148行
<Host name="localhost" appBase="webapps"unpackWARs="true" autoDeploy="true" xmlValidation="false" xmlNamespaceAware="false"><Context docBase="/usr/local/tomcat/webapps/test" path="" reloadable="true" />
4、开启8080端口
[root@slave1 opt]# cd /usr/local/tomcat/bin/ [root@slave1 bin]# ./shutdown.sh Using CATALINA_BASE: /usr/local/tomcat Using CATALINA_HOME: /usr/local/tomcat Using CATALINA_TMPDIR: /usr/local/tomcat/temp Using JRE_HOME: /usr/java/jdk1.8.0_201-amd64 Using CLASSPATH: /usr/local/tomcat/bin/bootstrap.jar:/usr/local/tomcat/bin/tomcat-juli.jar [root@slave1 bin]# ./startup.sh Using CATALINA_BASE: /usr/local/tomcat Using CATALINA_HOME: /usr/local/tomcat Using CATALINA_TMPDIR: /usr/local/tomcat/temp Using JRE_HOME: /usr/java/jdk1.8.0_201-amd64 Using CLASSPATH: /usr/local/tomcat/bin/bootstrap.jar:/usr/local/tomcat/bin/tomcat-juli.jar Tomcat started. [root@slave1 bin]# netstat -antp | grep 8080 tcp6 0 0 :::8080 :::* LISTEN 29638/java tcp6 0 0 ::1:42910 ::1:8080 TIME_WAIT -
5、Nginx server 配置两个slave都配置
[root@slave1 bin]# cd /usr/local/nginx/conf/
[root@slave1 conf]# cp nginx.conf{,.bak}
[root@slave1 conf]# vim nginx.conf#gzip on;upstream tomcat_server {server 192.168.60.92:8080 weight=1;server 192.168.60.93:8080 weight=1;}
server_name www.web1.com;
charset utf-8;
location ~ .*.jsp$ {proxy_pass http://tomcat_server;proxy_set_header HOST $host;proxy_set_header X-Real-IP $remote_addr;proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;}
location ~ .*\.(gif|jpg|jpeg|png|bmp|swf|css)$ {root /usr/local/nginx/html;expires 10d;
}
五、 配置keeplived
1、主服务器ha01
[root@ha01 ~]# yum install ipvsadm keepalived -y
[root@ha01 keepalived]# cp keepalived.conf{,.bak}
[root@ha01 keepalived]# vim keepalived.conf
[root@ha01 keepalived]# systemctl start keepalived.service [root@ha01 keepalived]# ip addr
2、备服务器 ha02
[root@ha02 ~]# yum install ipvsadm keepalived -y [root@ha02 ~]# cd /etc/keepalived/ [root@ha02 keepalived]# vim keepalived.conf
[root@ha02 keepalived]# systemctl start keepalived.service
3、调整内核 proc 响应参数
[root@ha02 keepalived]# vim /etc/sysctl.confnet.ipv4.conf.all.send_redirects = 0net.ipv4.conf.default.send_redirects = 0net.ipv4.conf.ens33.send_redirects = 0[root@ha02 keepalived]# sysctl -p
六、部署MySQL集群MHA高可用
1、修改 Master、Slave1、Slave2 节点的 Mysql主配置文件/etc/my.cnf
主： [root@master ~]# vim /etc/my.cnf server-id = 1 log_bin = master-bin log-slave-updates = true relay_log_recovery = 1 systemctl restart mysqld
从1： vim /etc/my.cnf server-id = 2 #三台服务器的 server-id 不能一样 log_bin = master-bin relay-log = relay-log-bin relay-log-index = slave-relay-bin.index relay_log_recovery = 1 systemctl restart mysqld
从2： vim /etc/my.cnf #三台服务器的 server-id 不能一样 server-id = 3 relay-log = relay-log-bin relay-log-index = slave-relay-bin.index relay_log_recovery = 1 systemctl restart mysqld
2、在 Master、Slave1、Slave2 节点上都创建两个软链接
ln -s /usr/local/mysql/bin/mysql /usr/sbin/ ln -s /usr/local/mysql/bin/mysqlbinlog /usr/sbin/
3、配置 mysql 一主两从
（1）所有数据库节点进行 mysql 授权
mysql -uroot -p123456 #从数据库同步使用 CREATE USER 'myslave'@'192.168.60.%' IDENTIFIED WITH mysql_native_password BY '123456'; GRANT REPLICATION SLAVE ON *.* TO 'myslave'@'192.168.60.%'; #manager 使用 CREATE USER 'mha'@'192.168.60.%' IDENTIFIED WITH mysql_native_password BY 'manager'; GRANT ALL PRIVILEGES ON *.* TO 'mha'@'192.168.60.%' WITH GRANT OPTION;#防止从库通过主机名连接不上主库 CREATE USER 'mha'@'master' IDENTIFIED WITH mysql_native_password BY 'manager'; GRANT ALL PRIVILEGES ON *.* TO 'mha'@'master'; CREATE USER 'mha'@'slave1' IDENTIFIED WITH mysql_native_password BY 'manager'; GRANT ALL PRIVILEGES ON *.* TO 'mha'@'slave1'; CREATE USER 'mha'@'slave2' IDENTIFIED WITH mysql_native_password BY 'manager'; GRANT ALL PRIVILEGES ON *.* TO 'mha'@'slave2'; flush privileges;
（2）在 Master 节点查看二进制文件和同步点
show master status; +-------------------+----------+--------------+------------------+-------------------+ | File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set | +-------------------+----------+--------------+------------------+-------------------+ | master-bin.000001 | 1747 | | | | +-------------------+----------+--------------+------------------+-------------------+
（3）在 Slave1、Slave2 节点执行同步操作
change master to master_host='192.168.60.91',master_user='myslave',master_password='123456',master_log_file='master-bin.000001',master_log_pos=2917; start slave;
（4）在 Slave1、Slave2 节点查看数据同步结果
show slave status\G;//确保 IO 和 SQL 线程都是 Yes，代表同步正常。 Slave_IO_Running: Yes Slave_SQL_Running: Yes
（5）两个从库必须设置为只读模式：
set global read_only=1;
（6）插入数据测试数据库同步
##在 Master 主库插入条数据，测试是否同步 create database test_db; use test_db; create table test(id int); insert into test(id) values (1);
7、安装 MHA 软件
（1）所有服务器上都安装 MHA 依赖的环境，首先安装 epel 源
yum install epel-release --nogpgcheck -y yum install -y perl-DBD-MySQL \ perl-Config-Tiny \ perl-Log-Dispatch \ perl-Parallel-ForkManager \ perl-ExtUtils-CBuilder \ perl-ExtUtils-MakeMaker \ perl-CPAN
（2）安装 MHA 软件包，先在所有服务器上必须先安装 node 组件
对于每个操作系统版本不一样，这里 CentOS7.6选择 0.57 版本。 在所有服务器上必须先安装 node 组件，最后在 MHA-manager 节点上安装 manager 组件， 因为 manager 依赖 node 组件。
cd /opt tar zxvf mha4mysql-node-0.57.tar.gz cd mha4mysql-node-0.57 perl Makefile.PL make && make install
（3）在 MHA manager 节点上安装 manager 组件
cd /opt tar zxvf mha4mysql-manager-0.57.tar.gz cd mha4mysql-manager-0.57 perl Makefile.PL make && make install
8、在所有服务器上配置无密码认证
（1）在 manager 节点上配置到所有数据库节点的无密码认证
ssh-keygen -t rsa #一路按回车键 ssh-copy-id 192.168.60.91 ssh-copy-id 192.168.60.92 ssh-copy-id 192.168.60.93
（2）在 master 上配置到数据库节点 slave1 和 slave2 的无密码认证
ssh-keygen -t rsa ssh-copy-id 192.168.60.92 ssh-copy-id 192.168.60.93
（3）在 slave1 上配置到数据库节点 master 和 slave2 的无密码认证
ssh-keygen -t rsa ssh-copy-id 192.168.60.91 ssh-copy-id 192.168.60.93
（4）在 slave2 上配置到数据库节点 master 和 slave1 的无密码认证
ssh-keygen -t rsa ssh-copy-id 192.168.60.91 ssh-copy-id 192.168.60.92
9、在 manager 节点上配置 MHA
（1）在 manager 节点上复制相关脚本到/usr/local/bin 目录
cp -rp /opt/mha4mysql-manager-0.57/samples/scripts /usr/local/bin
（2）复制上述的自动切换时 VIP 管理的脚本到 /usr/local/bin 目录，
这里使用master_ip_failover脚本来管理 VIP 和故障切换
cp /usr/local/bin/scripts/master_ip_failover /usr/local/bin
（3）修改内容如下：（删除原有内容，直接复制并修改vip相关参数）
vim /usr/local/bin/master_ip_failover
#!/usr/bin/env perl
use strict;
use warnings FATAL => 'all';
use Getopt::Long;
my (
$command, $ssh_user, $orig_master_host, $orig_master_ip,
$orig_master_port, $new_master_host, $new_master_ip, $new_master_port
);
#############################添加内容部分#########################################
my $vip = '192.168.60.100'; #指定vip的地址
my $brdc = '192.168.60.255'; #指定vip的广播地址
my $ifdev = 'ens33'; #指定vip绑定的网卡
my $key = '1'; #指定vip绑定的虚拟网卡序列号
my $ssh_start_vip = "/sbin/ifconfig ens33:$key $vip"; #代表此变量值为ifconfig ens33:1 192.168.60.100
my $ssh_stop_vip = "/sbin/ifconfig ens33:$key down"; #代表此变量值为ifconfig ens33:1 192.168.60.100 down
my $exit_code = 0; #指定退出状态码为0
#my $ssh_start_vip = "/usr/sbin/ip addr add $vip/24 brd $brdc dev $ifdev label $ifdev:$key;/usr/sbin/arping -q -A -c 1 -I $ifdev $vip;iptables -F;";
#my $ssh_stop_vip = "/usr/sbin/ip addr del $vip/24 dev $ifdev label $ifdev:$key";
##################################################################################
GetOptions(
'command=s' => \$command,
'ssh_user=s' => \$ssh_user,
'orig_master_host=s' => \$orig_master_host,
'orig_master_ip=s' => \$orig_master_ip,
'orig_master_port=i' => \$orig_master_port,
'new_master_host=s' => \$new_master_host,
'new_master_ip=s' => \$new_master_ip,
'new_master_port=i' => \$new_master_port,
);
exit &main();
sub main {
print "\n\nIN SCRIPT TEST====$ssh_stop_vip==$ssh_start_vip===\n\n";
if ( $command eq "stop" || $command eq "stopssh" ) {
my $exit_code = 1;
eval {
print "Disabling the VIP on old master: $orig_master_host \n";
&stop_vip();
$exit_code = 0;
};
if ($@) {
warn "Got Error: $@\n";
exit $exit_code;
}
exit $exit_code;
}
elsif ( $command eq "start" ) {
my $exit_code = 10;
eval {
print "Enabling the VIP - $vip on the new master - $new_master_host \n";
&start_vip();
$exit_code = 0;
};
if ($@) {
warn $@;
exit $exit_code;
}
exit $exit_code;
}
elsif ( $command eq "status" ) {
print "Checking the Status of the script.. OK \n";
exit 0;
}
else {
&usage();
exit 1;
}
}
sub start_vip() {
`ssh $ssh_user\@$new_master_host \" $ssh_start_vip \"`;
}
## A simple system call that disable the VIP on the old_master
sub stop_vip() {
`ssh $ssh_user\@$orig_master_host \" $ssh_stop_vip \"`;
}
sub usage {
print
"Usage: master_ip_failover --command=start|stop|stopssh|status --orig_master_host=host --orig_master_ip=ip --orig_master_port=port --new_master_host=host --new_master_ip=ip --new_master_port=port\n";
}
（4）创建 MHA 软件目录并拷贝配置文件，这里使用app1.cnf配置文件来管理 mysql 节点服务器
mkdir /etc/masterha cp /opt/mha4mysql-manager-0.57/samples/conf/app1.cnf /etc/masterha vim /etc/masterha/app1.cnf #删除原有内容，直接复制并修改节点服务器的IP地址 [server default] manager_log=/var/log/masterha/app1/manager.log manager_workdir=/var/log/masterha/app1 master_binlog_dir=/usr/local/mysql/data master_ip_failover_script=/usr/local/bin/master_ip_failover master_ip_online_change_script=/usr/local/bin/master_ip_online_change password=manager ping_interval=1 remote_workdir=/tmp repl_password=123456 repl_user=myslave secondary_check_script=/usr/local/bin/masterha_secondary_check -s 192.168.60.92 -s 192.168.60.93 #从对主监听 shutdown_script="" ssh_user=root user=mha [server1] hostname=192.168.60.91 #主服务器 port=3306 [server2] candidate_master=1 check_repl_delay=0 hostname=192.168.60.92 #备用主服务器 port=3306 [server3] hostname=192.168.60.93 #从服务器2 port=3306
10、第一次配置需要在 Master 节点上手动开启虚拟IP
/sbin/ifconfig ens33:1 192.168.60.120/24
11、在 manager 节点上测试 ssh 无密码认证，如果正常最后会输出 successfully，如下所示。
masterha_check_ssh -conf=/etc/masterha/app1.cnf
12、在 manager 节点上测试 mysql 主从连接情况，最后出现 MySQL Replication Health is OK 字样说明正常。如下所示。
masterha_check_repl -conf=/etc/masterha/app1.cnf
13、在 manager 节点上启动 MHA
nohup masterha_manager --conf=/etc/masterha/app1.cnf --remove_dead_master_conf --ignore_last_failover < /dev/null > /var/log/masterha/app1/manager.log 2>&1 &
14、查看 MHA 状态，可以看到当前的 master 是 master 节点。
masterha_check_status --conf=/etc/masterha/app1.cnf
15、查看 MHA 日志，也以看到当前的 master 是 192.168.233.21，如下所示。
cat /var/log/masterha/app1/manager.log | grep "current master"
//若要关闭 manager 服务，可以使用如下命令。 masterha_stop --conf=/etc/masterha/app1.cnf
mysql>set global read_only=1;一般设置为从数据库为只读 mysql>set global read_only=0;关闭数据库为只读
