1 尝试在线添加私有网络的新接口是否成功。

使用oifcfg命令在线添加新的网卡接口，在还没有配置bond1的条件下
也是可以添加成功的。

[grid@orcldb1 ~]$ oifcfg getif
eno3 192.168.224.0 global public
ens3f0 10.2.0.0 global cluster_interconnect,asm
eno3 4509:8002:5A06:0120:0010:0000:0002:D000 global public
[grid@orcldb1 ~]$ oifcfg setif -global bond1/10.2.0.0:cluster_interconnect,asm
[grid@orcldb1 ~]$ oifcfg getif
eno3 192.168.224.0 global public
ens3f0 10.2.0.0 global cluster_interconnect,asm
eno3 4509:8002:5A06:0120:0010:0000:0002:D000 global public
bond1 10.2.0.0 global cluster_interconnect,asm

[grid@orcldb1 ~]$ oifcfg delif -global bond1/10.2.0.0
[grid@orcldb1 ~]$ oifcfg getif
eno3 192.168.224.0 global public
ens3f0 10.2.0.0 global cluster_interconnect,asm
eno3 4509:8002:5A06:0120:0010:0000:0002:D000 global public
[grid@orcldb1 ~]$

2 OCR备份

[root@orcldb1 bin]# ./ocrconfig -showbackup
orcldb2 2023/06/29 06:53:23 +DATA:/orcldb/OCRBACKUP/backup00.ocr.272.1140764001 479021767
orcldb2 2023/06/29 02:53:20 +DATA:/orcldb/OCRBACKUP/backup01.ocr.258.1140749597 479021767
orcldb2 2023/06/28 22:53:15 +DATA:/orcldb/OCRBACKUP/backup02.ocr.274.1140735193 479021767
orcldb2 2023/06/27 06:51:06 +DATA:/orcldb/OCRBACKUP/day.ocr.262.1140591067 479021767
orcldb2 2023/06/17 18:58:03 +DATA:/orcldb/OCRBACKUP/week.ocr.260.1139770683 479021767
orcldb2 2023/03/27 10:33:33 +DATA:/orcldb/OCRBACKUP/backup_20230327_103333.ocr.273.1132569213 479021767
orcldb2 2023/03/27 10:22:42 +DATA:/orcldb/OCRBACKUP/backup_20230327_102242.ocr.271.1132568563 479021767
orcldb2 2023/03/24 14:09:26 +DATA:/orcldb/OCRBACKUP/backup_20230324_140926.ocr.259.1132322967 294213661
orcldb1 2020/08/04 15:28:17 +DATA:/orcldb/OCRBACKUP/backup_20200804_152817.ocr.269.1047569297 294213661
orcldb1 2020/05/13 13:17:36 +DATA:/orcldb/OCRBACKUP/backup_20200513_131736.ocr.261.1040303857 2798127435
[root@orcldb1 bin]# ./ocrconfig -manualbackup
orcldb2 2023/06/29 10:21:21 +DATA:/orcldb/OCRBACKUP/backup_20230629_102121.ocr.263.1140776483 479021767
orcldb2 2023/03/27 10:33:33 +DATA:/orcldb/OCRBACKUP/backup_20230327_103333.ocr.273.1132569213 479021767
orcldb2 2023/03/27 10:22:42 +DATA:/orcldb/OCRBACKUP/backup_20230327_102242.ocr.271.1132568563 479021767

orcldb2 2023/03/24 14:09:26 +DATA:/orcldb/OCRBACKUP/backup_20230324_140926.ocr.259.1132322967 294213661
orcldb1 2020/08/04 15:28:17 +DATA:/orcldb/OCRBACKUP/backup_20200804_152817.ocr.269.1047569297 294213661
[root@orcldb1 bin]#

3 查看当前网络信息

[root@orcldb1 bin]# ./oifcfg getif
eno3 192.168.224.0 global public
ens3f0 10.2.0.0 global cluster_interconnect,asm
eno3 4509:8002:5A06:0120:0010:0000:0002:D000 global public
[root@orcldb1 bin]#
oifcfg setif -global bond1/10.2.0.0:cluster_interconnect,asm
[root@orcldb2 bin]# ./oifcfg getif
eno3 192.168.224.0 global public
ens3f0 10.2.0.0 global cluster_interconnect,asm
eno3 4509:8002:5A06:0120:0010:0000:0002:D000 global public
bond1 10.2.0.0 global cluster_interconnect,asm
[root@orcldb2 bin]#

4 停止OGG复制进程

GGSCI (orcl_DB_01) 6> info all ---之前关闭集群总是出现 ogg无法umount的error，将dpe传输进程停止之后，报错消失。

Program Status Group Lag at Chkpt Time Since Chkpt

MANAGER RUNNING
EXTRACT RUNNING DPEARS_A 00:00:02 00:00:05
EXTRACT RUNNING DPEARS_B 00:00:00 00:00:07
EXTRACT STOPPED DPERAC_A 00:00:02 00:00:01
EXTRACT RUNNING EXTARS_A 00:00:00 00:00:01
EXTRACT RUNNING EXTARS_B 00:00:02 00:00:02
EXTRACT RUNNING EXTRAC_A 00:00:01 00:00:07

GGSCI (orcl_DB_01) 7>

5 关闭节点HAS

节点1节点2关闭has，
./crsctl stop has

6 配置私有网络BOND

---配置bond1 双网卡。
slave添加如下5行

USERCTL=no
MASTER=bond1
SLAVE=yes
NM_CONTROLLED=no
HOTPLUG=no

master配置

DEVICE=bond1
IPADDR=10.2.0.120
NETMASK=255.255.255.0
#GATEWAY=10.2.0.1
ONBOOT=yes
BOOTPROTO=none
USERCTL=no

BONDING_OPTS="mode=1 miimon=100"
NM_CONTROLLED=no
HOTPLUG=no

7 重启网络服务

systemctl restart network

[root@orcldb1 ~]# ip addr

[orcl@orcldb1 ~]$ cat /proc/net/bonding/bond1
Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)

Bonding Mode: fault-tolerance (active-backup)
Primary Slave: None
Currently Active Slave: ens3f0
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 0
Down Delay (ms): 0

Slave Interface: ens3f0
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 34:0a:98:9c:ed:45
Slave queue ID: 0

Slave Interface: ens3f1
MII Status: down
Speed: Unknown
Duplex: Unknown
Link Failure Count: 0
Permanent HW addr: 34:0a:98:9c:ed:46
Slave queue ID: 0
[orcl@orcldb1 ~]$

[root@orcldb1 network-scripts]# more ifcfg-bond1
DEVICE=bond1
IPADDR=10.2.0.120
NETMASK=255.255.255.0
#GATEWAY=10.2.0.1
ONBOOT=yes
BOOTPROTO=none
USERCTL=no

BONDING_OPTS="mode=1 miimon=100"
NM_CONTROLLED=no
HOTPLUG=no
[root@orcldb1 network-scripts]# more ifcfg-ens3f0
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens3f0
UUID=cc18f2de-d837-40d8-8445-a13c5da7a8cc
DEVICE=ens3f0
ONBOOT=yes
USERCTL=no
MASTER=bond1
SLAVE=yes
NM_CONTROLLED=no
HOTPLUG=no
[root@orcldb1 network-scripts]# more ifcfg-ens3f1
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=none
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens3f1
UUID=207bf934-ff44-4e05-aea7-bc330672369b
DEVICE=ens3f1
ONBOOT=yes
USERCTL=no
MASTER=bond1
SLAVE=yes
NM_CONTROLLED=no
HOTPLUG=no
[root@orcldb1 network-scripts]#

节点2 按照上面操作配置bond1即可。

8 双网卡测试：

9 启动集群。

10 查看GIPC日志

2023-06-29 14:54:28.170 :GIPCDCLT:221996800: gipcdClientInterfaceMetrics: Received type(gipcdmsgtypeInterfaceMetrics), endp(000000000000028f), len(1024), buf(0x7f1ff4359488), inf(ip: 10.2.0.120:43466, mask: 255.255.255.0, subnet: 10.2.0.0, mac: , ifname: bond1) time(0), retry(0), stamp(3), send(2), recv(3)
2023-06-29 14:54:28.170 :GIPCDCLT:221996800: gipcdClientInterfaceMetrics: enqueue interface metrics (1:orcldb1:bond1) to worklist
2023-06-29 14:54:31.231 : CLSINET:217794304: (:CLSINE0018:)WARNING: failed to find interface available for interface definition ens3f0(:.*)?:10.2.0.0
2023-06-29 14:54:33.222 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 2.413793 [30/29/29]
2023-06-29 14:54:33.222 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 2.413793 [30/29/29]
2023-06-29 14:54:35.335 : CLSINET:217794304: (:CLSINE0018:)WARNING: failed to find interface available for interface definition ens3f0(:.*)?:10.2.0.0
2023-06-29 14:54:37.178 :GIPCHTHR:210466560: gipchaWorkerWork: workerThread heart beat, time interval since last heartBeat 30020 loopCount 31 sendCount 0 recvCount 18 postCount 0 sendCmplCount 0 recvCmplCount 0
2023-06-29 14:54:38.328 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 1.333333 [45/45/45]
2023-06-29 14:54:38.328 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 1.333333 [45/45/45]
2023-06-29 14:54:39.441 : CLSINET:217794304: (:CLSINE0018:)WARNING: failed to find interface available for interface definition ens3f0(:.*)?:10.2.0.0
2023-06-29 14:54:40.380 :GIPCHTHR:208365312: gipchaDaemonWork: DaemonThread heart beat, time interval since last heartBeat 30030 loopCount 42
发现原来的网卡信息报错，bond1 rank正常

11 删除旧的私网信息

[grid@orcldb1 ~]$ oifcfg delif -global ens3f0/10.2.0.0
[grid@orcldb1 ~]$ oifcfg getif
eno3 192.168.224.0 global public
eno3 4509:8002:5A06:0120:0010:0000:0002:D000 global public
bond1 10.2.0.0 global cluster_interconnect,asm
[grid@orcldb1 ~]$

再次查看gipcd 的日志
grid@orcldb1 trace]$ tail -f gipcd.trc
2023-06-29 16:03:51.324 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 2.173913 [23/23/23]
2023-06-29 16:03:51.324 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 2.173913 [23/23/23]
2023-06-29 16:03:56.428 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 0.400000 [23/25/25]
2023-06-29 16:03:56.428 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 0.400000 [23/25/25]
2023-06-29 16:03:56.428 :GIPCDMON:217794304: gipcdMonitorPublishDiags: key gipc_round_trip_time handle 0x7f1fec392160 writeTime 12345264 value <orcldb2:58:12339264:192us>
2023-06-29 16:04:01.524 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 0.384615 [23/26/26]
2023-06-29 16:04:01.524 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 0.384615 [23/26/26]
2023-06-29 16:04:05.708 :GIPCHTHR:208365312: gipchaDaemonWork: DaemonThread heart beat, time interval since last heartBeat 30030 loopCount 42
2023-06-29 16:04:07.631 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 0.416667 [23/24/24]
2023-06-29 16:04:07.631 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 0.416667 [23/24/24]
2023-06-29 16:04:12.737 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 1.739130 [23/23/23]
2023-06-29 16:04:12.737 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 1.739130 [23/23/23]
2023-06-29 16:04:16.321 :GIPCHTHR:210466560: gipchaWorkerWork: workerThread heart beat, time interval since last heartBeat 30040 loopCount 48 sendCount 0 recvCount 18 postCount 0 sendCmplCount 0 recvCmplCount 0
2023-06-29 16:04:17.832 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 1.304348 [25/23/23]
2023-06-29 16:04:17.832 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 1.304348 [25/23/23]
2023-06-29 16:04:22.937 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 0] bond1 - rank 99, node orcldb2 , avgms 1.363636 [21/22/22]
2023-06-29 16:04:22.937 :GIPCDMON:217794304: gipcdMonitorSaveInfMetricsNonOda: inf[ 1] bond1 - rank 99, node orcldb1 , avgms 1.363636 [21/22/22]