I needed to remove HAIP from ODA after migrating to version 18.8.0.0.0 and decided to prepare this procedure.
This action plan should only require one clusterware restart vs. patching which can result in two or three clusterware restarts.
Let’s go to the procedure.
1. Backup gpnp profile
[grid@testoda1 peer]$ cd /u01/app/18.0.0.0/grid/gpnp/'hostname'/profiles/peer
[grid@testoda1 peer]$ cp -p profile.xml profile.xml.bkp
[grid@testoda2 peer]$ /u01/app/18.0.0.0/grid/gpnp/'hostname'/profiles/peer
[grid@testoda2 peer]$ cp -p profile.xml profile.xml.bkp
2. Get the cluster_interconnect interfaces (only on node0)
[grid@testoda1 ~]$ /u01/app/18.0.0.0/grid/bin/oifcfg getif
btbond1 10.32.16.0 global public
p1p1 192.168.16.0 global cluster_interconnect,asm
p1p2 192.168.17.0 global cluster_interconnect,asm
Please note: That private interface names might be different depending on the model, and/or ODA version which was used to deploy the machine, etc.
For the rest of this note, we are using p1p1/p1p2 as an example in the steps below.
3. Backup existing ifcfg- files
[root@testoda1 ~]# cd /etc/sysconfig/network-scripts
[root@testoda1 network-scripts]# cp ifcfg-p1p1 backupifcfgFiles/ifcfg-p1p1.bak
[root@testoda1 network-scripts]# cp ifcfg-p1p2 backupifcfgFiles/ifcfg-p1p2.bak
[root@testoda2 ~]# cd /etc/sysconfig/network-scripts
[root@testoda2 network-scripts]# cp ifcfg-p1p1 backupifcfgFiles/ifcfg-p1p1.bak
[root@testoda2 network-scripts]# cp ifcfg-p1p2 backupifcfgFiles/ifcfg-p1p2.bak
4. Create ifcfg-icbond0 and modify ifcfg- files
[root@testoda1 network-scripts]# vi /etc/sysconfig/network-scripts/ifcfg-icbond0
# This file is automatically created by the ODA software.
DEVICE=icbond0
ONBOOT=yes
BOOTPROTO=none
USERCTL=no
TYPE=BOND
IPV6INIT=no
NM_CONTROLLED=no
PEERDNS=no
MTU=9000
BONDING_OPTS=”mode=active-backup miimon=100 primary=p1p1″
IPADDR=192.168.16.24
NETMASK=255.255.255.0
[root@testoda1 network-scripts]# vi /etc/sysconfig/network-scripts/ifcfg-p1p1
# This file is automatically created by the ODA software.
DEVICE=p1p1
TYPE=Ethernet
ONBOOT=yes
NM_CONTROLLED=no
USERCTL=no
# disable generic and large receive offloads on all interfaces,
# to prevent known problems, specifically in bridge configurations.
ETHTOOL_OFFLOAD_OPTS=”lro off gro off”
IPV6INIT=no
PEERDNS=no
BOOTPROTO=none
MASTER=icbond0
SLAVE=yes
MTU=9000
[root@testoda1 network-scripts]# vi /etc/sysconfig/network-scripts/ifcfg-p1p2
/etc/sysconfig/network-scripts/ifcfg-p1p2
# This file is automatically created by the ODA software.
DEVICE=p1p2
TYPE=Ethernet
ONBOOT=yes
NM_CONTROLLED=no
USERCTL=no
# disable generic and large receive offloads on all interfaces,
# to prevent known problems, specifically in bridge configurations.
ETHTOOL_OFFLOAD_OPTS=”lro off gro off”
IPV6INIT=no
PEERDNS=no
BOOTPROTO=none
MASTER=icbond0
SLAVE=yes
MTU=9000
[root@testoda2 network-scripts]# vi /etc/sysconfig/network-scripts/ifcfg-icbond0
# This file is automatically created by the ODA software.
DEVICE=icbond0
ONBOOT=yes
BOOTPROTO=none
USERCTL=no
TYPE=BOND
IPV6INIT=no
NM_CONTROLLED=no
PEERDNS=no
MTU=9000
BONDING_OPTS=”mode=active-backup miimon=100 primary=p1p1″
IPADDR=192.168.16.25
NETMASK=255.255.255.0
[root@testoda2 network-scripts]# vi /etc/sysconfig/network-scripts/ifcfg-p1p1
# This file is automatically created by the ODA software.
DEVICE=p1p1
TYPE=Ethernet
ONBOOT=yes
NM_CONTROLLED=no
USERCTL=no
# disable generic and large receive offloads on all interfaces,
# to prevent known problems, specifically in bridge configurations.
ETHTOOL_OFFLOAD_OPTS=”lro off gro off”
IPV6INIT=no
PEERDNS=no
BOOTPROTO=none
MASTER=icbond0
SLAVE=yes
MTU=9000
[root@testoda2 network-scripts]# vi /etc/sysconfig/network-scripts/ifcfg-p1p2
# This file is automatically created by the ODA software.
DEVICE=p1p2
TYPE=Ethernet
ONBOOT=yes
NM_CONTROLLED=no
USERCTL=no
# disable generic and large receive offloads on all interfaces,
# to prevent known problems, specifically in bridge configurations.
ETHTOOL_OFFLOAD_OPTS=”lro off gro off”
IPV6INIT=no
PEERDNS=no
BOOTPROTO=none
MASTER=icbond0
SLAVE=yes
MTU=9000
5. Creating/replacing init.ora-s for APX instances
[grid@testoda1]$ echo "+APX1.cluster_interconnects='192.168.16.24'" > $ORACLE_HOME/dbs/init+APX1.ora
[grid@testoda2]$ echo “+APX2.cluster_interconnects=’192.168.16.25′” > $ORACLE_HOME/dbs/init+APX2.ora
6. Stop the Clusterware on node2
[root@testoda2 ~]# /u01/app/18.0.0.0/grid/bin/crsctl stop crs -f
7. Set the new, bonded cluster_interconnect interface and remove p1p1/p1p2 interfaces from the configuration (only on node0)
[grid@testoda1 ~]$ oifcfg setif -global icbond0/192.168.16.0:cluster_interconnect,asm
[grid@testoda1 ~]$ oifcfg getif
btbond1 10.209.244.0 global public
p1p1 192.168.16.0 global cluster_interconnect,asm
p1p2 192.168.17.0 global cluster_interconnect,asm
icbond0 192.168.16.0 global cluster_interconnect,asm
[grid@testoda1 ~]$ oifcfg delif -global p1p1/192.168.16.0
[grid@testoda1 ~]$ oifcfg delif -global p1p2/192.168.17.0
[grid@testoda1 ~]$ oifcfg getif
btbond1 10.209.244.0 global public
icbond0 192.168.16.0 global cluster_interconnect,asm
8. Remove HAIP dependency in ora.asm
[root@testoda1 ~]# /u01/app/18.0.0.0/grid/bin/crsctl modify res ora.cluster_interconnect.haip -attr ENABLED=0 -init
[root@testoda2 ~]# /u01/app/18.0.0.0/grid/bin/crsctl modify res ora.cluster_interconnect.haip -attr ENABLED=0 -init
[root@testoda1 ~]# /u01/app/18.0.0.0/grid/bin/crsctl modify res ora.asm -attr “START_DEPENDENCIES=’hard(ora.cssd,ora.ctssd) pullup(ora.cssd,ora.ctssd) weak(ora.drivers.acfs)’, STOP_DEPENDENCIES=’hard(intermediate:ora.cssd)'” -init
[root@testoda2 ~]# /u01/app/18.0.0.0/grid/bin/crsctl modify res ora.asm -attr “START_DEPENDENCIES=’hard(ora.cssd,ora.ctssd) pullup(ora.cssd,ora.ctssd) weak(ora.drivers.acfs)’, STOP_DEPENDENCIES=’hard(intermediate:ora.cssd)'” -init
9. Removing ora.cluster_interconnect.haip resource
[root@testoda1 ~]# /u01/app/18.0.0.0/grid/bin/crsctl delete resource ora.cluster_interconnect.haip -init –f
[root@testoda2 ~]# /u01/app/18.0.0.0/grid/bin/crsctl delete resource ora.cluster_interconnect.haip -init –f
10. Stop the Clusterware on node1
[root@testoda1 ~]# /u01/app/18.0.0.0/grid/bin/crsctl stop crs -f
11. Restart the network
[root@testoda1 network-scripts]# service network restart
[root@testoda1 network-scripts]# ifconfig -a
[root@testoda1 network-scripts]# cat /proc/net/bondinf/icbond0
[root@testoda2 network-scripts]# service network restart
[root@testoda2 network-scripts]# ifconfig -a
[root@testoda2 network-scripts]# cat /proc/net/bondinf/icbond0
12. Restart the Clusterware
[root@testoda1 ~]# /u01/app/18.0.0.0/grid/bin/crsctl start crs
[root@testoda2 ~]# /u01/app/18.0.0.0/grid/bin/crsctl start crs
13. Restart dcs-agent to rediscover the interfaces automatically
[grid@testoda1 ~]# /opt/oracle/dcs/bin/restartagent.sh
[grid@testoda2 ~]# /opt/oracle/dcs/bin/restartagent.sh
14. Checking the cluster service after removing the service
[root@testoda1 ~]#
[root@testoda1 ~]# /u01/app/18.0.0.0/grid/bin/crsctl check cluster -all
**************************************************************
testoda1:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
testoda2:
CRS-4537: Cluster Ready Services is online
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
**************************************************************
I hope I helped with this procedure
Andre Luiz Dutra Ontalba
Disclaimer: “The postings on this site are my own and don’t necessarily represent may actual employer positions, strategies or opinions. The information here was edited to be useful for general purpose, specific data and identifications was removed to allow reach generic audience and to be useful.”
