Here We have below nodes :
Orchestrator Node : FlyWithEdu-Monitor : 172.31.46.134 Source/Master Node : FlyWithEduDB1 : 172.31.18.17 Replica/Slave Node : FlyWithEduDB2 : 172.31.7.71
Pre-requisites
— Enabled below parameters on all database nodes of replication topology
root@FlyWithEduDB1:~# cat /etc/mysql/percona-server.conf.d/mysqld.cnf |grep -A 7 -i Orchestrator ##Orchestrator report_host=172.31.18.17 report_port=3306 log-slave-updates binlog_format=ROW master_info_repository=TABLE relay_log_info_repository= TABLE read_only=1
root@FlyWithEduDB2:~# cat /etc/mysql/percona-server.conf.d/mysqld.cnf |grep -A 7 -i Orchestrator ##Orchestrator report_host=172.31.7.71 report_port=3306 log-slave-updates binlog_format=ROW master_info_repository=TABLE relay_log_info_repository=TABLE read_only=1
— Create MySQLTopologyUser (MySQLTopologyPassword) for orchestrator configuration on Source/Master database node.
GRANT RELOAD, PROCESS, SUPER, REPLICATION SLAVE ON *.* TO 'orchestrator'@'172.31.%' IDENTIFIED BY '****************'; GRANT DROP ON `_pseudo_gtid_`.* TO 'orchestrator'@'172.31.%'; GRANT SELECT ON `mysql`.`slave_master_info` TO 'orchestrator'@'172.31.%';
Installation
— On orchestrator node (FlyWithEdu-Monitor), Download and install below packages from Orchestrator GitHub release section. https://github.com/github/orchestrator/releases
- orchestrator
- orchestrator-cli
- orchestrator-client
root@FlyWithEdu-Monitor:~# wget https://github.com/openark/orchestrator/releases/download/v3.2.2/orchestrator_3.2.2_amd64.deb root@FlyWithEdu-Monitor:~# wget https://github.com/openark/orchestrator/releases/download/v3.2.2/orchestrator-cli_3.2.2_amd64.deb root@FlyWithEdu-Monitor:~# wget https://github.com/openark/orchestrator/releases/download/v3.2.2/orchestrator-client_3.2.2_amd64.deb
— On orchestrator node(FlyWithEdu-Monitor), Install MySQL database.
root@FlyWithEdu-Monitor:~# apt install percona-server-server-5.7
— On orchestrator node(FlyWithEdu-Monitor), create orchestrator database.
FlyWithEdu-Monitor > create database orchestrator;
— On orchestrator node(FlyWithEdu-Monitor), Create MySQLOrchestratorUser(MySQLOrchestratorPassword) user account.
FlyWithEdu-Monitor > GRANT ALL PRIVILEGES ON *.* TO 'orchestrator'@'172.31.%' IDENTIFIED BY '****************';
Configuration
— Create orchestrator config file from one of the sample template
root@FlyWithEdu-Monitor:~# cd /usr/local/orchestrator root@FlyWithEdu-Monitor:/usr/local/orchestrator# cp orchestrator-sample.conf.json /etc/orchestrator.conf.json
— Add or update below parameters inside orchestrator config file (/etc/orchestrator.conf.json).
"MySQLTopologyUser": "orchestrator", "MySQLTopologyPassword": "xxxxxxxxx", "MySQLOrchestratorUser": "orchestrator", "MySQLOrchestratorPassword": "xxxxxxxxx", "ApplyMySQLPromotionAfterMasterFailover": true, "MasterFailoverDetachSlaveMasterHost": true, "AutoPseudoGTID": true, "UseSuperReadOnly": true,
— Start orchestrator service.
root@FlyWithEdu-Monitor:~# systemctl start orchestrator
Discover
— To search replication/cluster topology, run below command with any node of replication setup.
root@FlyWithEdu-Monitor:~# orchestrator-client -c discover -i FlyWithEduDB1 FlyWithEduDB1:3306 root@FlyWithEdu-Monitor:~#
— Check list of available cluster aliases
root@FlyWithEdu-Monitor:~# orchestrator-client -c clusters-alias FlyWithEduDB1:3306,FlyWithEduDB1 root@FlyWithEdu-Monitor:~#
— Check replication architecture of any cluster alias
root@FlyWithEdu-Monitor:~# orchestrator-client -c topology -a FlyWithEduDB1 FlyWithEduDB1:3306 [0s,ok,5.7.29-32-log,rw,ROW,>>,P-GTID] + FlyWithEduDB2:3306 [0s,ok,5.7.29-32-log,ro,ROW,>>,P-GTID] root@FlyWithEdu-Monitor:~#
Graceful Failover To FlyWithEduDB2
— Performing graceful failover once, we have Source/Master node is ONLINE.
— Current replication topology
root@FlyWithEdu-Monitor:~# orchestrator-client -c topology -a FlyWithEduDB1 FlyWithEduDB1:3306 [0s,ok,5.7.29-32-log,rw,ROW,>>,P-GTID] + FlyWithEduDB2:3306 [0s,ok,5.7.29-32-log,ro,ROW,>>,P-GTID] root@FlyWithEdu-Monitor:~#
— Graceful master takeover to FlyWithEduDB2
root@FlyWithEdu-Monitor:~# orchestrator-client -c graceful-master-takeover -a FlyWithEduDB1 -d FlyWithEduDB2 FlyWithEduDB2:3306 root@FlyWithEdu-Monitor:~#
— Check replication topology
root@FlyWithEdu-Monitor:~# orchestrator-client -c topology -a FlyWithEduDB1 FlyWithEduDB2:3306 [0s,ok,5.7.29-32-log,rw,ROW,>>,P-GTID] - FlyWithEduDB1:3306 [null,nonreplicating,5.7.29-32-log,ro,ROW,>>,P-GTID,downtimed] root@FlyWithEdu-Monitor:~#
— Start replication on old source/master node ie on FlyWithEduDB1
root@FlyWithEdu-Monitor:~# orchestrator-client -c start-replica -a FlyWithEduDB1 -i FlyWithEduDB1 FlyWithEduDB1:3306 root@FlyWithEdu-Monitor:~# orchestrator-client -c topology -a FlyWithEduDB1 FlyWithEduDB2:3306 [0s,ok,5.7.29-32-log,rw,ROW,>>,P-GTID] + FlyWithEduDB1:3306 [0s,ok,5.7.29-32-log,ro,ROW,>>,P-GTID] root@FlyWithEdu-Monitor:~#
Forceful Failover To FlyWithEduDB2
— Forceful failover can be performed once the source/master node is dead/offline.
— Replication topology once source/master node is dead/offline.
root@FlyWithEdu-Monitor:~# orchestrator-client -c topology -a FlyWithEduDB1 FlyWithEduDB1:3306 [unknown,invalid,5.7.29-32-log,rw,ROW,>>,P-GTID] - FlyWithEduDB2:3306 [null,nonreplicating,5.7.29-32-log,ro,ROW,>>,P-GTID] root@FlyWithEdu-Monitor:~#
— Perform forceful failover to FlyWithEduDB2.
root@FlyWithEdu-Monitor:~# orchestrator-client -c force-master-failover -a FlyWithEduDB1 -d FlyWithEduDB2
— After performing the forceful failover, old source/master node(FlyWithEduDB1) will be out of replication which can be again rebuilt/add under the replication topology later.