/**
* get candidate master from all nodes
*
* The current available rule: get the server with MIN(node_id)
* node_id comes from 'wsrep_local_index' variable
*
* @param servers The monitored servers list
* @return The candidate master on success, NULL on failure
*/
static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *servers) {
MONITOR_SERVERS *ptr = servers;
MONITOR_SERVERS *candidate_master = NULL;
long min_id = -1;
/* set min_id to the lowest value of ptr->server->node_id */
while(ptr) {
if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) {
ptr->server->depth = 0;
if ((ptr->server->node_id < min_id) && min_id >= 0) {
min_id = ptr->server->node_id;
candidate_master = ptr;
} else {
if (min_id < 0) {
min_id = ptr->server->node_id;
candidate_master = ptr;
}
}
}
ptr = ptr->next;
}
return candidate_master;
}
/**
* set the master server in the cluster
*
* master could be the last one from previous monitor cycle Iis running) or
* the candidate master.
* The selection is based on the configuration option mapped to master_stickiness
* The candidate master may change over time due to
* 'wsrep_local_index' value change in the Galera Cluster
* Enabling master_stickiness will avoid master change unless a failure is spotted
*
* @param current_master Previous master server
* @param candidate_master The candidate master server accordingly to the selection rule
* @return The master node pointer (could be NULL)
*/
static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *current_master, MONITOR_SERVERS *candidate_master, int master_stickiness)
{
/*
* if current master is not set or master_stickiness is not enable
* just return candidate_master.
*/
if (current_master == NULL || master_stickiness == 0)
{
return candidate_master;
}
else
{
/*
* if current_master is still a cluster member use it
*
*/
if (SERVER_IS_JOINED(current_master->server) && (!SERVER_IN_MAINT(current_master->server)))
{
return current_master;
}
else
{
return candidate_master;
}
}
}
/**
* get candidate master from all nodes
*
* The current available rule: get the server with MIN(node_id)
* node_id comes from 'wsrep_local_index' variable
*
* @param servers The monitored servers list
* @return The candidate master on success, NULL on failure
*/
static MONITOR_SERVERS *get_candidate_master(MONITOR* mon)
{
MONITOR_SERVERS *moitor_servers = mon->databases;
MONITOR_SERVERS *candidate_master = NULL;
GALERA_MONITOR* handle = mon->handle;
long min_id = -1;
int minval = INT_MAX;
int currval;
char* value;
/* set min_id to the lowest value of moitor_servers->server->node_id */
while (moitor_servers)
{
if (!SERVER_IN_MAINT(moitor_servers->server) && SERVER_IS_JOINED(moitor_servers->server))
{
moitor_servers->server->depth = 0;
if (handle->use_priority && (value = serverGetParameter(moitor_servers->server, "priority")) != NULL)
{
currval = atoi(value);
if (currval < minval && currval > 0)
{
minval = currval;
candidate_master = moitor_servers;
}
}
else if (moitor_servers->server->node_id >= 0 &&
(!handle->use_priority || /** Server priority disabled*/
candidate_master == NULL || /** No candidate chosen */
serverGetParameter(candidate_master->server, "priority") == NULL)) /** Candidate has no priority */
{
if (min_id < 0 || moitor_servers->server->node_id < min_id)
{
min_id = moitor_servers->server->node_id;
candidate_master = moitor_servers;
}
}
}
moitor_servers = moitor_servers->next;
}
return candidate_master;
}
/**
* The entry point for the monitoring module thread
*
* @param arg The handle of the monitor
*/
static void
monitorMain(void *arg)
{
MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg;
MONITOR_SERVERS *ptr;
size_t nrounds = 0;
MONITOR_SERVERS *candidate_master = NULL;
int master_stickiness = handle->disableMasterFailback;
int is_cluster=0;
int log_no_members = 1;
if (mysql_thread_init())
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Fatal : mysql_thread_init failed in monitor "
"module. Exiting.\n")));
return;
}
handle->status = MONITOR_RUNNING;
while (1)
{
if (handle->shutdown)
{
handle->status = MONITOR_STOPPING;
mysql_thread_end();
handle->status = MONITOR_STOPPED;
return;
}
/** Wait base interval */
thread_millisleep(MON_BASE_INTERVAL_MS);
/**
* Calculate how far away the monitor interval is from its full
* cycle and if monitor interval time further than the base
* interval, then skip monitoring checks. Excluding the first
* round.
*/
if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= MON_BASE_INTERVAL_MS)
{
nrounds += 1;
continue;
}
nrounds += 1;
/* reset cluster members counter */
is_cluster=0;
ptr = handle->databases;
while (ptr)
{
monitorDatabase(handle, ptr);
/* clear bits for non member nodes */
if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) {
ptr->server->depth = -1;
/* clear M/S status */
server_clear_status(ptr->server, SERVER_SLAVE);
server_clear_status(ptr->server, SERVER_MASTER);
/* clear master sticky status */
server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
}
/* Log server status change */
if (mon_status_changed(ptr))
{
LOGIF(LD, (skygw_log_write_flush(
LOGFILE_DEBUG,
"Backend server %s:%d state : %s",
ptr->server->name,
ptr->server->port,
STRSRVSTATUS(ptr->server))));
}
if (SERVER_IS_DOWN(ptr->server))
{
/** Increase this server'e error count */
ptr->mon_err_count += 1;
}
else
{
/** Reset this server's error count */
ptr->mon_err_count = 0;
}
ptr = ptr->next;
}
/*
* Let's select a master server:
* it could be the candidate master following MIN(node_id) rule or
* the server that was master in the previous monitor polling cycle
* Decision depends on master_stickiness value set in configuration
*/
/* get the candidate master, followinf MIN(node_id) rule */
candidate_master = get_candidate_master(handle->databases);
/* Select the master, based on master_stickiness */
handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness);
ptr = handle->databases;
while (ptr && handle->master) {
if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) {
ptr = ptr->next;
continue;
}
if (ptr != handle->master) {
/* set the Slave role */
server_set_status(ptr->server, SERVER_SLAVE);
server_clear_status(ptr->server, SERVER_MASTER);
/* clear master stickyness */
server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
} else {
/* set the Master role */
server_set_status(handle->master->server, SERVER_MASTER);
server_clear_status(handle->master->server, SERVER_SLAVE);
if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) {
/* set master stickyness */
server_set_status(handle->master->server, SERVER_MASTER_STICKINESS);
} else {
/* clear master stickyness */
server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
}
}
is_cluster++;
ptr = ptr->next;
}
if (is_cluster == 0 && log_no_members) {
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Error: there are no cluster members")));
log_no_members = 0;
} else {
if (is_cluster > 0 && log_no_members == 0) {
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Info: found cluster members")));
log_no_members = 1;
}
}
}
}
/**
* The entry point for the monitoring module thread
*
* @param arg The handle of the monitor
*/
static void
monitorMain(void *arg)
{
MONITOR* mon = (MONITOR*) arg;
GALERA_MONITOR *handle;
MONITOR_SERVERS *ptr;
size_t nrounds = 0;
MONITOR_SERVERS *candidate_master = NULL;
int master_stickiness;
int is_cluster = 0;
int log_no_members = 1;
monitor_event_t evtype;
spinlock_acquire(&mon->lock);
handle = (GALERA_MONITOR *) mon->handle;
spinlock_release(&mon->lock);
master_stickiness = handle->disableMasterFailback;
if (mysql_thread_init())
{
MXS_ERROR("mysql_thread_init failed in monitor module. Exiting.");
return;
}
handle->status = MONITOR_RUNNING;
while (1)
{
if (handle->shutdown)
{
handle->status = MONITOR_STOPPING;
mysql_thread_end();
handle->status = MONITOR_STOPPED;
return;
}
/** Wait base interval */
thread_millisleep(MON_BASE_INTERVAL_MS);
/**
* Calculate how far away the monitor interval is from its full
* cycle and if monitor interval time further than the base
* interval, then skip monitoring checks. Excluding the first
* round.
*/
if (nrounds != 0 && ((nrounds * MON_BASE_INTERVAL_MS) % mon->interval) >= MON_BASE_INTERVAL_MS)
{
nrounds += 1;
continue;
}
nrounds += 1;
/* reset cluster members counter */
is_cluster = 0;
ptr = mon->databases;
while (ptr)
{
ptr->mon_prev_status = ptr->server->status;
monitorDatabase(mon, ptr);
/* Log server status change */
if (mon_status_changed(ptr))
{
MXS_DEBUG("Backend server %s:%d state : %s",
ptr->server->name,
ptr->server->port,
STRSRVSTATUS(ptr->server));
}
if (!(SERVER_IS_RUNNING(ptr->server)) ||
!(SERVER_IS_IN_CLUSTER(ptr->server)))
{
dcb_hangup_foreach(ptr->server);
}
if (SERVER_IS_DOWN(ptr->server))
{
/** Increase this server'e error count */
dcb_hangup_foreach(ptr->server);
ptr->mon_err_count += 1;
}
else
{
/** Reset this server's error count */
ptr->mon_err_count = 0;
}
ptr = ptr->next;
}
/*
* Let's select a master server:
* it could be the candidate master following MIN(node_id) rule or
* the server that was master in the previous monitor polling cycle
* Decision depends on master_stickiness value set in configuration
*/
/* get the candidate master, following MIN(node_id) rule */
candidate_master = get_candidate_master(mon);
/* Select the master, based on master_stickiness */
if (1 == handle->disableMasterRoleSetting)
{
handle->master = NULL;
}
else
{
handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness);
}
ptr = mon->databases;
while (ptr)
{
const int repl_bits = (SERVER_SLAVE | SERVER_MASTER | SERVER_MASTER_STICKINESS);
if (SERVER_IS_JOINED(ptr->server))
{
if (handle->master)
{
if (ptr != handle->master)
{
/* set the Slave role and clear master stickiness */
server_clear_set_status(ptr->server, repl_bits, SERVER_SLAVE);
}
else
{
if (candidate_master &&
handle->master->server->node_id != candidate_master->server->node_id)
{
/* set master role and master stickiness */
server_clear_set_status(ptr->server, repl_bits,
(SERVER_MASTER | SERVER_MASTER_STICKINESS));
}
else
{
/* set master role and clear master stickiness */
server_clear_set_status(ptr->server, repl_bits, SERVER_MASTER);
}
}
}
is_cluster++;
}
else
{
server_clear_set_status(ptr->server, repl_bits, 0);
}
ptr = ptr->next;
}
if (is_cluster == 0 && log_no_members)
{
MXS_ERROR("There are no cluster members");
log_no_members = 0;
}
else
{
if (is_cluster > 0 && log_no_members == 0)
{
MXS_NOTICE("Found cluster members");
log_no_members = 1;
}
}
ptr = mon->databases;
while (ptr)
{
/** Execute monitor script if a server state has changed */
if (mon_status_changed(ptr))
{
evtype = mon_get_event_type(ptr);
if (isGaleraEvent(evtype))
{
MXS_NOTICE("Server changed state: %s[%s:%u]: %s",
ptr->server->unique_name,
ptr->server->name, ptr->server->port,
mon_get_event_name(ptr));
if (handle->script && handle->events[evtype])
{
monitor_launch_script(mon, ptr, handle->script);
}
}
}
ptr = ptr->next;
}
}
}
/**
* Monitor an individual server. Does not deal with the setting of master or
* slave bits, except for clearing them when a server is not joined to the
* cluster.
*
* @param handle The MySQL Monitor object
* @param database The database to probe
*/
static void
monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database)
{
GALERA_MONITOR* handle = (GALERA_MONITOR*) mon->handle;
MYSQL_ROW row;
MYSQL_RES *result, *result2;
int isjoined = 0;
unsigned long int server_version = 0;
char *server_string;
SERVER temp_server;
/* Don't even probe server flagged as in maintenance */
if (SERVER_IN_MAINT(database->server))
return;
/** Store previous status */
database->mon_prev_status = database->server->status;
server_transfer_status(&temp_server, database->server);
server_clear_status(&temp_server, SERVER_RUNNING);
/* Also clear Joined */
server_clear_status(&temp_server, SERVER_JOINED);
connect_result_t rval = mon_connect_to_db(mon, database);
if (rval != MONITOR_CONN_OK)
{
if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR)
{
server_set_status(&temp_server, SERVER_AUTH_ERROR);
}
else
{
server_clear_status(&temp_server, SERVER_AUTH_ERROR);
}
database->server->node_id = -1;
if (mon_status_changed(database) && mon_print_fail_status(database))
{
mon_log_connect_error(database, rval);
}
server_transfer_status(database->server, &temp_server);
return;
}
/* If we get this far then we have a working connection */
server_set_status(&temp_server, SERVER_RUNNING);
/* get server version string */
server_string = (char *) mysql_get_server_info(database->con);
if (server_string)
{
server_set_version_string(database->server, server_string);
}
/* Check if the the Galera FSM shows this node is joined to the cluster */
if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state'") == 0
&& (result = mysql_store_result(database->con)) != NULL)
{
if (mysql_field_count(database->con) < 2)
{
mysql_free_result(result);
MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_state'\". "
"Expected 2 columns."
" MySQL Version: %s", version_str);
return;
}
while ((row = mysql_fetch_row(result)))
{
if (strcmp(row[1], "4") == 0)
isjoined = 1;
/* Check if the node is a donor and is using xtrabackup, in this case it can stay alive */
else if (strcmp(row[1], "2") == 0 && handle->availableWhenDonor == 1)
{
if (mysql_query(database->con, "SHOW VARIABLES LIKE 'wsrep_sst_method'") == 0
&& (result2 = mysql_store_result(database->con)) != NULL)
{
if (mysql_field_count(database->con) < 2)
{
mysql_free_result(result);
mysql_free_result(result2);
MXS_ERROR("Unexpected result for \"SHOW VARIABLES LIKE "
"'wsrep_sst_method'\". Expected 2 columns."
" MySQL Version: %s", version_str);
return;
}
while ((row = mysql_fetch_row(result2)))
{
if (strncmp(row[1], "xtrabackup", 10) == 0)
isjoined = 1;
}
mysql_free_result(result2);
}
}
}
mysql_free_result(result);
}
/* Check the the Galera node index in the cluster */
if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0
&& (result = mysql_store_result(database->con)) != NULL)
{
long local_index = -1;
if (mysql_field_count(database->con) < 2)
{
mysql_free_result(result);
MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_index'\". "
"Expected 2 columns."
" MySQL Version: %s", version_str);
return;
}
while ((row = mysql_fetch_row(result)))
{
local_index = strtol(row[1], NULL, 10);
if ((errno == ERANGE && (local_index == LONG_MAX
|| local_index == LONG_MIN)) || (errno != 0 && local_index == 0))
{
local_index = -1;
}
database->server->node_id = local_index;
}
mysql_free_result(result);
}
if (isjoined)
{
server_set_status(&temp_server, SERVER_JOINED);
}
else
{
server_clear_status(&temp_server, SERVER_JOINED);
}
/* clear bits for non member nodes */
if (!SERVER_IN_MAINT(database->server) && (!SERVER_IS_JOINED(&temp_server)))
{
database->server->depth = -1;
/* clear M/S status */
server_clear_status(&temp_server, SERVER_SLAVE);
server_clear_status(&temp_server, SERVER_MASTER);
/* clear master sticky status */
server_clear_status(&temp_server, SERVER_MASTER_STICKINESS);
}
server_transfer_status(database->server, &temp_server);
}