/** * get candidate master from all nodes * * The current available rule: get the server with MIN(node_id) * node_id comes from 'wsrep_local_index' variable * * @param servers The monitored servers list * @return The candidate master on success, NULL on failure */ static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *servers) { MONITOR_SERVERS *ptr = servers; MONITOR_SERVERS *candidate_master = NULL; long min_id = -1; /* set min_id to the lowest value of ptr->server->node_id */ while(ptr) { if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { ptr->server->depth = 0; if ((ptr->server->node_id < min_id) && min_id >= 0) { min_id = ptr->server->node_id; candidate_master = ptr; } else { if (min_id < 0) { min_id = ptr->server->node_id; candidate_master = ptr; } } } ptr = ptr->next; } return candidate_master; }
/** * set the master server in the cluster * * master could be the last one from previous monitor cycle Iis running) or * the candidate master. * The selection is based on the configuration option mapped to master_stickiness * The candidate master may change over time due to * 'wsrep_local_index' value change in the Galera Cluster * Enabling master_stickiness will avoid master change unless a failure is spotted * * @param current_master Previous master server * @param candidate_master The candidate master server accordingly to the selection rule * @return The master node pointer (could be NULL) */ static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *current_master, MONITOR_SERVERS *candidate_master, int master_stickiness) { /* * if current master is not set or master_stickiness is not enable * just return candidate_master. */ if (current_master == NULL || master_stickiness == 0) { return candidate_master; } else { /* * if current_master is still a cluster member use it * */ if (SERVER_IS_JOINED(current_master->server) && (!SERVER_IN_MAINT(current_master->server))) { return current_master; } else { return candidate_master; } } }
/** * get candidate master from all nodes * * The current available rule: get the server with MIN(node_id) * node_id comes from 'wsrep_local_index' variable * * @param servers The monitored servers list * @return The candidate master on success, NULL on failure */ static MONITOR_SERVERS *get_candidate_master(MONITOR* mon) { MONITOR_SERVERS *moitor_servers = mon->databases; MONITOR_SERVERS *candidate_master = NULL; GALERA_MONITOR* handle = mon->handle; long min_id = -1; int minval = INT_MAX; int currval; char* value; /* set min_id to the lowest value of moitor_servers->server->node_id */ while (moitor_servers) { if (!SERVER_IN_MAINT(moitor_servers->server) && SERVER_IS_JOINED(moitor_servers->server)) { moitor_servers->server->depth = 0; if (handle->use_priority && (value = serverGetParameter(moitor_servers->server, "priority")) != NULL) { currval = atoi(value); if (currval < minval && currval > 0) { minval = currval; candidate_master = moitor_servers; } } else if (moitor_servers->server->node_id >= 0 && (!handle->use_priority || /** Server priority disabled*/ candidate_master == NULL || /** No candidate chosen */ serverGetParameter(candidate_master->server, "priority") == NULL)) /** Candidate has no priority */ { if (min_id < 0 || moitor_servers->server->node_id < min_id) { min_id = moitor_servers->server->node_id; candidate_master = moitor_servers; } } } moitor_servers = moitor_servers->next; } return candidate_master; }
/** * The entry point for the monitoring module thread * * @param arg The handle of the monitor */ static void monitorMain(void *arg) { MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; MONITOR_SERVERS *ptr; size_t nrounds = 0; MONITOR_SERVERS *candidate_master = NULL; int master_stickiness = handle->disableMasterFailback; int is_cluster=0; int log_no_members = 1; if (mysql_thread_init()) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Fatal : mysql_thread_init failed in monitor " "module. Exiting.\n"))); return; } handle->status = MONITOR_RUNNING; while (1) { if (handle->shutdown) { handle->status = MONITOR_STOPPING; mysql_thread_end(); handle->status = MONITOR_STOPPED; return; } /** Wait base interval */ thread_millisleep(MON_BASE_INTERVAL_MS); /** * Calculate how far away the monitor interval is from its full * cycle and if monitor interval time further than the base * interval, then skip monitoring checks. Excluding the first * round. */ if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= MON_BASE_INTERVAL_MS) { nrounds += 1; continue; } nrounds += 1; /* reset cluster members counter */ is_cluster=0; ptr = handle->databases; while (ptr) { monitorDatabase(handle, ptr); /* clear bits for non member nodes */ if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) { ptr->server->depth = -1; /* clear M/S status */ server_clear_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); /* clear master sticky status */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } /* Log server status change */ if (mon_status_changed(ptr)) { LOGIF(LD, (skygw_log_write_flush( LOGFILE_DEBUG, "Backend server %s:%d state : %s", ptr->server->name, ptr->server->port, STRSRVSTATUS(ptr->server)))); } if (SERVER_IS_DOWN(ptr->server)) { /** Increase this server'e error count */ ptr->mon_err_count += 1; } else { /** Reset this server's error count */ ptr->mon_err_count = 0; } ptr = ptr->next; } /* * Let's select a master server: * it could be the candidate master following MIN(node_id) rule or * the server that was master in the previous monitor polling cycle * Decision depends on master_stickiness value set in configuration */ /* get the candidate master, followinf MIN(node_id) rule */ candidate_master = get_candidate_master(handle->databases); /* Select the master, based on master_stickiness */ handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness); ptr = handle->databases; while (ptr && handle->master) { if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) { ptr = ptr->next; continue; } if (ptr != handle->master) { /* set the Slave role */ server_set_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); /* clear master stickyness */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } else { /* set the Master role */ server_set_status(handle->master->server, SERVER_MASTER); server_clear_status(handle->master->server, SERVER_SLAVE); if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) { /* set master stickyness */ server_set_status(handle->master->server, SERVER_MASTER_STICKINESS); } else { /* clear master stickyness */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } } is_cluster++; ptr = ptr->next; } if (is_cluster == 0 && log_no_members) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error: there are no cluster members"))); log_no_members = 0; } else { if (is_cluster > 0 && log_no_members == 0) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Info: found cluster members"))); log_no_members = 1; } } } }
/** * The entry point for the monitoring module thread * * @param arg The handle of the monitor */ static void monitorMain(void *arg) { MONITOR* mon = (MONITOR*) arg; GALERA_MONITOR *handle; MONITOR_SERVERS *ptr; size_t nrounds = 0; MONITOR_SERVERS *candidate_master = NULL; int master_stickiness; int is_cluster = 0; int log_no_members = 1; monitor_event_t evtype; spinlock_acquire(&mon->lock); handle = (GALERA_MONITOR *) mon->handle; spinlock_release(&mon->lock); master_stickiness = handle->disableMasterFailback; if (mysql_thread_init()) { MXS_ERROR("mysql_thread_init failed in monitor module. Exiting."); return; } handle->status = MONITOR_RUNNING; while (1) { if (handle->shutdown) { handle->status = MONITOR_STOPPING; mysql_thread_end(); handle->status = MONITOR_STOPPED; return; } /** Wait base interval */ thread_millisleep(MON_BASE_INTERVAL_MS); /** * Calculate how far away the monitor interval is from its full * cycle and if monitor interval time further than the base * interval, then skip monitoring checks. Excluding the first * round. */ if (nrounds != 0 && ((nrounds * MON_BASE_INTERVAL_MS) % mon->interval) >= MON_BASE_INTERVAL_MS) { nrounds += 1; continue; } nrounds += 1; /* reset cluster members counter */ is_cluster = 0; ptr = mon->databases; while (ptr) { ptr->mon_prev_status = ptr->server->status; monitorDatabase(mon, ptr); /* Log server status change */ if (mon_status_changed(ptr)) { MXS_DEBUG("Backend server %s:%d state : %s", ptr->server->name, ptr->server->port, STRSRVSTATUS(ptr->server)); } if (!(SERVER_IS_RUNNING(ptr->server)) || !(SERVER_IS_IN_CLUSTER(ptr->server))) { dcb_hangup_foreach(ptr->server); } if (SERVER_IS_DOWN(ptr->server)) { /** Increase this server'e error count */ dcb_hangup_foreach(ptr->server); ptr->mon_err_count += 1; } else { /** Reset this server's error count */ ptr->mon_err_count = 0; } ptr = ptr->next; } /* * Let's select a master server: * it could be the candidate master following MIN(node_id) rule or * the server that was master in the previous monitor polling cycle * Decision depends on master_stickiness value set in configuration */ /* get the candidate master, following MIN(node_id) rule */ candidate_master = get_candidate_master(mon); /* Select the master, based on master_stickiness */ if (1 == handle->disableMasterRoleSetting) { handle->master = NULL; } else { handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness); } ptr = mon->databases; while (ptr) { const int repl_bits = (SERVER_SLAVE | SERVER_MASTER | SERVER_MASTER_STICKINESS); if (SERVER_IS_JOINED(ptr->server)) { if (handle->master) { if (ptr != handle->master) { /* set the Slave role and clear master stickiness */ server_clear_set_status(ptr->server, repl_bits, SERVER_SLAVE); } else { if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) { /* set master role and master stickiness */ server_clear_set_status(ptr->server, repl_bits, (SERVER_MASTER | SERVER_MASTER_STICKINESS)); } else { /* set master role and clear master stickiness */ server_clear_set_status(ptr->server, repl_bits, SERVER_MASTER); } } } is_cluster++; } else { server_clear_set_status(ptr->server, repl_bits, 0); } ptr = ptr->next; } if (is_cluster == 0 && log_no_members) { MXS_ERROR("There are no cluster members"); log_no_members = 0; } else { if (is_cluster > 0 && log_no_members == 0) { MXS_NOTICE("Found cluster members"); log_no_members = 1; } } ptr = mon->databases; while (ptr) { /** Execute monitor script if a server state has changed */ if (mon_status_changed(ptr)) { evtype = mon_get_event_type(ptr); if (isGaleraEvent(evtype)) { MXS_NOTICE("Server changed state: %s[%s:%u]: %s", ptr->server->unique_name, ptr->server->name, ptr->server->port, mon_get_event_name(ptr)); if (handle->script && handle->events[evtype]) { monitor_launch_script(mon, ptr, handle->script); } } } ptr = ptr->next; } } }
/** * Monitor an individual server. Does not deal with the setting of master or * slave bits, except for clearing them when a server is not joined to the * cluster. * * @param handle The MySQL Monitor object * @param database The database to probe */ static void monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database) { GALERA_MONITOR* handle = (GALERA_MONITOR*) mon->handle; MYSQL_ROW row; MYSQL_RES *result, *result2; int isjoined = 0; unsigned long int server_version = 0; char *server_string; SERVER temp_server; /* Don't even probe server flagged as in maintenance */ if (SERVER_IN_MAINT(database->server)) return; /** Store previous status */ database->mon_prev_status = database->server->status; server_transfer_status(&temp_server, database->server); server_clear_status(&temp_server, SERVER_RUNNING); /* Also clear Joined */ server_clear_status(&temp_server, SERVER_JOINED); connect_result_t rval = mon_connect_to_db(mon, database); if (rval != MONITOR_CONN_OK) { if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status(&temp_server, SERVER_AUTH_ERROR); } else { server_clear_status(&temp_server, SERVER_AUTH_ERROR); } database->server->node_id = -1; if (mon_status_changed(database) && mon_print_fail_status(database)) { mon_log_connect_error(database, rval); } server_transfer_status(database->server, &temp_server); return; } /* If we get this far then we have a working connection */ server_set_status(&temp_server, SERVER_RUNNING); /* get server version string */ server_string = (char *) mysql_get_server_info(database->con); if (server_string) { server_set_version_string(database->server, server_string); } /* Check if the the Galera FSM shows this node is joined to the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state'") == 0 && (result = mysql_store_result(database->con)) != NULL) { if (mysql_field_count(database->con) < 2) { mysql_free_result(result); MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_state'\". " "Expected 2 columns." " MySQL Version: %s", version_str); return; } while ((row = mysql_fetch_row(result))) { if (strcmp(row[1], "4") == 0) isjoined = 1; /* Check if the node is a donor and is using xtrabackup, in this case it can stay alive */ else if (strcmp(row[1], "2") == 0 && handle->availableWhenDonor == 1) { if (mysql_query(database->con, "SHOW VARIABLES LIKE 'wsrep_sst_method'") == 0 && (result2 = mysql_store_result(database->con)) != NULL) { if (mysql_field_count(database->con) < 2) { mysql_free_result(result); mysql_free_result(result2); MXS_ERROR("Unexpected result for \"SHOW VARIABLES LIKE " "'wsrep_sst_method'\". Expected 2 columns." " MySQL Version: %s", version_str); return; } while ((row = mysql_fetch_row(result2))) { if (strncmp(row[1], "xtrabackup", 10) == 0) isjoined = 1; } mysql_free_result(result2); } } } mysql_free_result(result); } /* Check the the Galera node index in the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0 && (result = mysql_store_result(database->con)) != NULL) { long local_index = -1; if (mysql_field_count(database->con) < 2) { mysql_free_result(result); MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_index'\". " "Expected 2 columns." " MySQL Version: %s", version_str); return; } while ((row = mysql_fetch_row(result))) { local_index = strtol(row[1], NULL, 10); if ((errno == ERANGE && (local_index == LONG_MAX || local_index == LONG_MIN)) || (errno != 0 && local_index == 0)) { local_index = -1; } database->server->node_id = local_index; } mysql_free_result(result); } if (isjoined) { server_set_status(&temp_server, SERVER_JOINED); } else { server_clear_status(&temp_server, SERVER_JOINED); } /* clear bits for non member nodes */ if (!SERVER_IN_MAINT(database->server) && (!SERVER_IS_JOINED(&temp_server))) { database->server->depth = -1; /* clear M/S status */ server_clear_status(&temp_server, SERVER_SLAVE); server_clear_status(&temp_server, SERVER_MASTER); /* clear master sticky status */ server_clear_status(&temp_server, SERVER_MASTER_STICKINESS); } server_transfer_status(database->server, &temp_server); }