/** * get candidate master from all nodes * * The current available rule: get the server with MIN(node_id) * node_id comes from 'wsrep_local_index' variable * * @param servers The monitored servers list * @return The candidate master on success, NULL on failure */ static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *servers) { MONITOR_SERVERS *ptr = servers; MONITOR_SERVERS *candidate_master = NULL; long min_id = -1; /* set min_id to the lowest value of ptr->server->node_id */ while(ptr) { if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) { ptr->server->depth = 0; if ((ptr->server->node_id < min_id) && min_id >= 0) { min_id = ptr->server->node_id; candidate_master = ptr; } else { if (min_id < 0) { min_id = ptr->server->node_id; candidate_master = ptr; } } } ptr = ptr->next; } return candidate_master; }
/** * set the master server in the cluster * * master could be the last one from previous monitor cycle Iis running) or * the candidate master. * The selection is based on the configuration option mapped to master_stickiness * The candidate master may change over time due to * 'wsrep_local_index' value change in the Galera Cluster * Enabling master_stickiness will avoid master change unless a failure is spotted * * @param current_master Previous master server * @param candidate_master The candidate master server accordingly to the selection rule * @return The master node pointer (could be NULL) */ static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *current_master, MONITOR_SERVERS *candidate_master, int master_stickiness) { /* * if current master is not set or master_stickiness is not enable * just return candidate_master. */ if (current_master == NULL || master_stickiness == 0) { return candidate_master; } else { /* * if current_master is still a cluster member use it * */ if (SERVER_IS_JOINED(current_master->server) && (!SERVER_IN_MAINT(current_master->server))) { return current_master; } else { return candidate_master; } } }
/** * get candidate master from all nodes * * The current available rule: get the server with MIN(node_id) * node_id comes from 'wsrep_local_index' variable * * @param servers The monitored servers list * @return The candidate master on success, NULL on failure */ static MONITOR_SERVERS *get_candidate_master(MONITOR* mon) { MONITOR_SERVERS *moitor_servers = mon->databases; MONITOR_SERVERS *candidate_master = NULL; GALERA_MONITOR* handle = mon->handle; long min_id = -1; int minval = INT_MAX; int currval; char* value; /* set min_id to the lowest value of moitor_servers->server->node_id */ while (moitor_servers) { if (!SERVER_IN_MAINT(moitor_servers->server) && SERVER_IS_JOINED(moitor_servers->server)) { moitor_servers->server->depth = 0; if (handle->use_priority && (value = serverGetParameter(moitor_servers->server, "priority")) != NULL) { currval = atoi(value); if (currval < minval && currval > 0) { minval = currval; candidate_master = moitor_servers; } } else if (moitor_servers->server->node_id >= 0 && (!handle->use_priority || /** Server priority disabled*/ candidate_master == NULL || /** No candidate chosen */ serverGetParameter(candidate_master->server, "priority") == NULL)) /** Candidate has no priority */ { if (min_id < 0 || moitor_servers->server->node_id < min_id) { min_id = moitor_servers->server->node_id; candidate_master = moitor_servers; } } } moitor_servers = moitor_servers->next; } return candidate_master; }
static MONITOR_SERVERS *get_current_master(MONITOR *mon) { MM_MONITOR* handle = mon->handle; MONITOR_SERVERS *ptr; ptr = mon->databases; while (ptr) { /* The server could be in SERVER_IN_MAINT * that means SERVER_IS_RUNNING returns 0 * Let's check only for SERVER_IS_DOWN: server is not running */ if (SERVER_IS_DOWN(ptr->server)) { ptr = ptr->next; continue; } if (ptr->server->depth == 0) { handle->master = ptr; } ptr = ptr->next; } /* * Return the root master */ if (handle->master != NULL) { /* If the root master is in MAINT, return NULL */ if (SERVER_IN_MAINT(handle->master->server)) { return NULL; } else { return handle->master; } } else { return NULL; } }
/** * Monitor an individual server * * @param database The database to probe */ static void monitorDatabase(MONITOR_SERVERS *database, char *defaultUser, char *defaultPasswd, MYSQL_MONITOR *handle) { MYSQL_ROW row; MYSQL_RES *result; int num_fields; int isjoined = 0; char *uname = defaultUser, *passwd = defaultPasswd; unsigned long int server_version = 0; char *server_string; if (database->server->monuser != NULL) { uname = database->server->monuser; passwd = database->server->monpw; } if (uname == NULL) return; /* Don't even probe server flagged as in maintenance */ if (SERVER_IN_MAINT(database->server)) return; if (database->con == NULL || mysql_ping(database->con) != 0) { char *dpwd = decryptPassword(passwd); int rc; int connect_timeout = handle->connect_timeout; int read_timeout = handle->read_timeout; int write_timeout = handle->write_timeout; database->con = mysql_init(NULL); rc = mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout); rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout); rc = mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *)&write_timeout); if (mysql_real_connect(database->con, database->server->name, uname, dpwd, NULL, database->server->port, NULL, 0) == NULL) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Monitor was unable to connect to " "server %s:%d : \"%s\"", database->server->name, database->server->port, mysql_error(database->con)))); server_clear_status(database->server, SERVER_RUNNING); if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status(database->server, SERVER_AUTH_ERROR); } database->server->node_id = -1; free(dpwd); return; } else { server_clear_status(database->server, SERVER_AUTH_ERROR); } free(dpwd); } /* If we get this far then we have a working connection */ server_set_status(database->server, SERVER_RUNNING); /* get server version from current server */ server_version = mysql_get_server_version(database->con); /* get server version string */ server_string = (char *)mysql_get_server_info(database->con); if (server_string) { database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1); if (database->server->server_string) strcpy(database->server->server_string, server_string); } /* Check if the the SQL node is able to contact one or more data nodes */ if (mysql_query(database->con, "SHOW STATUS LIKE 'Ndb_number_of_ready_data_nodes'") == 0 && (result = mysql_store_result(database->con)) != NULL) { num_fields = mysql_num_fields(result); while ((row = mysql_fetch_row(result))) { if (atoi(row[1]) > 0) isjoined = 1; } mysql_free_result(result); } /* Check the the SQL node id in the MySQL cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'Ndb_cluster_node_id'") == 0 && (result = mysql_store_result(database->con)) != NULL) { long cluster_node_id = -1; num_fields = mysql_num_fields(result); while ((row = mysql_fetch_row(result))) { cluster_node_id = strtol(row[1], NULL, 10); if ((errno == ERANGE && (cluster_node_id == LONG_MAX || cluster_node_id == LONG_MIN)) || (errno != 0 && cluster_node_id == 0)) { cluster_node_id = -1; } database->server->node_id = cluster_node_id; } mysql_free_result(result); } if (isjoined) { server_set_status(database->server, SERVER_NDB); database->server->depth = 0; } else { server_clear_status(database->server, SERVER_NDB); database->server->depth = -1; } }
/** * The entry point for the monitoring module thread * * @param arg The handle of the monitor */ static void monitorMain(void *arg) { MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg; MONITOR_SERVERS *ptr; size_t nrounds = 0; MONITOR_SERVERS *candidate_master = NULL; int master_stickiness = handle->disableMasterFailback; int is_cluster=0; int log_no_members = 1; if (mysql_thread_init()) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Fatal : mysql_thread_init failed in monitor " "module. Exiting.\n"))); return; } handle->status = MONITOR_RUNNING; while (1) { if (handle->shutdown) { handle->status = MONITOR_STOPPING; mysql_thread_end(); handle->status = MONITOR_STOPPED; return; } /** Wait base interval */ thread_millisleep(MON_BASE_INTERVAL_MS); /** * Calculate how far away the monitor interval is from its full * cycle and if monitor interval time further than the base * interval, then skip monitoring checks. Excluding the first * round. */ if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= MON_BASE_INTERVAL_MS) { nrounds += 1; continue; } nrounds += 1; /* reset cluster members counter */ is_cluster=0; ptr = handle->databases; while (ptr) { monitorDatabase(handle, ptr); /* clear bits for non member nodes */ if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) { ptr->server->depth = -1; /* clear M/S status */ server_clear_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); /* clear master sticky status */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } /* Log server status change */ if (mon_status_changed(ptr)) { LOGIF(LD, (skygw_log_write_flush( LOGFILE_DEBUG, "Backend server %s:%d state : %s", ptr->server->name, ptr->server->port, STRSRVSTATUS(ptr->server)))); } if (SERVER_IS_DOWN(ptr->server)) { /** Increase this server'e error count */ ptr->mon_err_count += 1; } else { /** Reset this server's error count */ ptr->mon_err_count = 0; } ptr = ptr->next; } /* * Let's select a master server: * it could be the candidate master following MIN(node_id) rule or * the server that was master in the previous monitor polling cycle * Decision depends on master_stickiness value set in configuration */ /* get the candidate master, followinf MIN(node_id) rule */ candidate_master = get_candidate_master(handle->databases); /* Select the master, based on master_stickiness */ handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness); ptr = handle->databases; while (ptr && handle->master) { if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) { ptr = ptr->next; continue; } if (ptr != handle->master) { /* set the Slave role */ server_set_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); /* clear master stickyness */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } else { /* set the Master role */ server_set_status(handle->master->server, SERVER_MASTER); server_clear_status(handle->master->server, SERVER_SLAVE); if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) { /* set master stickyness */ server_set_status(handle->master->server, SERVER_MASTER_STICKINESS); } else { /* clear master stickyness */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } } is_cluster++; ptr = ptr->next; } if (is_cluster == 0 && log_no_members) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error: there are no cluster members"))); log_no_members = 0; } else { if (is_cluster > 0 && log_no_members == 0) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Info: found cluster members"))); log_no_members = 1; } } } }
/** * Monitor an individual server * * @param handle The MySQL Monitor object * @param database The database to probe */ static void monitorDatabase(MYSQL_MONITOR *handle, MONITOR_SERVERS *database) { MYSQL_ROW row; MYSQL_RES *result; int num_fields; int isjoined = 0; char *uname = handle->defaultUser; char *passwd = handle->defaultPasswd; unsigned long int server_version = 0; char *server_string; if (database->server->monuser != NULL) { uname = database->server->monuser; passwd = database->server->monpw; } if (uname == NULL) return; /* Don't even probe server flagged as in maintenance */ if (SERVER_IN_MAINT(database->server)) return; /** Store previous status */ database->mon_prev_status = database->server->status; if (database->con == NULL || mysql_ping(database->con) != 0) { char *dpwd = decryptPassword(passwd); int rc; int connect_timeout = handle->connect_timeout; int read_timeout = handle->read_timeout; int write_timeout = handle->write_timeout;; database->con = mysql_init(NULL); rc = mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout); rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout); rc = mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *)&write_timeout); if (mysql_real_connect(database->con, database->server->name, uname, dpwd, NULL, database->server->port, NULL, 0) == NULL) { free(dpwd); server_clear_status(database->server, SERVER_RUNNING); /* Also clear Joined, M/S and Stickiness bits */ server_clear_status(database->server, SERVER_JOINED); server_clear_status(database->server, SERVER_SLAVE); server_clear_status(database->server, SERVER_MASTER); server_clear_status(database->server, SERVER_MASTER_STICKINESS); if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status(database->server, SERVER_AUTH_ERROR); } database->server->node_id = -1; if (mon_status_changed(database) && mon_print_fail_status(database)) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Monitor was unable to connect to " "server %s:%d : \"%s\"", database->server->name, database->server->port, mysql_error(database->con)))); } return; } else { server_clear_status(database->server, SERVER_AUTH_ERROR); } free(dpwd); } /* If we get this far then we have a working connection */ server_set_status(database->server, SERVER_RUNNING); /* get server version from current server */ server_version = mysql_get_server_version(database->con); /* get server version string */ server_string = (char *)mysql_get_server_info(database->con); if (server_string) { database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1); if (database->server->server_string) strcpy(database->server->server_string, server_string); } /* Check if the the Galera FSM shows this node is joined to the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state_comment'") == 0 && (result = mysql_store_result(database->con)) != NULL) { num_fields = mysql_num_fields(result); while ((row = mysql_fetch_row(result))) { if (strncasecmp(row[1], "SYNCED", 3) == 0) isjoined = 1; } mysql_free_result(result); } /* Check the the Galera node index in the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0 && (result = mysql_store_result(database->con)) != NULL) { long local_index = -1; num_fields = mysql_num_fields(result); while ((row = mysql_fetch_row(result))) { local_index = strtol(row[1], NULL, 10); if ((errno == ERANGE && (local_index == LONG_MAX || local_index == LONG_MIN)) || (errno != 0 && local_index == 0)) { local_index = -1; } database->server->node_id = local_index; } mysql_free_result(result); } if (isjoined) server_set_status(database->server, SERVER_JOINED); else server_clear_status(database->server, SERVER_JOINED); }
/** * Monitor an individual server. Does not deal with the setting of master or * slave bits, except for clearing them when a server is not joined to the * cluster. * * @param handle The MySQL Monitor object * @param database The database to probe */ static void monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database) { GALERA_MONITOR* handle = (GALERA_MONITOR*) mon->handle; MYSQL_ROW row; MYSQL_RES *result, *result2; int isjoined = 0; unsigned long int server_version = 0; char *server_string; SERVER temp_server; /* Don't even probe server flagged as in maintenance */ if (SERVER_IN_MAINT(database->server)) return; /** Store previous status */ database->mon_prev_status = database->server->status; server_transfer_status(&temp_server, database->server); server_clear_status(&temp_server, SERVER_RUNNING); /* Also clear Joined */ server_clear_status(&temp_server, SERVER_JOINED); connect_result_t rval = mon_connect_to_db(mon, database); if (rval != MONITOR_CONN_OK) { if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status(&temp_server, SERVER_AUTH_ERROR); } else { server_clear_status(&temp_server, SERVER_AUTH_ERROR); } database->server->node_id = -1; if (mon_status_changed(database) && mon_print_fail_status(database)) { mon_log_connect_error(database, rval); } server_transfer_status(database->server, &temp_server); return; } /* If we get this far then we have a working connection */ server_set_status(&temp_server, SERVER_RUNNING); /* get server version string */ server_string = (char *) mysql_get_server_info(database->con); if (server_string) { server_set_version_string(database->server, server_string); } /* Check if the the Galera FSM shows this node is joined to the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state'") == 0 && (result = mysql_store_result(database->con)) != NULL) { if (mysql_field_count(database->con) < 2) { mysql_free_result(result); MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_state'\". " "Expected 2 columns." " MySQL Version: %s", version_str); return; } while ((row = mysql_fetch_row(result))) { if (strcmp(row[1], "4") == 0) isjoined = 1; /* Check if the node is a donor and is using xtrabackup, in this case it can stay alive */ else if (strcmp(row[1], "2") == 0 && handle->availableWhenDonor == 1) { if (mysql_query(database->con, "SHOW VARIABLES LIKE 'wsrep_sst_method'") == 0 && (result2 = mysql_store_result(database->con)) != NULL) { if (mysql_field_count(database->con) < 2) { mysql_free_result(result); mysql_free_result(result2); MXS_ERROR("Unexpected result for \"SHOW VARIABLES LIKE " "'wsrep_sst_method'\". Expected 2 columns." " MySQL Version: %s", version_str); return; } while ((row = mysql_fetch_row(result2))) { if (strncmp(row[1], "xtrabackup", 10) == 0) isjoined = 1; } mysql_free_result(result2); } } } mysql_free_result(result); } /* Check the the Galera node index in the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0 && (result = mysql_store_result(database->con)) != NULL) { long local_index = -1; if (mysql_field_count(database->con) < 2) { mysql_free_result(result); MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_index'\". " "Expected 2 columns." " MySQL Version: %s", version_str); return; } while ((row = mysql_fetch_row(result))) { local_index = strtol(row[1], NULL, 10); if ((errno == ERANGE && (local_index == LONG_MAX || local_index == LONG_MIN)) || (errno != 0 && local_index == 0)) { local_index = -1; } database->server->node_id = local_index; } mysql_free_result(result); } if (isjoined) { server_set_status(&temp_server, SERVER_JOINED); } else { server_clear_status(&temp_server, SERVER_JOINED); } /* clear bits for non member nodes */ if (!SERVER_IN_MAINT(database->server) && (!SERVER_IS_JOINED(&temp_server))) { database->server->depth = -1; /* clear M/S status */ server_clear_status(&temp_server, SERVER_SLAVE); server_clear_status(&temp_server, SERVER_MASTER); /* clear master sticky status */ server_clear_status(&temp_server, SERVER_MASTER_STICKINESS); } server_transfer_status(database->server, &temp_server); }
/** * Associate a new session with this instance of the router. * * @param instance The router instance data * @param session The session itself * @return Session specific data for this session */ static void * newSession(ROUTER *instance, SESSION *session) { ROUTER_INSTANCE *inst = (ROUTER_INSTANCE *)instance; ROUTER_CLIENT_SES *client_rses; BACKEND *candidate = NULL; int i; BACKEND *master_host = NULL; LOGIF(LD, (skygw_log_write_flush( LOGFILE_DEBUG, "%lu [newSession] new router session with session " "%p, and inst %p.", pthread_self(), session, inst))); client_rses = (ROUTER_CLIENT_SES *)calloc(1, sizeof(ROUTER_CLIENT_SES)); if (client_rses == NULL) { return NULL; } #if defined(SS_DEBUG) client_rses->rses_chk_top = CHK_NUM_ROUTER_SES; client_rses->rses_chk_tail = CHK_NUM_ROUTER_SES; #endif /** * Find the Master host from available servers */ master_host = get_root_master(inst->servers); /** * Find a backend server to connect to. This is the extent of the * load balancing algorithm we need to implement for this simple * connection router. */ /* * Loop over all the servers and find any that have fewer connections * than the candidate server. * * If a server has less connections than the current candidate we mark this * as the new candidate to connect to. * * If a server has the same number of connections currently as the candidate * and has had less connections over time than the candidate it will also * become the new candidate. This has the effect of spreading the * connections over different servers during periods of very low load. */ for (i = 0; inst->servers[i]; i++) { if(inst->servers[i]) { LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, "%lu [newSession] Examine server in port %d with " "%d connections. Status is %s, " "inst->bitvalue is %d", pthread_self(), inst->servers[i]->server->port, inst->servers[i]->current_connection_count, STRSRVSTATUS(inst->servers[i]->server), inst->bitmask))); } if (SERVER_IN_MAINT(inst->servers[i]->server)) continue; if (inst->servers[i]->weight == 0) continue; /* Check server status bits against bitvalue from router_options */ if (inst->servers[i] && SERVER_IS_RUNNING(inst->servers[i]->server) && (inst->servers[i]->server->status & inst->bitmask & inst->bitvalue)) { if (master_host) { if (inst->servers[i] == master_host && (inst->bitvalue & SERVER_SLAVE)) { /* skip root Master here, as it could also be slave of an external server * that is not in the configuration. * Intermediate masters (Relay Servers) are also slave and will be selected * as Slave(s) */ continue; } if (inst->servers[i] == master_host && (inst->bitvalue & SERVER_MASTER)) { /* If option is "master" return only the root Master as there * could be intermediate masters (Relay Servers) * and they must not be selected. */ candidate = master_host; break; } } else { /* master_host is NULL, no master server. * If requested router_option is 'master' * candidate wll be NULL. */ if (inst->bitvalue & SERVER_MASTER) { candidate = NULL; break; } } /* If no candidate set, set first running server as our initial candidate server */ if (candidate == NULL) { candidate = inst->servers[i]; } else if ((inst->servers[i]->current_connection_count * 1000) / inst->servers[i]->weight < (candidate->current_connection_count * 1000) / candidate->weight) { /* This running server has fewer connections, set it as a new candidate */ candidate = inst->servers[i]; } else if ((inst->servers[i]->current_connection_count * 1000) / inst->servers[i]->weight == (candidate->current_connection_count * 1000) / candidate->weight && inst->servers[i]->server->stats.n_connections < candidate->server->stats.n_connections) { /* This running server has the same number of connections currently as the candidate but has had fewer connections over time than candidate, set this server to candidate*/ candidate = inst->servers[i]; } } } /* There is no candidate server here! * With router_option=slave a master_host could be set, so route traffic there. * Otherwise, just clean up and return NULL */ if (!candidate) { if (master_host) { candidate = master_host; } else { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Failed to create new routing session. " "Couldn't find eligible candidate server. Freeing " "allocated resources."))); free(client_rses); return NULL; } } client_rses->rses_capabilities = RCAP_TYPE_PACKET_INPUT; /* * We now have the server with the least connections. * Bump the connection count for this server */ atomic_add(&candidate->current_connection_count, 1); client_rses->backend = candidate; LOGIF(LD, (skygw_log_write( LOGFILE_DEBUG, "%lu [newSession] Selected server in port %d. " "Connections : %d\n", pthread_self(), candidate->server->port, candidate->current_connection_count))); /* * Open a backend connection, putting the DCB for this * connection in the client_rses->backend_dcb */ client_rses->backend_dcb = dcb_connect(candidate->server, session, candidate->server->protocol); if (client_rses->backend_dcb == NULL) { atomic_add(&candidate->current_connection_count, -1); free(client_rses); return NULL; } dcb_add_callback( client_rses->backend_dcb, DCB_REASON_NOT_RESPONDING, &handle_state_switch, client_rses); inst->stats.n_sessions++; /** * Add this session to the list of active sessions. */ spinlock_acquire(&inst->lock); client_rses->next = inst->connections; inst->connections = client_rses; spinlock_release(&inst->lock); CHK_CLIENT_RSES(client_rses); skygw_log_write( LOGFILE_TRACE, "Readconnroute: New session for server %s. " "Connections : %d", candidate->server->unique_name, candidate->current_connection_count); return (void *)client_rses; }
/** * The entry point for the monitoring module thread * * @param arg The handle of the monitor */ static void monitorMain(void *arg) { MONITOR* mon = (MONITOR*)arg; MM_MONITOR *handle; MONITOR_SERVERS *ptr; int detect_stale_master; MONITOR_SERVERS *root_master; size_t nrounds = 0; spinlock_acquire(&mon->lock); handle = (MM_MONITOR *)mon->handle; spinlock_release(&mon->lock); detect_stale_master = handle->detectStaleMaster; if (mysql_thread_init()) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Fatal : mysql_thread_init failed in monitor " "module. Exiting.\n"))); return; } handle->status = MONITOR_RUNNING; while (1) { if (handle->shutdown) { handle->status = MONITOR_STOPPING; mysql_thread_end(); handle->status = MONITOR_STOPPED; return; } /** Wait base interval */ thread_millisleep(MON_BASE_INTERVAL_MS); /** * Calculate how far away the monitor interval is from its full * cycle and if monitor interval time further than the base * interval, then skip monitoring checks. Excluding the first * round. */ if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%mon->interval) >= MON_BASE_INTERVAL_MS) { nrounds += 1; continue; } nrounds += 1; /* start from the first server in the list */ ptr = mon->databases; while (ptr) { /* copy server status into monitor pending_status */ ptr->pending_status = ptr->server->status; /* monitor current node */ monitorDatabase(mon, ptr); if (mon_status_changed(ptr)) { dcb_call_foreach(ptr->server,DCB_REASON_NOT_RESPONDING); } if (mon_status_changed(ptr) || mon_print_fail_status(ptr)) { LOGIF(LD, (skygw_log_write_flush( LOGFILE_DEBUG, "Backend server %s:%d state : %s", ptr->server->name, ptr->server->port, STRSRVSTATUS(ptr->server)))); } if (SERVER_IS_DOWN(ptr->server)) { /** Increase this server'e error count */ ptr->mon_err_count += 1; } else { /** Reset this server's error count */ ptr->mon_err_count = 0; } ptr = ptr->next; } /* Get Master server pointer */ root_master = get_current_master(mon); /* Update server status from monitor pending status on that server*/ ptr = mon->databases; while (ptr) { if (! SERVER_IN_MAINT(ptr->server)) { /* If "detect_stale_master" option is On, let's use the previus master */ if (detect_stale_master && root_master && (!strcmp(ptr->server->name, root_master->server->name) && ptr->server->port == root_master->server->port) && (ptr->server->status & SERVER_MASTER) && !(ptr->pending_status & SERVER_MASTER)) { /* in this case server->status will not be updated from pending_status */ LOGIF(LM, (skygw_log_write_flush( LOGFILE_MESSAGE, "[mysql_mon]: root server [%s:%i] is no longer Master, let's use it again even if it could be a stale master, you have been warned!", ptr->server->name, ptr->server->port))); /* Set the STALE bit for this server in server struct */ server_set_status(ptr->server, SERVER_STALE_STATUS); } else { ptr->server->status = ptr->pending_status; } } ptr = ptr->next; } ptr = mon->databases; monitor_event_t evtype; while(ptr) { if(mon_status_changed(ptr)) { evtype = mon_get_event_type(ptr); if(isMySQLEvent(evtype)) { skygw_log_write(LOGFILE_TRACE,"Server changed state: %s[%s:%u]: %s", ptr->server->unique_name, ptr->server->name,ptr->server->port, mon_get_event_name(ptr)); if(handle->script && handle->events[evtype]) { monitor_launch_script(mon,ptr,handle->script); } } } ptr = ptr->next; } } }
/** * Monitor an individual server * * @param handle The MySQL Monitor object * @param database The database to probe */ static void monitorDatabase(MONITOR* mon, MONITOR_SERVERS *database) { MM_MONITOR *handle = mon->handle; MYSQL_ROW row; MYSQL_RES *result; int isslave = 0; int ismaster = 0; char *uname = mon->user; char *passwd = mon->password; unsigned long int server_version = 0; char *server_string; if (database->server->monuser != NULL) { uname = database->server->monuser; passwd = database->server->monpw; } if (uname == NULL) return; /* Don't probe servers in maintenance mode */ if (SERVER_IN_MAINT(database->server)) return; /** Store previous status */ database->mon_prev_status = database->server->status; if (database->con == NULL || mysql_ping(database->con) != 0) { char *dpwd = decryptPassword(passwd); int read_timeout = 1; if(database->con) mysql_close(database->con); database->con = mysql_init(NULL); mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout); if (mysql_real_connect(database->con, database->server->name, uname, dpwd, NULL, database->server->port, NULL, 0) == NULL) { free(dpwd); if (mon_print_fail_status(database)) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Monitor was unable to connect to " "server %s:%d : \"%s\"", database->server->name, database->server->port, mysql_error(database->con)))); } /* The current server is not running * * Store server NOT running in server and monitor server pending struct * */ if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status(database->server, SERVER_AUTH_ERROR); monitor_set_pending_status(database, SERVER_AUTH_ERROR); } server_clear_status(database->server, SERVER_RUNNING); monitor_clear_pending_status(database, SERVER_RUNNING); /* Also clear M/S state in both server and monitor server pending struct */ server_clear_status(database->server, SERVER_SLAVE); server_clear_status(database->server, SERVER_MASTER); monitor_clear_pending_status(database, SERVER_SLAVE); monitor_clear_pending_status(database, SERVER_MASTER); /* Clean addition status too */ server_clear_status(database->server, SERVER_STALE_STATUS); monitor_clear_pending_status(database, SERVER_STALE_STATUS); return; } else { server_clear_status(database->server, SERVER_AUTH_ERROR); monitor_clear_pending_status(database, SERVER_AUTH_ERROR); } free(dpwd); } /* Store current status in both server and monitor server pending struct */ server_set_status(database->server, SERVER_RUNNING); monitor_set_pending_status(database, SERVER_RUNNING); /* get server version from current server */ server_version = mysql_get_server_version(database->con); /* get server version string */ server_string = (char *)mysql_get_server_info(database->con); if (server_string) { database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1); if (database->server->server_string) strcpy(database->server->server_string, server_string); } /* get server_id form current node */ if (mysql_query(database->con, "SELECT @@server_id") == 0 && (result = mysql_store_result(database->con)) != NULL) { long server_id = -1; if(mysql_field_count(database->con) != 1) { mysql_free_result(result); skygw_log_write(LE,"Error: Unexpected result for 'SELECT @@server_id'. Expected 1 column." " MySQL Version: %s",version_str); return; } while ((row = mysql_fetch_row(result))) { server_id = strtol(row[0], NULL, 10); if ((errno == ERANGE && (server_id == LONG_MAX || server_id == LONG_MIN)) || (errno != 0 && server_id == 0)) { server_id = -1; } database->server->node_id = server_id; } mysql_free_result(result); } /* Check if the Slave_SQL_Running and Slave_IO_Running status is * set to Yes */ /* Check first for MariaDB 10.x.x and get status for multimaster replication */ if (server_version >= 100000) { if (mysql_query(database->con, "SHOW ALL SLAVES STATUS") == 0 && (result = mysql_store_result(database->con)) != NULL) { int i = 0; long master_id = -1; if(mysql_field_count(database->con) < 42) { mysql_free_result(result); skygw_log_write(LE,"Error: \"SHOW ALL SLAVES STATUS\" " "returned less than the expected amount of columns. Expected 42 columns" " MySQL Version: %s",version_str); return; } while ((row = mysql_fetch_row(result))) { /* get Slave_IO_Running and Slave_SQL_Running values*/ if (strncmp(row[12], "Yes", 3) == 0 && strncmp(row[13], "Yes", 3) == 0) { isslave += 1; } /* If Slave_IO_Running = Yes, assign the master_id to current server: this allows building * the replication tree, slaves ids will be added to master(s) and we will have at least the * root master server. * Please note, there could be no slaves at all if Slave_SQL_Running == 'No' */ if (strncmp(row[12], "Yes", 3) == 0) { /* get Master_Server_Id values */ master_id = atol(row[41]); if (master_id == 0) master_id = -1; } i++; } /* store master_id of current node */ memcpy(&database->server->master_id, &master_id, sizeof(long)); mysql_free_result(result); /* If all configured slaves are running set this node as slave */ if (isslave > 0 && isslave == i) isslave = 1; else isslave = 0; } } else { if (mysql_query(database->con, "SHOW SLAVE STATUS") == 0 && (result = mysql_store_result(database->con)) != NULL) { long master_id = -1; if(mysql_field_count(database->con) < 40) { mysql_free_result(result); if(server_version < 5*10000 + 5*100) { if(database->log_version_err) { skygw_log_write(LE,"Error: \"SHOW SLAVE STATUS\" " " for versions less than 5.5 does not have master_server_id, " "replication tree cannot be resolved for server %s." " MySQL Version: %s",database->server->unique_name,version_str); database->log_version_err = false; } } else { skygw_log_write(LE,"Error: \"SHOW SLAVE STATUS\" " "returned less than the expected amount of columns. Expected 40 columns." " MySQL Version: %s",version_str); } return; } while ((row = mysql_fetch_row(result))) { /* get Slave_IO_Running and Slave_SQL_Running values*/ if (strncmp(row[10], "Yes", 3) == 0 && strncmp(row[11], "Yes", 3) == 0) { isslave = 1; } /* If Slave_IO_Running = Yes, assign the master_id to current server: this allows building * the replication tree, slaves ids will be added to master(s) and we will have at least the * root master server. * Please note, there could be no slaves at all if Slave_SQL_Running == 'No' */ if (strncmp(row[10], "Yes", 3) == 0) { /* get Master_Server_Id values */ master_id = atol(row[39]); if (master_id == 0) master_id = -1; } } /* store master_id of current node */ memcpy(&database->server->master_id, &master_id, sizeof(long)); mysql_free_result(result); } } /* get variable 'read_only' set by an external component */ if (mysql_query(database->con, "SHOW GLOBAL VARIABLES LIKE 'read_only'") == 0 && (result = mysql_store_result(database->con)) != NULL) { if(mysql_field_count(database->con) < 2) { mysql_free_result(result); skygw_log_write(LE,"Error: Unexpected result for \"SHOW GLOBAL VARIABLES LIKE 'read_only'\". Expected 2 columns." " MySQL Version: %s",version_str); return; } while ((row = mysql_fetch_row(result))) { if (strncasecmp(row[1], "OFF", 3) == 0) { ismaster = 1; } else { isslave = 1; } } mysql_free_result(result); } /* Remove addition info */ monitor_clear_pending_status(database, SERVER_STALE_STATUS); /* Set the Slave Role */ if (isslave) { monitor_set_pending_status(database, SERVER_SLAVE); /* Avoid any possible stale Master state */ monitor_clear_pending_status(database, SERVER_MASTER); /* Set replication depth to 1 */ database->server->depth = 1; } else { /* Avoid any possible Master/Slave stale state */ monitor_clear_pending_status(database, SERVER_SLAVE); monitor_clear_pending_status(database, SERVER_MASTER); } /* Set the Master role */ if (ismaster) { monitor_clear_pending_status(database, SERVER_SLAVE); monitor_set_pending_status(database, SERVER_MASTER); /* Set replication depth to 0 */ database->server->depth = 0; } }
/** * The entry point for the monitoring module thread * * @param arg The handle of the monitor */ static void monitorMain(void *arg) { MONITOR* mon = (MONITOR*)arg; GALERA_MONITOR *handle; MONITOR_SERVERS *ptr; size_t nrounds = 0; MONITOR_SERVERS *candidate_master = NULL; int master_stickiness; int is_cluster=0; int log_no_members = 1; monitor_event_t evtype; spinlock_acquire(&mon->lock); handle = (GALERA_MONITOR *)mon->handle; spinlock_release(&mon->lock); master_stickiness = handle->disableMasterFailback; if (mysql_thread_init()) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Fatal : mysql_thread_init failed in monitor " "module. Exiting.\n"))); return; } handle->status = MONITOR_RUNNING; while (1) { if (handle->shutdown) { handle->status = MONITOR_STOPPING; mysql_thread_end(); handle->status = MONITOR_STOPPED; return; } /** Wait base interval */ thread_millisleep(MON_BASE_INTERVAL_MS); /** * Calculate how far away the monitor interval is from its full * cycle and if monitor interval time further than the base * interval, then skip monitoring checks. Excluding the first * round. */ if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%mon->interval) >= MON_BASE_INTERVAL_MS) { nrounds += 1; continue; } nrounds += 1; /* reset cluster members counter */ is_cluster=0; ptr = mon->databases; while (ptr) { ptr->mon_prev_status = ptr->server->status; monitorDatabase(mon, ptr); /* clear bits for non member nodes */ if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) { ptr->server->depth = -1; /* clear M/S status */ server_clear_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); /* clear master sticky status */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } /* Log server status change */ if (mon_status_changed(ptr)) { LOGIF(LD, (skygw_log_write_flush( LOGFILE_DEBUG, "Backend server %s:%d state : %s", ptr->server->name, ptr->server->port, STRSRVSTATUS(ptr->server)))); } if (!(SERVER_IS_RUNNING(ptr->server)) || !(SERVER_IS_IN_CLUSTER(ptr->server))) { dcb_call_foreach(ptr->server,DCB_REASON_NOT_RESPONDING); } if (SERVER_IS_DOWN(ptr->server)) { /** Increase this server'e error count */ dcb_call_foreach(ptr->server,DCB_REASON_NOT_RESPONDING); ptr->mon_err_count += 1; } else { /** Reset this server's error count */ ptr->mon_err_count = 0; } ptr = ptr->next; } /* * Let's select a master server: * it could be the candidate master following MIN(node_id) rule or * the server that was master in the previous monitor polling cycle * Decision depends on master_stickiness value set in configuration */ /* get the candidate master, following MIN(node_id) rule */ candidate_master = get_candidate_master(mon->databases); /* Select the master, based on master_stickiness */ if (1 == handle->disableMasterRoleSetting) { handle->master = NULL; } else { handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness); } ptr = mon->databases; while (ptr) { if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) { ptr = ptr->next; continue; } if (handle->master) { if (ptr != handle->master) { /* set the Slave role */ server_set_status(ptr->server, SERVER_SLAVE); server_clear_status(ptr->server, SERVER_MASTER); /* clear master stickiness */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } else { /* set the Master role */ server_set_status(handle->master->server, SERVER_MASTER); server_clear_status(handle->master->server, SERVER_SLAVE); if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) { /* set master stickiness */ server_set_status(handle->master->server, SERVER_MASTER_STICKINESS); } else { /* clear master stickiness */ server_clear_status(ptr->server, SERVER_MASTER_STICKINESS); } } } is_cluster++; ptr = ptr->next; } if (is_cluster == 0 && log_no_members) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error: there are no cluster members"))); log_no_members = 0; } else { if (is_cluster > 0 && log_no_members == 0) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Info: found cluster members"))); log_no_members = 1; } } ptr = mon->databases; while(ptr) { /** Execute monitor script if a server state has changed */ if(mon_status_changed(ptr)) { evtype = mon_get_event_type(ptr); if(isGaleraEvent(evtype)) { skygw_log_write(LOGFILE_TRACE,"Server changed state: %s[%s:%u]: %s", ptr->server->unique_name, ptr->server->name,ptr->server->port, mon_get_event_name(ptr)); if(handle->script && handle->events[evtype]) { monitor_launch_script(mon,ptr,handle->script); } } } ptr = ptr->next; } } }
/** * Monitor an individual server * * @param handle The MySQL Monitor object * @param database The database to probe */ static void monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database) { GALERA_MONITOR* handle = (GALERA_MONITOR*)mon->handle; MYSQL_ROW row; MYSQL_RES *result,*result2; int isjoined = 0; char *uname = mon->user; char *passwd = mon->password; unsigned long int server_version = 0; char *server_string; if (database->server->monuser != NULL) { uname = database->server->monuser; passwd = database->server->monpw; } if (uname == NULL) return; /* Don't even probe server flagged as in maintenance */ if (SERVER_IN_MAINT(database->server)) return; /** Store previous status */ database->mon_prev_status = database->server->status; if (database->con == NULL || mysql_ping(database->con) != 0) { char *dpwd = decryptPassword(passwd); int connect_timeout = mon->connect_timeout; int read_timeout = mon->read_timeout; int write_timeout = mon->write_timeout; if(database->con) mysql_close(database->con); database->con = mysql_init(NULL); mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout); mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout); mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *)&write_timeout); if (mysql_real_connect(database->con, database->server->name, uname, dpwd, NULL, database->server->port, NULL, 0) == NULL) { free(dpwd); server_clear_status(database->server, SERVER_RUNNING); /* Also clear Joined, M/S and Stickiness bits */ server_clear_status(database->server, SERVER_JOINED); server_clear_status(database->server, SERVER_SLAVE); server_clear_status(database->server, SERVER_MASTER); server_clear_status(database->server, SERVER_MASTER_STICKINESS); if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR) { server_set_status(database->server, SERVER_AUTH_ERROR); } database->server->node_id = -1; if (mon_status_changed(database) && mon_print_fail_status(database)) { LOGIF(LE, (skygw_log_write_flush( LOGFILE_ERROR, "Error : Monitor was unable to connect to " "server %s:%d : \"%s\"", database->server->name, database->server->port, mysql_error(database->con)))); } return; } else { server_clear_status(database->server, SERVER_AUTH_ERROR); } free(dpwd); } /* If we get this far then we have a working connection */ server_set_status(database->server, SERVER_RUNNING); /* get server version string */ server_string = (char *)mysql_get_server_info(database->con); if (server_string) { database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1); if (database->server->server_string) strcpy(database->server->server_string, server_string); } /* Check if the the Galera FSM shows this node is joined to the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state'") == 0 && (result = mysql_store_result(database->con)) != NULL) { if(mysql_field_count(database->con) < 2) { mysql_free_result(result); skygw_log_write(LE,"Error: Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_state'\". Expected 2 columns." " MySQL Version: %s",version_str); return; } while ((row = mysql_fetch_row(result))) { if (strcmp(row[1], "4") == 0) isjoined = 1; /* Check if the node is a donor and is using xtrabackup, in this case it can stay alive */ else if (strcmp(row[1], "2") == 0 && handle->availableWhenDonor == 1) { if (mysql_query(database->con, "SHOW VARIABLES LIKE 'wsrep_sst_method'") == 0 && (result2 = mysql_store_result(database->con)) != NULL) { if(mysql_field_count(database->con) < 2) { mysql_free_result(result); mysql_free_result(result2); skygw_log_write(LE,"Error: Unexpected result for \"SHOW VARIABLES LIKE 'wsrep_sst_method'\". Expected 2 columns." " MySQL Version: %s",version_str); return; } while ((row = mysql_fetch_row(result))) { if (strncmp(row[1], "xtrabackup", 10) == 0) isjoined = 1; } mysql_free_result(result2); } } } mysql_free_result(result); } /* Check the the Galera node index in the cluster */ if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0 && (result = mysql_store_result(database->con)) != NULL) { long local_index = -1; if(mysql_field_count(database->con) < 2) { mysql_free_result(result); skygw_log_write(LE,"Error: Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_index'\". Expected 2 columns." " MySQL Version: %s",version_str); return; } while ((row = mysql_fetch_row(result))) { local_index = strtol(row[1], NULL, 10); if ((errno == ERANGE && (local_index == LONG_MAX || local_index == LONG_MIN)) || (errno != 0 && local_index == 0)) { local_index = -1; } database->server->node_id = local_index; } mysql_free_result(result); } if (isjoined) server_set_status(database->server, SERVER_JOINED); else server_clear_status(database->server, SERVER_JOINED); }