Exemplo n.º 1
0
/**
 * get candidate master from all nodes
 *
 * The current available rule: get the server with MIN(node_id)
 * node_id comes from 'wsrep_local_index' variable
 *
 * @param	servers The monitored servers list
 * @return	The candidate master on success, NULL on failure
 */
static MONITOR_SERVERS *get_candidate_master(MONITOR_SERVERS *servers) {
	MONITOR_SERVERS *ptr = servers;
	MONITOR_SERVERS *candidate_master = NULL;
	long min_id = -1;
	
	/* set min_id to the lowest value of ptr->server->node_id */
	while(ptr) {
		if ((! SERVER_IN_MAINT(ptr->server)) && ptr->server->node_id >= 0 && SERVER_IS_JOINED(ptr->server)) {
			ptr->server->depth = 0;
			if ((ptr->server->node_id < min_id) && min_id >= 0) {
				min_id = ptr->server->node_id;
				candidate_master = ptr;
			} else {
				if (min_id < 0) {
					min_id = ptr->server->node_id;
					candidate_master = ptr;
				}
			}
		}

		ptr = ptr->next;
	}

	return candidate_master;
}
Exemplo n.º 2
0
/**
 * set the master server in the cluster
 *
 * master could be the last one from previous monitor cycle Iis running) or
 * the candidate master.
 * The selection is based on the configuration option mapped to master_stickiness
 * The candidate master may change over time due to
 * 'wsrep_local_index' value change in the Galera Cluster
 * Enabling master_stickiness will avoid master change unless a failure is spotted
 *
 * @param	current_master Previous master server
 * @param	candidate_master The candidate master server accordingly to the selection rule
 * @return	The  master node pointer (could be NULL)
 */
static MONITOR_SERVERS *set_cluster_master(MONITOR_SERVERS *current_master, MONITOR_SERVERS *candidate_master, int master_stickiness)
{
    /*
     * if current master is not set or master_stickiness is not enable
     * just return candidate_master.
     */
    if (current_master == NULL || master_stickiness == 0)
    {
        return candidate_master;
    }
    else
    {
        /*
         * if current_master is still a cluster member use it
         *
         */
        if (SERVER_IS_JOINED(current_master->server) && (!SERVER_IN_MAINT(current_master->server)))
        {
            return current_master;
        }
        else
        {
            return candidate_master;
        }
    }
}
Exemplo n.º 3
0
/**
 * get candidate master from all nodes
 *
 * The current available rule: get the server with MIN(node_id)
 * node_id comes from 'wsrep_local_index' variable
 *
 * @param	servers The monitored servers list
 * @return	The candidate master on success, NULL on failure
 */
static MONITOR_SERVERS *get_candidate_master(MONITOR* mon)
{
    MONITOR_SERVERS *moitor_servers = mon->databases;
    MONITOR_SERVERS *candidate_master = NULL;
    GALERA_MONITOR* handle = mon->handle;
    long min_id = -1;
    int minval = INT_MAX;
    int currval;
    char* value;
    /* set min_id to the lowest value of moitor_servers->server->node_id */
    while (moitor_servers)
    {
        if (!SERVER_IN_MAINT(moitor_servers->server) && SERVER_IS_JOINED(moitor_servers->server))
        {

            moitor_servers->server->depth = 0;

            if (handle->use_priority && (value = serverGetParameter(moitor_servers->server, "priority")) != NULL)
            {
                currval = atoi(value);
                if (currval < minval && currval > 0)
                {
                    minval = currval;
                    candidate_master = moitor_servers;
                }
            }
            else if (moitor_servers->server->node_id >= 0 &&
                     (!handle->use_priority || /** Server priority disabled*/
                      candidate_master == NULL || /** No candidate chosen */
                      serverGetParameter(candidate_master->server, "priority") == NULL)) /** Candidate has no priority */
            {
                if (min_id < 0 || moitor_servers->server->node_id < min_id)
                {
                    min_id = moitor_servers->server->node_id;
                    candidate_master = moitor_servers;
                }
            }
        }
        moitor_servers = moitor_servers->next;
    }

    return candidate_master;
}
Exemplo n.º 4
0
static MONITOR_SERVERS *get_current_master(MONITOR *mon) {
    MM_MONITOR* handle = mon->handle;
MONITOR_SERVERS *ptr;

	ptr = mon->databases;

	while (ptr)
	{
		/* The server could be in SERVER_IN_MAINT
		 * that means SERVER_IS_RUNNING returns 0
		 * Let's check only for SERVER_IS_DOWN: server is not running
		 */
		if (SERVER_IS_DOWN(ptr->server)) {
			ptr = ptr->next;
			continue;
		}

		if (ptr->server->depth == 0) {
			handle->master = ptr;
		}

		ptr = ptr->next;
	}


	/*
	 * Return the root master
	 */

	if (handle->master != NULL) {
		/* If the root master is in MAINT, return NULL */
		if (SERVER_IN_MAINT(handle->master->server)) {
			return NULL;
		} else {
			return handle->master;
		}
	} else {
		return NULL;
	}
}
Exemplo n.º 5
0
/**
 * Monitor an individual server
 *
 * @param database	The database to probe
 */
static void
monitorDatabase(MONITOR_SERVERS	*database, char *defaultUser, char *defaultPasswd, MYSQL_MONITOR *handle)
{
MYSQL_ROW	row;
MYSQL_RES	*result;
int		num_fields;
int		isjoined = 0;
char            *uname = defaultUser, *passwd = defaultPasswd;
unsigned long int	server_version = 0;
char 			*server_string;

	if (database->server->monuser != NULL)
	{
		uname = database->server->monuser;
		passwd = database->server->monpw;
	}
	if (uname == NULL)
		return;

	/* Don't even probe server flagged as in maintenance */
	if (SERVER_IN_MAINT(database->server))
		return;

	if (database->con == NULL || mysql_ping(database->con) != 0)
	{
		char *dpwd = decryptPassword(passwd);
		int rc;
                int connect_timeout = handle->connect_timeout;
                int read_timeout = handle->read_timeout;
                int write_timeout = handle->write_timeout;

                database->con = mysql_init(NULL);

                rc = mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout);
                rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
                rc = mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *)&write_timeout);

		if (mysql_real_connect(database->con, database->server->name,
			uname, dpwd, NULL, database->server->port, NULL, 0) == NULL)
		{
			LOGIF(LE, (skygw_log_write_flush(
				LOGFILE_ERROR,
				"Error : Monitor was unable to connect to "
				"server %s:%d : \"%s\"",
				database->server->name,
				database->server->port,
				mysql_error(database->con))));
			server_clear_status(database->server, SERVER_RUNNING);
			if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR)
			{
				server_set_status(database->server, SERVER_AUTH_ERROR);
			}
			database->server->node_id = -1;
			free(dpwd);
			return;
		}
		else
		{
			server_clear_status(database->server, SERVER_AUTH_ERROR);
		}
		free(dpwd);
	}

	/* If we get this far then we have a working connection */
	server_set_status(database->server, SERVER_RUNNING);

	/* get server version from current server */
	server_version = mysql_get_server_version(database->con);

	/* get server version string */
	server_string = (char *)mysql_get_server_info(database->con);
	if (server_string) {
		database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1);
		if (database->server->server_string)
			strcpy(database->server->server_string, server_string);
	}	

	/* Check if the the SQL node is able to contact one or more data nodes */
	if (mysql_query(database->con, "SHOW STATUS LIKE 'Ndb_number_of_ready_data_nodes'") == 0
		&& (result = mysql_store_result(database->con)) != NULL)
	{
		num_fields = mysql_num_fields(result);
		while ((row = mysql_fetch_row(result)))
		{
			if (atoi(row[1]) > 0)
				isjoined = 1;
		}
		mysql_free_result(result);
	}

	/* Check the the SQL node id in the MySQL cluster */
	if (mysql_query(database->con, "SHOW STATUS LIKE 'Ndb_cluster_node_id'") == 0
		&& (result = mysql_store_result(database->con)) != NULL)
	{
		long cluster_node_id = -1;
		num_fields = mysql_num_fields(result);
		while ((row = mysql_fetch_row(result)))
		{
			cluster_node_id = strtol(row[1], NULL, 10);
			if ((errno == ERANGE && (cluster_node_id == LONG_MAX
				|| cluster_node_id == LONG_MIN)) || (errno != 0 && cluster_node_id == 0))
			{
				cluster_node_id = -1;
			}
			database->server->node_id = cluster_node_id;
		}
		mysql_free_result(result);
	}

	if (isjoined) {
		server_set_status(database->server, SERVER_NDB);
		database->server->depth = 0;
	} else {
		server_clear_status(database->server, SERVER_NDB);
		database->server->depth = -1;
	}
}
Exemplo n.º 6
0
/**
 * The entry point for the monitoring module thread
 *
 * @param arg	The handle of the monitor
 */
static void
monitorMain(void *arg)
{
MYSQL_MONITOR		*handle = (MYSQL_MONITOR *)arg;
MONITOR_SERVERS		*ptr;
size_t			nrounds = 0;
MONITOR_SERVERS		*candidate_master = NULL;
int			master_stickiness = handle->disableMasterFailback;
int			is_cluster=0;
int			log_no_members = 1;

	if (mysql_thread_init())
	{
                LOGIF(LE, (skygw_log_write_flush(
                                   LOGFILE_ERROR,
                                   "Fatal : mysql_thread_init failed in monitor "
                                   "module. Exiting.\n")));
                return;
	}                         
	handle->status = MONITOR_RUNNING;
	
	while (1)
	{
		if (handle->shutdown)
		{
			handle->status = MONITOR_STOPPING;
			mysql_thread_end();
			handle->status = MONITOR_STOPPED;
			return;
		}
		/** Wait base interval */
		thread_millisleep(MON_BASE_INTERVAL_MS);
		/** 
		 * Calculate how far away the monitor interval is from its full 
		 * cycle and if monitor interval time further than the base 
		 * interval, then skip monitoring checks. Excluding the first
		 * round.
		 */ 

		if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= MON_BASE_INTERVAL_MS) 
		{
			nrounds += 1;
			continue;
		}

		nrounds += 1;

		/* reset cluster members counter */
		is_cluster=0;

		ptr = handle->databases;

		while (ptr)
		{
			monitorDatabase(handle, ptr);

			/* clear bits for non member nodes */
			if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) {
				ptr->server->depth = -1;

				/* clear M/S status */
				server_clear_status(ptr->server, SERVER_SLAVE);
                		server_clear_status(ptr->server, SERVER_MASTER);
				
				/* clear master sticky status */
				server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
			}

			/* Log server status change */
			if (mon_status_changed(ptr))
			{
				LOGIF(LD, (skygw_log_write_flush(
					LOGFILE_DEBUG,
					"Backend server %s:%d state : %s",
					ptr->server->name,
					ptr->server->port,
					STRSRVSTATUS(ptr->server))));
			}

			if (SERVER_IS_DOWN(ptr->server))
			{
				/** Increase this server'e error count */
				ptr->mon_err_count += 1;
			}
			else
			{
				/** Reset this server's error count */
				ptr->mon_err_count = 0;
			}

			ptr = ptr->next;
		}

		/*
		 * Let's select a master server:
		 * it could be the candidate master following MIN(node_id) rule or
		 * the server that was master in the previous monitor polling cycle
		 * Decision depends on master_stickiness value set in configuration
		 */

		/* get the candidate master, followinf MIN(node_id) rule */
		candidate_master = get_candidate_master(handle->databases);

		/* Select the master, based on master_stickiness */
		handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness);

		ptr = handle->databases;

		while (ptr && handle->master) {
			if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) {
				ptr = ptr->next;
				continue;
			}

			if (ptr != handle->master) {
				/* set the Slave role */
				server_set_status(ptr->server, SERVER_SLAVE);
				server_clear_status(ptr->server, SERVER_MASTER);

				/* clear master stickyness */
				server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
			} else {
				/* set the Master role */
				server_set_status(handle->master->server, SERVER_MASTER);
				server_clear_status(handle->master->server, SERVER_SLAVE);

				if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) {
					/* set master stickyness */
					server_set_status(handle->master->server, SERVER_MASTER_STICKINESS);
				} else {
					/* clear master stickyness */
					server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
				}			
			}

			is_cluster++;

			ptr = ptr->next;
		}

		if (is_cluster == 0 && log_no_members) {
			LOGIF(LE, (skygw_log_write_flush(
					LOGFILE_ERROR,
					"Error: there are no cluster members")));
			log_no_members = 0;
		} else {
			if (is_cluster > 0 && log_no_members == 0) {
				LOGIF(LE, (skygw_log_write_flush(
					LOGFILE_ERROR,
					"Info: found cluster members")));
				log_no_members = 1;
			}
		}
	}
}
Exemplo n.º 7
0
/**
 * Monitor an individual server
 *
 * @param handle        The MySQL Monitor object
 * @param database      The database to probe
 */
static void
monitorDatabase(MYSQL_MONITOR *handle, MONITOR_SERVERS *database)
{
MYSQL_ROW	row;
MYSQL_RES	*result;
int		num_fields;
int		isjoined = 0;
char		*uname  = handle->defaultUser;
char		*passwd = handle->defaultPasswd;
unsigned long int	server_version = 0;
char 			*server_string;

	if (database->server->monuser != NULL)
	{
		uname = database->server->monuser;
		passwd = database->server->monpw;
	}
	if (uname == NULL)
		return;

	/* Don't even probe server flagged as in maintenance */
	if (SERVER_IN_MAINT(database->server))
		return;

	/** Store previous status */
	database->mon_prev_status = database->server->status;

	if (database->con == NULL || mysql_ping(database->con) != 0)
	{
		char *dpwd = decryptPassword(passwd);
		int rc;
		int connect_timeout = handle->connect_timeout;
		int read_timeout = handle->read_timeout;
		int write_timeout = handle->write_timeout;;

		database->con = mysql_init(NULL);

		rc = mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout);
		rc = mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
		rc = mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *)&write_timeout);

		if (mysql_real_connect(database->con, database->server->name,
			uname, dpwd, NULL, database->server->port, NULL, 0) == NULL)
		{
			free(dpwd);

			server_clear_status(database->server, SERVER_RUNNING);

			/* Also clear Joined, M/S and Stickiness bits */
			server_clear_status(database->server, SERVER_JOINED);
			server_clear_status(database->server, SERVER_SLAVE);
			server_clear_status(database->server, SERVER_MASTER);
			server_clear_status(database->server, SERVER_MASTER_STICKINESS);

			if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR)
			{
				server_set_status(database->server, SERVER_AUTH_ERROR);
			}

			database->server->node_id = -1;

			if (mon_status_changed(database) && mon_print_fail_status(database))
			{
				LOGIF(LE, (skygw_log_write_flush(
					LOGFILE_ERROR,
					"Error : Monitor was unable to connect to "
					"server %s:%d : \"%s\"",
					database->server->name,
					database->server->port,
					mysql_error(database->con))));
			}

			return;
		}
		else
		{
			server_clear_status(database->server, SERVER_AUTH_ERROR);
		}
		free(dpwd);
	}

	/* If we get this far then we have a working connection */
	server_set_status(database->server, SERVER_RUNNING);

	/* get server version from current server */
	server_version = mysql_get_server_version(database->con);

	/* get server version string */
	server_string = (char *)mysql_get_server_info(database->con);
	if (server_string) {
		database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1);
		if (database->server->server_string)
			strcpy(database->server->server_string, server_string);
	}	

	/* Check if the the Galera FSM shows this node is joined to the cluster */
	if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state_comment'") == 0
		&& (result = mysql_store_result(database->con)) != NULL)
	{
		num_fields = mysql_num_fields(result);
		while ((row = mysql_fetch_row(result)))
		{
			if (strncasecmp(row[1], "SYNCED", 3) == 0)
				isjoined = 1;
		}
		mysql_free_result(result);
	}

	/* Check the the Galera node index in the cluster */
	if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0
		&& (result = mysql_store_result(database->con)) != NULL)
	{
		long local_index = -1;
		num_fields = mysql_num_fields(result);
		while ((row = mysql_fetch_row(result)))
		{
			local_index = strtol(row[1], NULL, 10);
			if ((errno == ERANGE && (local_index == LONG_MAX
				|| local_index == LONG_MIN)) || (errno != 0 && local_index == 0))
			{
				local_index = -1;
			}
			database->server->node_id = local_index;
		}
		mysql_free_result(result);
	}

	if (isjoined)
		server_set_status(database->server, SERVER_JOINED);
	else
		server_clear_status(database->server, SERVER_JOINED);
}
Exemplo n.º 8
0
/**
 * Monitor an individual server. Does not deal with the setting of master or
 * slave bits, except for clearing them when a server is not joined to the
 * cluster.
 *
 * @param handle        The MySQL Monitor object
 * @param database      The database to probe
 */
static void
monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database)
{
    GALERA_MONITOR* handle = (GALERA_MONITOR*) mon->handle;
    MYSQL_ROW row;
    MYSQL_RES *result, *result2;
    int isjoined = 0;
    unsigned long int server_version = 0;
    char *server_string;
    SERVER temp_server;

    /* Don't even probe server flagged as in maintenance */
    if (SERVER_IN_MAINT(database->server))
        return;

    /** Store previous status */
    database->mon_prev_status = database->server->status;

    server_transfer_status(&temp_server, database->server);
    server_clear_status(&temp_server, SERVER_RUNNING);
    /* Also clear Joined */
    server_clear_status(&temp_server, SERVER_JOINED);

    connect_result_t rval = mon_connect_to_db(mon, database);
    if (rval != MONITOR_CONN_OK)
    {
        if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR)
        {
            server_set_status(&temp_server, SERVER_AUTH_ERROR);
        }
        else
        {
            server_clear_status(&temp_server, SERVER_AUTH_ERROR);
        }

        database->server->node_id = -1;

        if (mon_status_changed(database) && mon_print_fail_status(database))
        {
            mon_log_connect_error(database, rval);
        }

        server_transfer_status(database->server, &temp_server);

        return;
    }

    /* If we get this far then we have a working connection */
    server_set_status(&temp_server, SERVER_RUNNING);

    /* get server version string */
    server_string = (char *) mysql_get_server_info(database->con);
    if (server_string)
    {
        server_set_version_string(database->server, server_string);
    }

    /* Check if the the Galera FSM shows this node is joined to the cluster */
    if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state'") == 0
        && (result = mysql_store_result(database->con)) != NULL)
    {
        if (mysql_field_count(database->con) < 2)
        {
            mysql_free_result(result);
            MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_state'\". "
                      "Expected 2 columns."
                      " MySQL Version: %s", version_str);
            return;
        }

        while ((row = mysql_fetch_row(result)))
        {
            if (strcmp(row[1], "4") == 0)
                isjoined = 1;

                /* Check if the node is a donor and is using xtrabackup, in this case it can stay alive */
            else if (strcmp(row[1], "2") == 0 && handle->availableWhenDonor == 1)
            {
                if (mysql_query(database->con, "SHOW VARIABLES LIKE 'wsrep_sst_method'") == 0
                    && (result2 = mysql_store_result(database->con)) != NULL)
                {
                    if (mysql_field_count(database->con) < 2)
                    {
                        mysql_free_result(result);
                        mysql_free_result(result2);
                        MXS_ERROR("Unexpected result for \"SHOW VARIABLES LIKE "
                                  "'wsrep_sst_method'\". Expected 2 columns."
                                  " MySQL Version: %s", version_str);
                        return;
                    }
                    while ((row = mysql_fetch_row(result2)))
                    {
                        if (strncmp(row[1], "xtrabackup", 10) == 0)
                            isjoined = 1;
                    }
                    mysql_free_result(result2);
                }
            }
        }
        mysql_free_result(result);
    }

    /* Check the the Galera node index in the cluster */
    if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0
        && (result = mysql_store_result(database->con)) != NULL)
    {
        long local_index = -1;

        if (mysql_field_count(database->con) < 2)
        {
            mysql_free_result(result);
            MXS_ERROR("Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_index'\". "
                      "Expected 2 columns."
                      " MySQL Version: %s", version_str);
            return;
        }

        while ((row = mysql_fetch_row(result)))
        {
            local_index = strtol(row[1], NULL, 10);
            if ((errno == ERANGE && (local_index == LONG_MAX
                                     || local_index == LONG_MIN)) || (errno != 0 && local_index == 0))
            {
                local_index = -1;
            }
            database->server->node_id = local_index;
        }
        mysql_free_result(result);
    }

    if (isjoined)
    {
        server_set_status(&temp_server, SERVER_JOINED);
    }
    else
    {
        server_clear_status(&temp_server, SERVER_JOINED);
    }

    /* clear bits for non member nodes */
    if (!SERVER_IN_MAINT(database->server) && (!SERVER_IS_JOINED(&temp_server)))
    {
        database->server->depth = -1;

        /* clear M/S status */
        server_clear_status(&temp_server, SERVER_SLAVE);
        server_clear_status(&temp_server, SERVER_MASTER);

        /* clear master sticky status */
        server_clear_status(&temp_server, SERVER_MASTER_STICKINESS);
    }

    server_transfer_status(database->server, &temp_server);
}
Exemplo n.º 9
0
/**
 * Associate a new session with this instance of the router.
 *
 * @param instance	The router instance data
 * @param session	The session itself
 * @return Session specific data for this session
 */
static	void	*
newSession(ROUTER *instance, SESSION *session)
{
ROUTER_INSTANCE	        *inst = (ROUTER_INSTANCE *)instance;
ROUTER_CLIENT_SES       *client_rses;
BACKEND                 *candidate = NULL;
int                     i;
BACKEND *master_host = NULL;

        LOGIF(LD, (skygw_log_write_flush(
                LOGFILE_DEBUG,
                "%lu [newSession] new router session with session "
                "%p, and inst %p.",
                pthread_self(),
                session,
                inst)));


	client_rses = (ROUTER_CLIENT_SES *)calloc(1, sizeof(ROUTER_CLIENT_SES));

        if (client_rses == NULL) {
                return NULL;
	}

#if defined(SS_DEBUG)
        client_rses->rses_chk_top = CHK_NUM_ROUTER_SES;
        client_rses->rses_chk_tail = CHK_NUM_ROUTER_SES;
#endif

	/**
         * Find the Master host from available servers
	 */
        master_host = get_root_master(inst->servers);

	/**
	 * Find a backend server to connect to. This is the extent of the
	 * load balancing algorithm we need to implement for this simple
	 * connection router.
	 */

	/*
	 * Loop over all the servers and find any that have fewer connections
         * than the candidate server.
	 *
	 * If a server has less connections than the current candidate we mark this
	 * as the new candidate to connect to.
	 *
	 * If a server has the same number of connections currently as the candidate
	 * and has had less connections over time than the candidate it will also
	 * become the new candidate. This has the effect of spreading the
         * connections over different servers during periods of very low load.
	 */
	for (i = 0; inst->servers[i]; i++) {
		if(inst->servers[i]) {
			LOGIF(LD, (skygw_log_write(
				LOGFILE_DEBUG,
				"%lu [newSession] Examine server in port %d with "
                                "%d connections. Status is %s, "
				"inst->bitvalue is %d",
                                pthread_self(),
				inst->servers[i]->server->port,
				inst->servers[i]->current_connection_count,
				STRSRVSTATUS(inst->servers[i]->server),
				inst->bitmask)));
		}

		if (SERVER_IN_MAINT(inst->servers[i]->server))
			continue;

		if (inst->servers[i]->weight == 0)
			continue;

		/* Check server status bits against bitvalue from router_options */
		if (inst->servers[i] &&
			SERVER_IS_RUNNING(inst->servers[i]->server) &&
			(inst->servers[i]->server->status & inst->bitmask & inst->bitvalue))
                {
			if (master_host) {
				if (inst->servers[i] == master_host && (inst->bitvalue & SERVER_SLAVE)) {
					/* skip root Master here, as it could also be slave of an external server
					 * that is not in the configuration.
					 * Intermediate masters (Relay Servers) are also slave and will be selected
					 * as Slave(s)
			 		 */

					continue;
				}
				if (inst->servers[i] == master_host && (inst->bitvalue & SERVER_MASTER)) {
					/* If option is "master" return only the root Master as there
					 * could be intermediate masters (Relay Servers)
					 * and they must not be selected.
			 	 	 */

					candidate = master_host;
					break;
				}
			} else {
					/* master_host is NULL, no master server.
					 * If requested router_option is 'master'
					 * candidate wll be NULL.
					 */
					if (inst->bitvalue & SERVER_MASTER) {
                                                candidate = NULL;
                                                break;
					}
			}

			/* If no candidate set, set first running server as
			our initial candidate server */
			if (candidate == NULL)
                        {
				candidate = inst->servers[i];
			}
                        else if ((inst->servers[i]->current_connection_count 
					* 1000) / inst->servers[i]->weight <
                                   (candidate->current_connection_count *
					1000) / candidate->weight)
                        {
				/* This running server has fewer
				connections, set it as a new candidate */
				candidate = inst->servers[i];
			}
                        else if ((inst->servers[i]->current_connection_count 
					* 1000) / inst->servers[i]->weight ==
                                   (candidate->current_connection_count *
					1000) / candidate->weight &&
                                 inst->servers[i]->server->stats.n_connections <
                                 candidate->server->stats.n_connections)
                        {
				/* This running server has the same number
				of connections currently as the candidate
				but has had fewer connections over time
				than candidate, set this server to candidate*/
				candidate = inst->servers[i];
			}
		}
	}

	/* There is no candidate server here!
	 * With router_option=slave a master_host could be set, so route traffic there.
	 * Otherwise, just clean up and return NULL
	 */
	if (!candidate) {
		if (master_host) {
			candidate = master_host;
		} else {
                	LOGIF(LE, (skygw_log_write_flush(
                      	  LOGFILE_ERROR,
                      	  "Error : Failed to create new routing session. "
                      	  "Couldn't find eligible candidate server. Freeing "
                       	 "allocated resources.")));
			free(client_rses);
			return NULL;
		}
	}

	client_rses->rses_capabilities = RCAP_TYPE_PACKET_INPUT;
        
	/*
	 * We now have the server with the least connections.
	 * Bump the connection count for this server
	 */
	atomic_add(&candidate->current_connection_count, 1);
	client_rses->backend = candidate;
        LOGIF(LD, (skygw_log_write(
                LOGFILE_DEBUG,
                "%lu [newSession] Selected server in port %d. "
                "Connections : %d\n",
                pthread_self(),
                candidate->server->port,
                candidate->current_connection_count)));

        /*
	 * Open a backend connection, putting the DCB for this
	 * connection in the client_rses->backend_dcb
	 */
        client_rses->backend_dcb = dcb_connect(candidate->server,
                                      session,
                                      candidate->server->protocol);
        if (client_rses->backend_dcb == NULL)
	{
                atomic_add(&candidate->current_connection_count, -1);
		free(client_rses);
		return NULL;
	}
        dcb_add_callback(
                         client_rses->backend_dcb,
                         DCB_REASON_NOT_RESPONDING,
                         &handle_state_switch,
                         client_rses);
        inst->stats.n_sessions++;

	/**
         * Add this session to the list of active sessions.
         */
	spinlock_acquire(&inst->lock);
	client_rses->next = inst->connections;
	inst->connections = client_rses;
	spinlock_release(&inst->lock);

        CHK_CLIENT_RSES(client_rses);

	skygw_log_write(
                LOGFILE_TRACE,
		 "Readconnroute: New session for server %s. "
                "Connections : %d",
		 candidate->server->unique_name,
		 candidate->current_connection_count);
	return (void *)client_rses;
}
Exemplo n.º 10
0
/**
 * The entry point for the monitoring module thread
 *
 * @param arg	The handle of the monitor
 */
static void
monitorMain(void *arg)
{
    MONITOR* mon = (MONITOR*)arg;
MM_MONITOR	*handle;
MONITOR_SERVERS	*ptr;
int detect_stale_master;
MONITOR_SERVERS *root_master;
size_t nrounds = 0;

spinlock_acquire(&mon->lock);
handle = (MM_MONITOR *)mon->handle;
spinlock_release(&mon->lock);
detect_stale_master = handle->detectStaleMaster;

	if (mysql_thread_init())
	{
		LOGIF(LE, (skygw_log_write_flush(
                                   LOGFILE_ERROR,
                                   "Fatal : mysql_thread_init failed in monitor "
                                   "module. Exiting.\n")));
		return;
	}                         

	handle->status = MONITOR_RUNNING;
	while (1)
	{
		if (handle->shutdown)
		{
			handle->status = MONITOR_STOPPING;
			mysql_thread_end();
			handle->status = MONITOR_STOPPED;
			return;
		}

		/** Wait base interval */
		thread_millisleep(MON_BASE_INTERVAL_MS);
		/**
		 * Calculate how far away the monitor interval is from its full
		 * cycle and if monitor interval time further than the base
		 * interval, then skip monitoring checks. Excluding the first
		 * round.
		 */
                if (nrounds != 0 &&
                        ((nrounds*MON_BASE_INTERVAL_MS)%mon->interval) >=
                        MON_BASE_INTERVAL_MS)
                {
                        nrounds += 1;
                        continue;
                }
                nrounds += 1;

		/* start from the first server in the list */
		ptr = mon->databases;

		while (ptr)
		{
			/* copy server status into monitor pending_status */
			ptr->pending_status = ptr->server->status;

			/* monitor current node */
			monitorDatabase(mon, ptr);

                        if (mon_status_changed(ptr))
                        {
                                dcb_call_foreach(ptr->server,DCB_REASON_NOT_RESPONDING);
                        }
                        
                        if (mon_status_changed(ptr) || 
                                mon_print_fail_status(ptr))
                        {
                                LOGIF(LD, (skygw_log_write_flush(
                                        LOGFILE_DEBUG,
                                        "Backend server %s:%d state : %s",
                                        ptr->server->name,
                                        ptr->server->port,
                                        STRSRVSTATUS(ptr->server))));                                
                        }
                        if (SERVER_IS_DOWN(ptr->server))
                        {
                                /** Increase this server'e error count */
                                ptr->mon_err_count += 1;                                
                        }
                        else
                        {
                                /** Reset this server's error count */
                                ptr->mon_err_count = 0;
                        }

			ptr = ptr->next;
		}
	
		/* Get Master server pointer */
		root_master = get_current_master(mon);

		/* Update server status from monitor pending status on that server*/

                ptr = mon->databases;
		while (ptr)
		{
			if (! SERVER_IN_MAINT(ptr->server)) {
				/* If "detect_stale_master" option is On, let's use the previus master */
				if (detect_stale_master && root_master && (!strcmp(ptr->server->name, root_master->server->name) && ptr->server->port == root_master->server->port) && (ptr->server->status & SERVER_MASTER) && !(ptr->pending_status & SERVER_MASTER)) {
					/* in this case server->status will not be updated from pending_status */
					LOGIF(LM, (skygw_log_write_flush(
						LOGFILE_MESSAGE, "[mysql_mon]: root server [%s:%i] is no longer Master, let's use it again even if it could be a stale master, you have been warned!", ptr->server->name, ptr->server->port)));
					/* Set the STALE bit for this server in server struct */
					server_set_status(ptr->server, SERVER_STALE_STATUS);
				} else {
					ptr->server->status = ptr->pending_status;
				}
			}
			ptr = ptr->next;
		}
		
		ptr = mon->databases;
		monitor_event_t evtype;
		while(ptr)
		{
		    if(mon_status_changed(ptr))
		    {
			evtype = mon_get_event_type(ptr);
			if(isMySQLEvent(evtype))
			{
			    skygw_log_write(LOGFILE_TRACE,"Server changed state: %s[%s:%u]: %s",
				     ptr->server->unique_name,
				     ptr->server->name,ptr->server->port,
				     mon_get_event_name(ptr));
			    if(handle->script && handle->events[evtype])
			    {
				monitor_launch_script(mon,ptr,handle->script);
			    }
			}
		    }
		    ptr = ptr->next;
		}
	}
}
Exemplo n.º 11
0
/**
 * Monitor an individual server
 *
 * @param handle        The MySQL Monitor object
 * @param database	The database to probe
 */
static void
monitorDatabase(MONITOR* mon, MONITOR_SERVERS *database)
{
    MM_MONITOR *handle = mon->handle;
MYSQL_ROW	  row;
MYSQL_RES	  *result;
int               isslave = 0;
int               ismaster = 0;
char		  *uname  = mon->user;
char              *passwd = mon->password;
unsigned long int server_version = 0;
char 		  *server_string;

        if (database->server->monuser != NULL)
	{
		uname = database->server->monuser;
		passwd = database->server->monpw;
	}
	
	if (uname == NULL)
		return;
        
	/* Don't probe servers in maintenance mode */
	if (SERVER_IN_MAINT(database->server))
		return;

        /** Store previous status */
        database->mon_prev_status = database->server->status;
        
	if (database->con == NULL || mysql_ping(database->con) != 0)
	{
		char *dpwd = decryptPassword(passwd);
                int  read_timeout = 1;
		if(database->con)
		    mysql_close(database->con);
                database->con = mysql_init(NULL);

                mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
                
		if (mysql_real_connect(database->con,
                                       database->server->name,
                                       uname,
                                       dpwd,
                                       NULL,
                                       database->server->port,
                                       NULL,
                                       0) == NULL)
		{
                        free(dpwd);
                        
                        if (mon_print_fail_status(database))
                        {
                                LOGIF(LE, (skygw_log_write_flush(
                                        LOGFILE_ERROR,
                                        "Error : Monitor was unable to connect to "
                                        "server %s:%d : \"%s\"",
                                        database->server->name,
                                        database->server->port,
                                        mysql_error(database->con))));
                        }

			/* The current server is not running
			 *
			 * Store server NOT running in server and monitor server pending struct
			 *
			 */
			if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR)
			{
				server_set_status(database->server, SERVER_AUTH_ERROR);
				monitor_set_pending_status(database, SERVER_AUTH_ERROR);
			}
			server_clear_status(database->server, SERVER_RUNNING);
			monitor_clear_pending_status(database, SERVER_RUNNING);

			/* Also clear M/S state in both server and monitor server pending struct */
			server_clear_status(database->server, SERVER_SLAVE);
			server_clear_status(database->server, SERVER_MASTER);
			monitor_clear_pending_status(database, SERVER_SLAVE);
			monitor_clear_pending_status(database, SERVER_MASTER);

			/* Clean addition status too */
			server_clear_status(database->server, SERVER_STALE_STATUS);
			monitor_clear_pending_status(database, SERVER_STALE_STATUS);

			return;
		}  else {
                        server_clear_status(database->server, SERVER_AUTH_ERROR);
                        monitor_clear_pending_status(database, SERVER_AUTH_ERROR);
                }
		free(dpwd);
	}
        /* Store current status in both server and monitor server pending struct */
	server_set_status(database->server, SERVER_RUNNING);
	monitor_set_pending_status(database, SERVER_RUNNING);

	/* get server version from current server */
	server_version = mysql_get_server_version(database->con);

	/* get server version string */
	server_string = (char *)mysql_get_server_info(database->con);
	if (server_string) {
		database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1);
		if (database->server->server_string)
			strcpy(database->server->server_string, server_string);
	}

        /* get server_id form current node */
        if (mysql_query(database->con, "SELECT @@server_id") == 0
                && (result = mysql_store_result(database->con)) != NULL)
        {
                long server_id = -1;

		if(mysql_field_count(database->con) != 1)
		{
		    mysql_free_result(result);
		    skygw_log_write(LE,"Error: Unexpected result for 'SELECT @@server_id'. Expected 1 column."
				    " MySQL Version: %s",version_str);
		    return;
		}

                while ((row = mysql_fetch_row(result)))
                {
                        server_id = strtol(row[0], NULL, 10);
                        if ((errno == ERANGE && (server_id == LONG_MAX
                                || server_id == LONG_MIN)) || (errno != 0 && server_id == 0))
                        {
                                server_id = -1;
                        }
                        database->server->node_id = server_id;
                }
                mysql_free_result(result);
        }

	/* Check if the Slave_SQL_Running and Slave_IO_Running status is
	 * set to Yes
	 */

	/* Check first for MariaDB 10.x.x and get status for multimaster replication */
	if (server_version >= 100000) {

		if (mysql_query(database->con, "SHOW ALL SLAVES STATUS") == 0
			&& (result = mysql_store_result(database->con)) != NULL)
		{
			int i = 0;
			long master_id = -1;

			if(mysql_field_count(database->con) < 42)
			{
			    mysql_free_result(result);
			    skygw_log_write(LE,"Error: \"SHOW ALL SLAVES STATUS\" "
				    "returned less than the expected amount of columns. Expected 42 columns"
				    " MySQL Version: %s",version_str);
			    return;
			}

			while ((row = mysql_fetch_row(result)))
			{
				/* get Slave_IO_Running and Slave_SQL_Running values*/
				if (strncmp(row[12], "Yes", 3) == 0
						&& strncmp(row[13], "Yes", 3) == 0) {
					isslave += 1;
				}

				/* If Slave_IO_Running = Yes, assign the master_id to current server: this allows building 
				 * the replication tree, slaves ids will be added to master(s) and we will have at least the 
				 * root master server.
				 * Please note, there could be no slaves at all if Slave_SQL_Running == 'No'
				 */
				if (strncmp(row[12], "Yes", 3) == 0) {
					/* get Master_Server_Id values */
                                        master_id = atol(row[41]);
                                        if (master_id == 0)
                                                master_id = -1;
				}

				i++;
			}
			/* store master_id of current node */
			memcpy(&database->server->master_id, &master_id, sizeof(long));

			mysql_free_result(result);

			/* If all configured slaves are running set this node as slave */
			if (isslave > 0 && isslave == i)
				isslave = 1;
			else
				isslave = 0;
		}
	} else {	
		if (mysql_query(database->con, "SHOW SLAVE STATUS") == 0
			&& (result = mysql_store_result(database->con)) != NULL)
		{
			long master_id = -1;

			if(mysql_field_count(database->con) < 40)
			{
			    mysql_free_result(result);

			    if(server_version < 5*10000 + 5*100)
			    {
				if(database->log_version_err)
				{
				    skygw_log_write(LE,"Error: \"SHOW SLAVE STATUS\" "
					    " for versions less than 5.5 does not have master_server_id, "
					    "replication tree cannot be resolved for server %s."
					    " MySQL Version: %s",database->server->unique_name,version_str);
				    database->log_version_err = false;
				}
			    }
			    else
			    {
				skygw_log_write(LE,"Error: \"SHOW SLAVE STATUS\" "
					"returned less than the expected amount of columns. Expected 40 columns."
					" MySQL Version: %s",version_str);
			    }
			    return;
			}

			while ((row = mysql_fetch_row(result)))
			{
				/* get Slave_IO_Running and Slave_SQL_Running values*/
				if (strncmp(row[10], "Yes", 3) == 0
						&& strncmp(row[11], "Yes", 3) == 0) {
					isslave = 1;
				}

				/* If Slave_IO_Running = Yes, assign the master_id to current server: this allows building 
				 * the replication tree, slaves ids will be added to master(s) and we will have at least the 
				 * root master server.
				 * Please note, there could be no slaves at all if Slave_SQL_Running == 'No'
				 */
				if (strncmp(row[10], "Yes", 3) == 0) {
					/* get Master_Server_Id values */
					master_id = atol(row[39]);
					if (master_id == 0)
						master_id = -1;
				}
			}
			/* store master_id of current node */
			memcpy(&database->server->master_id, &master_id, sizeof(long));

			mysql_free_result(result);
		}
	}

	/* get variable 'read_only' set by an external component */
	if (mysql_query(database->con, "SHOW GLOBAL VARIABLES LIKE 'read_only'") == 0
		&& (result = mysql_store_result(database->con)) != NULL)
	{
		if(mysql_field_count(database->con) < 2)
		{
		    mysql_free_result(result);
		    skygw_log_write(LE,"Error: Unexpected result for \"SHOW GLOBAL VARIABLES LIKE 'read_only'\". Expected 2 columns."
				    " MySQL Version: %s",version_str);
		    return;
		}

		while ((row = mysql_fetch_row(result)))
		{
			if (strncasecmp(row[1], "OFF", 3) == 0) {
				ismaster = 1;
			} else {
				isslave = 1;
			}
		}
		mysql_free_result(result);
	}

	/* Remove addition info */
	monitor_clear_pending_status(database, SERVER_STALE_STATUS);

	/* Set the Slave Role */
	if (isslave)
	{
		monitor_set_pending_status(database, SERVER_SLAVE);
		/* Avoid any possible stale Master state */
		monitor_clear_pending_status(database, SERVER_MASTER);

		/* Set replication depth to 1 */
		database->server->depth = 1;
	} else {
		/* Avoid any possible Master/Slave stale state */
		monitor_clear_pending_status(database, SERVER_SLAVE);
		monitor_clear_pending_status(database, SERVER_MASTER);
	}

	/* Set the Master role */
        if (ismaster)
        {
		monitor_clear_pending_status(database, SERVER_SLAVE);
                monitor_set_pending_status(database, SERVER_MASTER);

		/* Set replication depth to 0 */
		database->server->depth = 0;
        }

}
Exemplo n.º 12
0
/**
 * The entry point for the monitoring module thread
 *
 * @param arg	The handle of the monitor
 */
static void
monitorMain(void *arg)
{
    MONITOR* mon = (MONITOR*)arg;
GALERA_MONITOR		*handle;
MONITOR_SERVERS		*ptr;
size_t			nrounds = 0;
MONITOR_SERVERS		*candidate_master = NULL;
int			master_stickiness;
int			is_cluster=0;
int			log_no_members = 1;
monitor_event_t evtype;

    spinlock_acquire(&mon->lock);
    handle = (GALERA_MONITOR *)mon->handle;
    spinlock_release(&mon->lock);
    master_stickiness = handle->disableMasterFailback;
	if (mysql_thread_init())
	{
                LOGIF(LE, (skygw_log_write_flush(
                                   LOGFILE_ERROR,
                                   "Fatal : mysql_thread_init failed in monitor "
                                   "module. Exiting.\n")));
                return;
	}                         
	handle->status = MONITOR_RUNNING;
	
	while (1)
	{
		if (handle->shutdown)
		{
			handle->status = MONITOR_STOPPING;
			mysql_thread_end();
			handle->status = MONITOR_STOPPED;
			return;
		}
		/** Wait base interval */
		thread_millisleep(MON_BASE_INTERVAL_MS);
		/** 
		 * Calculate how far away the monitor interval is from its full 
		 * cycle and if monitor interval time further than the base 
		 * interval, then skip monitoring checks. Excluding the first
		 * round.
		 */ 

		if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%mon->interval) >= MON_BASE_INTERVAL_MS)
		{
			nrounds += 1;
			continue;
		}

		nrounds += 1;

		/* reset cluster members counter */
		is_cluster=0;

		ptr = mon->databases;

		while (ptr)
		{
			ptr->mon_prev_status = ptr->server->status;

			monitorDatabase(mon, ptr);

			/* clear bits for non member nodes */
			if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) {
				ptr->server->depth = -1;

				/* clear M/S status */
				server_clear_status(ptr->server, SERVER_SLAVE);
                		server_clear_status(ptr->server, SERVER_MASTER);
				
				/* clear master sticky status */
				server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
			}

			/* Log server status change */
			if (mon_status_changed(ptr))
			{
				LOGIF(LD, (skygw_log_write_flush(
					LOGFILE_DEBUG,
					"Backend server %s:%d state : %s",
					ptr->server->name,
					ptr->server->port,
					STRSRVSTATUS(ptr->server))));
			}
                        
                        if (!(SERVER_IS_RUNNING(ptr->server)) || 
                            !(SERVER_IS_IN_CLUSTER(ptr->server)))
                        {
					dcb_call_foreach(ptr->server,DCB_REASON_NOT_RESPONDING);
                        }

			if (SERVER_IS_DOWN(ptr->server))
			{
				/** Increase this server'e error count */
				dcb_call_foreach(ptr->server,DCB_REASON_NOT_RESPONDING);
				ptr->mon_err_count += 1;

			}
			else
			{
				/** Reset this server's error count */
				ptr->mon_err_count = 0;
			}

			ptr = ptr->next;
		}

		/*
		 * Let's select a master server:
		 * it could be the candidate master following MIN(node_id) rule or
		 * the server that was master in the previous monitor polling cycle
		 * Decision depends on master_stickiness value set in configuration
		 */

		/* get the candidate master, following MIN(node_id) rule */
		candidate_master = get_candidate_master(mon->databases);

		/* Select the master, based on master_stickiness */
                if (1 == handle->disableMasterRoleSetting) {
                    handle->master = NULL;
                }
                else {
                    handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness);
                }

		ptr = mon->databases;

		while (ptr) {
			if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) {
				ptr = ptr->next;
				continue;
			}

                        if (handle->master) {
                            if (ptr != handle->master) {
				/* set the Slave role */
				server_set_status(ptr->server, SERVER_SLAVE);
				server_clear_status(ptr->server, SERVER_MASTER);

				/* clear master stickiness */
				server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
                            } else {
				/* set the Master role */
				server_set_status(handle->master->server, SERVER_MASTER);
				server_clear_status(handle->master->server, SERVER_SLAVE);

				if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) {
					/* set master stickiness */
					server_set_status(handle->master->server, SERVER_MASTER_STICKINESS);
				} else {
					/* clear master stickiness */
					server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
				}			
                            }
                        }

			is_cluster++;

			ptr = ptr->next;
		}

		if (is_cluster == 0 && log_no_members) {
			LOGIF(LE, (skygw_log_write_flush(
					LOGFILE_ERROR,
					"Error: there are no cluster members")));
			log_no_members = 0;
		} else {
			if (is_cluster > 0 && log_no_members == 0) {
				LOGIF(LE, (skygw_log_write_flush(
					LOGFILE_ERROR,
					"Info: found cluster members")));
				log_no_members = 1;
			}
		}


		ptr = mon->databases;

		while(ptr)
		{

		    /** Execute monitor script if a server state has changed */
		    if(mon_status_changed(ptr))
		    {
			evtype = mon_get_event_type(ptr);
			if(isGaleraEvent(evtype))
			{
			    skygw_log_write(LOGFILE_TRACE,"Server changed state: %s[%s:%u]: %s",
				     ptr->server->unique_name,
				     ptr->server->name,ptr->server->port,
				     mon_get_event_name(ptr));
			    if(handle->script && handle->events[evtype])
			    {
				monitor_launch_script(mon,ptr,handle->script);
			    }
			}
		    }
		    ptr = ptr->next;
		}
	}
}
Exemplo n.º 13
0
/**
 * Monitor an individual server
 *
 * @param handle        The MySQL Monitor object
 * @param database      The database to probe
 */
static void
monitorDatabase(MONITOR *mon, MONITOR_SERVERS *database)
{
    GALERA_MONITOR* handle = (GALERA_MONITOR*)mon->handle;
MYSQL_ROW	row;
MYSQL_RES	*result,*result2;
int		isjoined = 0;
char		*uname  = mon->user;
char		*passwd = mon->password;
unsigned long int	server_version = 0;
char 			*server_string;

	if (database->server->monuser != NULL)
	{
		uname = database->server->monuser;
		passwd = database->server->monpw;
	}
	if (uname == NULL)
		return;

	/* Don't even probe server flagged as in maintenance */
	if (SERVER_IN_MAINT(database->server))
		return;

	/** Store previous status */
	database->mon_prev_status = database->server->status;

	if (database->con == NULL || mysql_ping(database->con) != 0)
	{
		char *dpwd = decryptPassword(passwd);
		int connect_timeout = mon->connect_timeout;
		int read_timeout = mon->read_timeout;
		int write_timeout = mon->write_timeout;

		if(database->con)
		    mysql_close(database->con);
		database->con = mysql_init(NULL);

		mysql_options(database->con, MYSQL_OPT_CONNECT_TIMEOUT, (void *)&connect_timeout);
		mysql_options(database->con, MYSQL_OPT_READ_TIMEOUT, (void *)&read_timeout);
		mysql_options(database->con, MYSQL_OPT_WRITE_TIMEOUT, (void *)&write_timeout);

		if (mysql_real_connect(database->con, database->server->name,
			uname, dpwd, NULL, database->server->port, NULL, 0) == NULL)
		{
			free(dpwd);

			server_clear_status(database->server, SERVER_RUNNING);

			/* Also clear Joined, M/S and Stickiness bits */
			server_clear_status(database->server, SERVER_JOINED);
			server_clear_status(database->server, SERVER_SLAVE);
			server_clear_status(database->server, SERVER_MASTER);
			server_clear_status(database->server, SERVER_MASTER_STICKINESS);

			if (mysql_errno(database->con) == ER_ACCESS_DENIED_ERROR)
			{
				server_set_status(database->server, SERVER_AUTH_ERROR);
			}

			database->server->node_id = -1;

			if (mon_status_changed(database) && mon_print_fail_status(database))
			{
				LOGIF(LE, (skygw_log_write_flush(
					LOGFILE_ERROR,
					"Error : Monitor was unable to connect to "
					"server %s:%d : \"%s\"",
					database->server->name,
					database->server->port,
					mysql_error(database->con))));
			}

			return;
		}
		else
		{
			server_clear_status(database->server, SERVER_AUTH_ERROR);
		}
		free(dpwd);
	}

	/* If we get this far then we have a working connection */
	server_set_status(database->server, SERVER_RUNNING);

	/* get server version string */
	server_string = (char *)mysql_get_server_info(database->con);
	if (server_string) {
		database->server->server_string = realloc(database->server->server_string, strlen(server_string)+1);
		if (database->server->server_string)
			strcpy(database->server->server_string, server_string);
	}	

	/* Check if the the Galera FSM shows this node is joined to the cluster */
	if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_state'") == 0
		&& (result = mysql_store_result(database->con)) != NULL)
	{
		if(mysql_field_count(database->con) < 2)
		{
		    mysql_free_result(result);
		    skygw_log_write(LE,"Error: Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_state'\". Expected 2 columns."
				    " MySQL Version: %s",version_str);
		    return;
		}

		while ((row = mysql_fetch_row(result)))
		{
			if (strcmp(row[1], "4") == 0) 
				isjoined = 1;
	
			/* Check if the node is a donor and is using xtrabackup, in this case it can stay alive */
			else if (strcmp(row[1], "2") == 0 && handle->availableWhenDonor == 1) {
				if (mysql_query(database->con, "SHOW VARIABLES LIKE 'wsrep_sst_method'") == 0
					&& (result2 = mysql_store_result(database->con)) != NULL)
				{
				    		if(mysql_field_count(database->con) < 2)
						{
						    mysql_free_result(result);
						    mysql_free_result(result2);
						    skygw_log_write(LE,"Error: Unexpected result for \"SHOW VARIABLES LIKE 'wsrep_sst_method'\". Expected 2 columns."
							    " MySQL Version: %s",version_str);
						    return;
						}
					while ((row = mysql_fetch_row(result)))
					{
						if (strncmp(row[1], "xtrabackup", 10) == 0)
							isjoined = 1;
					}
					mysql_free_result(result2);
				}
			}
		}
		mysql_free_result(result);
	}

	/* Check the the Galera node index in the cluster */
	if (mysql_query(database->con, "SHOW STATUS LIKE 'wsrep_local_index'") == 0
		&& (result = mysql_store_result(database->con)) != NULL)
	{
		long local_index = -1;

		if(mysql_field_count(database->con) < 2)
		{
		    mysql_free_result(result);
		    skygw_log_write(LE,"Error: Unexpected result for \"SHOW STATUS LIKE 'wsrep_local_index'\". Expected 2 columns."
							    " MySQL Version: %s",version_str);
		    return;
		}

		while ((row = mysql_fetch_row(result)))
		{
			local_index = strtol(row[1], NULL, 10);
			if ((errno == ERANGE && (local_index == LONG_MAX
				|| local_index == LONG_MIN)) || (errno != 0 && local_index == 0))
			{
				local_index = -1;
			}
			database->server->node_id = local_index;
		}
		mysql_free_result(result);
	}

	if (isjoined)
		server_set_status(database->server, SERVER_JOINED);
	else
		server_clear_status(database->server, SERVER_JOINED);
}