/**
* The entry point for the monitoring module thread
*
* @param arg The handle of the monitor
*/
static void
monitorMain(void *arg)
{
MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg;
MONITOR_SERVERS *ptr;
long master_id;
if (mysql_thread_init())
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Fatal : mysql_thread_init failed in monitor "
"module. Exiting.\n")));
return;
}
handle->status = MONITOR_RUNNING;
while (1)
{
master_id = -1;
if (handle->shutdown)
{
handle->status = MONITOR_STOPPING;
mysql_thread_end();
handle->status = MONITOR_STOPPED;
return;
}
ptr = handle->databases;
while (ptr)
{
unsigned int prev_status = ptr->server->status;
monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd);
if (ptr->server->status != prev_status ||
SERVER_IS_DOWN(ptr->server))
{
LOGIF(LM, (skygw_log_write_flush(
LOGFILE_MESSAGE,
"Backend server %s:%d state : %s",
ptr->server->name,
ptr->server->port,
STRSRVSTATUS(ptr->server))));
}
ptr = ptr->next;
}
thread_millisleep(handle->interval);
}
}
/**
* The housekeeper thread implementation.
*
* This function is responsible for executing the housekeeper tasks.
*
* The implementation of the callng of the task functions is such that
* the tasks are called without the tasklock spinlock being held. This
* allows manipulation of the housekeeper task list during execution of
* one of the tasks. The resutl is that upon completion of a task the
* search for tasks to run must restart from the start of the queue.
* It is vital that the task->nextdue tiem is updated before the task
* is run.
*
* @param data Unused, here to satisfy the thread system
*/
void
hkthread(void *data)
{
HKTASK *ptr;
time_t now;
void (*taskfn)(void *);
void *taskdata;
int i;
for (;;)
{
for (i = 0; i < 10; i++)
{
if (do_shutdown)
return;
thread_millisleep(100);
hkheartbeat++;
}
now = time(0);
spinlock_acquire(&tasklock);
ptr = tasks;
while (ptr)
{
if (ptr->nextdue <= now)
{
ptr->nextdue = now + ptr->frequency;
taskfn = ptr->task;
taskdata = ptr->data;
spinlock_release(&tasklock);
(*taskfn)(taskdata);
if (ptr->type == HK_ONESHOT)
hktask_remove(ptr->name);
spinlock_acquire(&tasklock);
ptr = tasks;
}
else
ptr = ptr->next;
}
spinlock_release(&tasklock);
}
}
/**
* The entry point for the monitoring module thread
*
* @param arg The handle of the monitor
*/
static void
monitorMain(void *arg)
{
MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg;
MONITOR_SERVERS *ptr;
long master_id;
size_t nrounds = 0;
if (mysql_thread_init())
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Fatal : mysql_thread_init failed in monitor "
"module. Exiting.\n")));
return;
}
handle->status = MONITOR_RUNNING;
while (1)
{
if (handle->shutdown)
{
handle->status = MONITOR_STOPPING;
mysql_thread_end();
handle->status = MONITOR_STOPPED;
return;
}
/** Wait base interval */
thread_millisleep(MON_BASE_INTERVAL_MS);
/**
* Calculate how far away the monitor interval is from its full
* cycle and if monitor interval time further than the base
* interval, then skip monitoring checks. Excluding the first
* round.
*/
if (nrounds != 0 &&
((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >=
MON_BASE_INTERVAL_MS)
{
nrounds += 1;
continue;
}
nrounds += 1;
master_id = -1;
ptr = handle->databases;
while (ptr)
{
unsigned int prev_status = ptr->server->status;
monitorDatabase(ptr, handle->defaultUser, handle->defaultPasswd,handle);
if (ptr->server->status != prev_status ||
SERVER_IS_DOWN(ptr->server))
{
LOGIF(LD, (skygw_log_write_flush(
LOGFILE_DEBUG,
"Backend server %s:%d state : %s",
ptr->server->name,
ptr->server->port,
STRSRVSTATUS(ptr->server))));
}
ptr = ptr->next;
}
}
}
/**
* The entry point for the monitoring module thread
*
* @param arg The handle of the monitor
*/
static void
monitorMain(void *arg)
{
MYSQL_MONITOR *handle = (MYSQL_MONITOR *)arg;
MONITOR_SERVERS *ptr;
size_t nrounds = 0;
MONITOR_SERVERS *candidate_master = NULL;
int master_stickiness = handle->disableMasterFailback;
int is_cluster=0;
int log_no_members = 1;
if (mysql_thread_init())
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Fatal : mysql_thread_init failed in monitor "
"module. Exiting.\n")));
return;
}
handle->status = MONITOR_RUNNING;
while (1)
{
if (handle->shutdown)
{
handle->status = MONITOR_STOPPING;
mysql_thread_end();
handle->status = MONITOR_STOPPED;
return;
}
/** Wait base interval */
thread_millisleep(MON_BASE_INTERVAL_MS);
/**
* Calculate how far away the monitor interval is from its full
* cycle and if monitor interval time further than the base
* interval, then skip monitoring checks. Excluding the first
* round.
*/
if (nrounds != 0 && ((nrounds*MON_BASE_INTERVAL_MS)%handle->interval) >= MON_BASE_INTERVAL_MS)
{
nrounds += 1;
continue;
}
nrounds += 1;
/* reset cluster members counter */
is_cluster=0;
ptr = handle->databases;
while (ptr)
{
monitorDatabase(handle, ptr);
/* clear bits for non member nodes */
if ( ! SERVER_IN_MAINT(ptr->server) && (ptr->server->node_id < 0 || ! SERVER_IS_JOINED(ptr->server))) {
ptr->server->depth = -1;
/* clear M/S status */
server_clear_status(ptr->server, SERVER_SLAVE);
server_clear_status(ptr->server, SERVER_MASTER);
/* clear master sticky status */
server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
}
/* Log server status change */
if (mon_status_changed(ptr))
{
LOGIF(LD, (skygw_log_write_flush(
LOGFILE_DEBUG,
"Backend server %s:%d state : %s",
ptr->server->name,
ptr->server->port,
STRSRVSTATUS(ptr->server))));
}
if (SERVER_IS_DOWN(ptr->server))
{
/** Increase this server'e error count */
ptr->mon_err_count += 1;
}
else
{
/** Reset this server's error count */
ptr->mon_err_count = 0;
}
ptr = ptr->next;
}
/*
* Let's select a master server:
* it could be the candidate master following MIN(node_id) rule or
* the server that was master in the previous monitor polling cycle
* Decision depends on master_stickiness value set in configuration
*/
/* get the candidate master, followinf MIN(node_id) rule */
candidate_master = get_candidate_master(handle->databases);
/* Select the master, based on master_stickiness */
handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness);
ptr = handle->databases;
while (ptr && handle->master) {
if (!SERVER_IS_JOINED(ptr->server) || SERVER_IN_MAINT(ptr->server)) {
ptr = ptr->next;
continue;
}
if (ptr != handle->master) {
/* set the Slave role */
server_set_status(ptr->server, SERVER_SLAVE);
server_clear_status(ptr->server, SERVER_MASTER);
/* clear master stickyness */
server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
} else {
/* set the Master role */
server_set_status(handle->master->server, SERVER_MASTER);
server_clear_status(handle->master->server, SERVER_SLAVE);
if (candidate_master && handle->master->server->node_id != candidate_master->server->node_id) {
/* set master stickyness */
server_set_status(handle->master->server, SERVER_MASTER_STICKINESS);
} else {
/* clear master stickyness */
server_clear_status(ptr->server, SERVER_MASTER_STICKINESS);
}
}
is_cluster++;
ptr = ptr->next;
}
if (is_cluster == 0 && log_no_members) {
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Error: there are no cluster members")));
log_no_members = 0;
} else {
if (is_cluster > 0 && log_no_members == 0) {
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Info: found cluster members")));
log_no_members = 1;
}
}
}
}
/**
* The entry point for the monitoring module thread
*
* @param arg The handle of the monitor
*/
static void
monitorMain(void *arg)
{
MONITOR* mon = (MONITOR*) arg;
GALERA_MONITOR *handle;
MONITOR_SERVERS *ptr;
size_t nrounds = 0;
MONITOR_SERVERS *candidate_master = NULL;
int master_stickiness;
int is_cluster = 0;
int log_no_members = 1;
monitor_event_t evtype;
spinlock_acquire(&mon->lock);
handle = (GALERA_MONITOR *) mon->handle;
spinlock_release(&mon->lock);
master_stickiness = handle->disableMasterFailback;
if (mysql_thread_init())
{
MXS_ERROR("mysql_thread_init failed in monitor module. Exiting.");
return;
}
handle->status = MONITOR_RUNNING;
while (1)
{
if (handle->shutdown)
{
handle->status = MONITOR_STOPPING;
mysql_thread_end();
handle->status = MONITOR_STOPPED;
return;
}
/** Wait base interval */
thread_millisleep(MON_BASE_INTERVAL_MS);
/**
* Calculate how far away the monitor interval is from its full
* cycle and if monitor interval time further than the base
* interval, then skip monitoring checks. Excluding the first
* round.
*/
if (nrounds != 0 && ((nrounds * MON_BASE_INTERVAL_MS) % mon->interval) >= MON_BASE_INTERVAL_MS)
{
nrounds += 1;
continue;
}
nrounds += 1;
/* reset cluster members counter */
is_cluster = 0;
ptr = mon->databases;
while (ptr)
{
ptr->mon_prev_status = ptr->server->status;
monitorDatabase(mon, ptr);
/* Log server status change */
if (mon_status_changed(ptr))
{
MXS_DEBUG("Backend server %s:%d state : %s",
ptr->server->name,
ptr->server->port,
STRSRVSTATUS(ptr->server));
}
if (!(SERVER_IS_RUNNING(ptr->server)) ||
!(SERVER_IS_IN_CLUSTER(ptr->server)))
{
dcb_hangup_foreach(ptr->server);
}
if (SERVER_IS_DOWN(ptr->server))
{
/** Increase this server'e error count */
dcb_hangup_foreach(ptr->server);
ptr->mon_err_count += 1;
}
else
{
/** Reset this server's error count */
ptr->mon_err_count = 0;
}
ptr = ptr->next;
}
/*
* Let's select a master server:
* it could be the candidate master following MIN(node_id) rule or
* the server that was master in the previous monitor polling cycle
* Decision depends on master_stickiness value set in configuration
*/
/* get the candidate master, following MIN(node_id) rule */
candidate_master = get_candidate_master(mon);
/* Select the master, based on master_stickiness */
if (1 == handle->disableMasterRoleSetting)
{
handle->master = NULL;
}
else
{
handle->master = set_cluster_master(handle->master, candidate_master, master_stickiness);
}
ptr = mon->databases;
while (ptr)
{
const int repl_bits = (SERVER_SLAVE | SERVER_MASTER | SERVER_MASTER_STICKINESS);
if (SERVER_IS_JOINED(ptr->server))
{
if (handle->master)
{
if (ptr != handle->master)
{
/* set the Slave role and clear master stickiness */
server_clear_set_status(ptr->server, repl_bits, SERVER_SLAVE);
}
else
{
if (candidate_master &&
handle->master->server->node_id != candidate_master->server->node_id)
{
/* set master role and master stickiness */
server_clear_set_status(ptr->server, repl_bits,
(SERVER_MASTER | SERVER_MASTER_STICKINESS));
}
else
{
/* set master role and clear master stickiness */
server_clear_set_status(ptr->server, repl_bits, SERVER_MASTER);
}
}
}
is_cluster++;
}
else
{
server_clear_set_status(ptr->server, repl_bits, 0);
}
ptr = ptr->next;
}
if (is_cluster == 0 && log_no_members)
{
MXS_ERROR("There are no cluster members");
log_no_members = 0;
}
else
{
if (is_cluster > 0 && log_no_members == 0)
{
MXS_NOTICE("Found cluster members");
log_no_members = 1;
}
}
ptr = mon->databases;
while (ptr)
{
/** Execute monitor script if a server state has changed */
if (mon_status_changed(ptr))
{
evtype = mon_get_event_type(ptr);
if (isGaleraEvent(evtype))
{
MXS_NOTICE("Server changed state: %s[%s:%u]: %s",
ptr->server->unique_name,
ptr->server->name, ptr->server->port,
mon_get_event_name(ptr));
if (handle->script && handle->events[evtype])
{
monitor_launch_script(mon, ptr, handle->script);
}
}
}
ptr = ptr->next;
}
}
}
/**
* The entry point for the monitoring module thread
*
* @param arg The handle of the monitor
*/
static void
monitorMain(void *arg)
{
MONITOR* mon = (MONITOR*)arg;
MM_MONITOR *handle;
MONITOR_SERVERS *ptr;
int detect_stale_master;
MONITOR_SERVERS *root_master;
size_t nrounds = 0;
spinlock_acquire(&mon->lock);
handle = (MM_MONITOR *)mon->handle;
spinlock_release(&mon->lock);
detect_stale_master = handle->detectStaleMaster;
if (mysql_thread_init())
{
LOGIF(LE, (skygw_log_write_flush(
LOGFILE_ERROR,
"Fatal : mysql_thread_init failed in monitor "
"module. Exiting.\n")));
return;
}
handle->status = MONITOR_RUNNING;
while (1)
{
if (handle->shutdown)
{
handle->status = MONITOR_STOPPING;
mysql_thread_end();
handle->status = MONITOR_STOPPED;
return;
}
/** Wait base interval */
thread_millisleep(MON_BASE_INTERVAL_MS);
/**
* Calculate how far away the monitor interval is from its full
* cycle and if monitor interval time further than the base
* interval, then skip monitoring checks. Excluding the first
* round.
*/
if (nrounds != 0 &&
((nrounds*MON_BASE_INTERVAL_MS)%mon->interval) >=
MON_BASE_INTERVAL_MS)
{
nrounds += 1;
continue;
}
nrounds += 1;
/* start from the first server in the list */
ptr = mon->databases;
while (ptr)
{
/* copy server status into monitor pending_status */
ptr->pending_status = ptr->server->status;
/* monitor current node */
monitorDatabase(mon, ptr);
if (mon_status_changed(ptr))
{
dcb_call_foreach(ptr->server,DCB_REASON_NOT_RESPONDING);
}
if (mon_status_changed(ptr) ||
mon_print_fail_status(ptr))
{
LOGIF(LD, (skygw_log_write_flush(
LOGFILE_DEBUG,
"Backend server %s:%d state : %s",
ptr->server->name,
ptr->server->port,
STRSRVSTATUS(ptr->server))));
}
if (SERVER_IS_DOWN(ptr->server))
{
/** Increase this server'e error count */
ptr->mon_err_count += 1;
}
else
{
/** Reset this server's error count */
ptr->mon_err_count = 0;
}
ptr = ptr->next;
}
/* Get Master server pointer */
root_master = get_current_master(mon);
/* Update server status from monitor pending status on that server*/
ptr = mon->databases;
while (ptr)
{
if (! SERVER_IN_MAINT(ptr->server)) {
/* If "detect_stale_master" option is On, let's use the previus master */
if (detect_stale_master && root_master && (!strcmp(ptr->server->name, root_master->server->name) && ptr->server->port == root_master->server->port) && (ptr->server->status & SERVER_MASTER) && !(ptr->pending_status & SERVER_MASTER)) {
/* in this case server->status will not be updated from pending_status */
LOGIF(LM, (skygw_log_write_flush(
LOGFILE_MESSAGE, "[mysql_mon]: root server [%s:%i] is no longer Master, let's use it again even if it could be a stale master, you have been warned!", ptr->server->name, ptr->server->port)));
/* Set the STALE bit for this server in server struct */
server_set_status(ptr->server, SERVER_STALE_STATUS);
} else {
ptr->server->status = ptr->pending_status;
}
}
ptr = ptr->next;
}
ptr = mon->databases;
monitor_event_t evtype;
while(ptr)
{
if(mon_status_changed(ptr))
{
evtype = mon_get_event_type(ptr);
if(isMySQLEvent(evtype))
{
skygw_log_write(LOGFILE_TRACE,"Server changed state: %s[%s:%u]: %s",
ptr->server->unique_name,
ptr->server->name,ptr->server->port,
mon_get_event_name(ptr));
if(handle->script && handle->events[evtype])
{
monitor_launch_script(mon,ptr,handle->script);
}
}
}
ptr = ptr->next;
}
}
}