gboolean
crm_timer_popped(gpointer data)
{
fsa_timer_t *timer = (fsa_timer_t *) data;
if (timer == wait_timer
|| timer == recheck_timer
|| timer == transition_timer || timer == finalization_timer || timer == election_trigger) {
crm_info("%s (%s) just popped (%dms)",
get_timer_desc(timer), fsa_input2string(timer->fsa_input), timer->period_ms);
timer->counter++;
} else {
crm_err("%s (%s) just popped in state %s! (%dms)",
get_timer_desc(timer), fsa_input2string(timer->fsa_input),
fsa_state2string(fsa_state), timer->period_ms);
}
if (timer == election_trigger && election_trigger->counter > 5) {
crm_notice("We appear to be in an election loop, something may be wrong");
crm_write_blackbox(0, NULL);
election_trigger->counter = 0;
}
if (timer->repeat == FALSE) {
crm_timer_stop(timer); /* make it _not_ go off again */
}
if (timer->fsa_input == I_INTEGRATED) {
crm_info("Welcomed: %d, Integrated: %d",
crmd_join_phase_count(crm_join_welcomed),
crmd_join_phase_count(crm_join_integrated));
if (crmd_join_phase_count(crm_join_welcomed) == 0) {
/* If we don't even have ourself, start again */
register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL, __FUNCTION__);
} else {
register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL);
}
} else if (timer == recheck_timer && fsa_state != S_IDLE) {
crm_debug("Discarding %s event in state: %s",
fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
} else if (timer == finalization_timer && fsa_state != S_FINALIZE_JOIN) {
crm_debug("Discarding %s event in state: %s",
fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
} else if (timer->fsa_input != I_NULL) {
register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL);
}
crm_trace("Triggering FSA: %s", __FUNCTION__);
mainloop_set_trigger(fsa_source);
return TRUE;
}
gboolean
crm_timer_popped(gpointer data)
{
fsa_timer_t *timer = (fsa_timer_t *) data;
if (timer == wait_timer
|| timer == recheck_timer
|| timer == transition_timer || timer == finalization_timer || timer == election_trigger) {
crm_info("%s (%s) just popped (%dms)",
get_timer_desc(timer), fsa_input2string(timer->fsa_input), timer->period_ms);
} else {
crm_err("%s (%s) just popped in state %s! (%dms)",
get_timer_desc(timer), fsa_input2string(timer->fsa_input),
fsa_state2string(fsa_state), timer->period_ms);
}
if (timer->repeat == FALSE) {
crm_timer_stop(timer); /* make it _not_ go off again */
}
if (timer->fsa_input == I_INTEGRATED) {
crm_info("Welcomed: %d, Integrated: %d",
g_hash_table_size(welcomed_nodes), g_hash_table_size(integrated_nodes));
if (g_hash_table_size(welcomed_nodes) == 0) {
/* If we don't even have ourself, start again */
register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION, NULL, NULL, __FUNCTION__);
} else {
register_fsa_input_before(C_TIMER_POPPED, timer->fsa_input, NULL);
}
} else if (timer == recheck_timer && fsa_state != S_IDLE) {
crm_debug("Discarding %s event in state: %s",
fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
} else if (timer == finalization_timer && fsa_state != S_FINALIZE_JOIN) {
crm_debug("Discarding %s event in state: %s",
fsa_input2string(timer->fsa_input), fsa_state2string(fsa_state));
} else if (timer->fsa_input != I_NULL) {
register_fsa_input(C_TIMER_POPPED, timer->fsa_input, NULL);
}
crm_trace("Triggering FSA: %s", __FUNCTION__);
mainloop_set_trigger(fsa_source);
return TRUE;
}
void
fsa_dump_queue(int log_level)
{
if(log_level < (int)crm_log_level) {
return;
}
slist_iter(
data, fsa_data_t, fsa_message_queue, lpc,
do_crm_log(log_level,
"queue[%d(%d)]: input %s raised by %s()\t(cause=%s)",
lpc, data->id, fsa_input2string(data->fsa_input),
data->origin, fsa_cause2string(data->fsa_cause));
);
gboolean
crm_timer_stop(fsa_timer_t * timer)
{
const char *timer_desc = get_timer_desc(timer);
if (timer == NULL) {
crm_err("Attempted to stop NULL timer");
return FALSE;
} else if (timer->source_id != 0) {
crm_trace("Stopping %s (%s:%dms), src=%d",
timer_desc, fsa_input2string(timer->fsa_input),
timer->period_ms, timer->source_id);
g_source_remove(timer->source_id);
timer->source_id = 0;
} else {
crm_trace("%s (%s:%dms) already stopped",
timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
return FALSE;
}
return TRUE;
}
gboolean
crm_timer_start(fsa_timer_t * timer)
{
const char *timer_desc = get_timer_desc(timer);
if (timer->source_id == 0 && timer->period_ms > 0) {
timer->source_id = g_timeout_add(timer->period_ms, timer->callback, (void *)timer);
CRM_ASSERT(timer->source_id != 0);
crm_debug("Started %s (%s:%dms), src=%d",
timer_desc, fsa_input2string(timer->fsa_input),
timer->period_ms, timer->source_id);
} else if (timer->period_ms < 0) {
crm_err("Tried to start %s (%s:%dms) with a -ve period",
timer_desc, fsa_input2string(timer->fsa_input), timer->period_ms);
} else {
crm_debug("%s (%s:%dms) already running: src=%d",
timer_desc, fsa_input2string(timer->fsa_input),
timer->period_ms, timer->source_id);
return FALSE;
}
return TRUE;
}
void
fsa_dump_queue(int log_level)
{
int offset = 0;
GListPtr lpc = NULL;
for (lpc = fsa_message_queue; lpc != NULL; lpc = lpc->next) {
fsa_data_t *data = (fsa_data_t *) lpc->data;
do_crm_log_unlikely(log_level,
"queue[%d(%d)]: input %s raised by %s()\t(cause=%s)",
offset++, data->id, fsa_input2string(data->fsa_input),
data->origin, fsa_cause2string(data->fsa_cause));
}
}
/* A_LOG, A_WARN, A_ERROR */
void
do_log(long long action,
enum crmd_fsa_cause cause,
enum crmd_fsa_state cur_state, enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
unsigned log_type = LOG_TRACE;
if (action & A_LOG) {
log_type = LOG_INFO;
} else if (action & A_WARN) {
log_type = LOG_WARNING;
} else if (action & A_ERROR) {
log_type = LOG_ERR;
}
do_crm_log(log_type, "Input %s received in state %s from %s",
fsa_input2string(msg_data->fsa_input),
fsa_state2string(cur_state), msg_data->origin);
if (msg_data->data_type == fsa_dt_ha_msg) {
ha_msg_input_t *input = fsa_typed_data(msg_data->data_type);
crm_log_xml_debug(input->msg, __FUNCTION__);
} else if (msg_data->data_type == fsa_dt_xml) {
xmlNode *input = fsa_typed_data(msg_data->data_type);
crm_log_xml_debug(input, __FUNCTION__);
} else if (msg_data->data_type == fsa_dt_lrm) {
lrmd_event_data_t *input = fsa_typed_data(msg_data->data_type);
do_crm_log(log_type,
"Resource %s: Call ID %d returned %d (%d)."
" New status if rc=0: %s",
input->rsc_id, input->call_id, input->rc,
input->op_status, (char *)input->user_data);
}
}
int
register_fsa_input_adv(enum crmd_fsa_cause cause, enum crmd_fsa_input input,
void *data, long long with_actions,
gboolean prepend, const char *raised_from)
{
unsigned old_len = g_list_length(fsa_message_queue);
fsa_data_t *fsa_data = NULL;
CRM_CHECK(raised_from != NULL, raised_from = "<unknown>");
if (input == I_NULL && with_actions == A_NOTHING /* && data == NULL */ ) {
/* no point doing anything */
crm_err("Cannot add entry to queue: no input and no action");
return 0;
}
if (input == I_WAIT_FOR_EVENT) {
do_fsa_stall = TRUE;
crm_debug("Stalling the FSA pending further input: source=%s cause=%s data=%p queue=%d",
raised_from, fsa_cause2string(cause), data, old_len);
if (old_len > 0) {
fsa_dump_queue(LOG_TRACE);
prepend = FALSE;
}
if (data == NULL) {
fsa_actions |= with_actions;
fsa_dump_actions(with_actions, "Restored");
return 0;
}
/* Store everything in the new event and reset fsa_actions */
with_actions |= fsa_actions;
fsa_actions = A_NOTHING;
}
last_data_id++;
crm_trace("%s %s FSA input %d (%s) (cause=%s) %s data",
raised_from, prepend ? "prepended" : "appended", last_data_id,
fsa_input2string(input), fsa_cause2string(cause), data ? "with" : "without");
fsa_data = calloc(1, sizeof(fsa_data_t));
fsa_data->id = last_data_id;
fsa_data->fsa_input = input;
fsa_data->fsa_cause = cause;
fsa_data->origin = raised_from;
fsa_data->data = NULL;
fsa_data->data_type = fsa_dt_none;
fsa_data->actions = with_actions;
if (with_actions != A_NOTHING) {
crm_trace("Adding actions %.16llx to input", with_actions);
}
if (data != NULL) {
switch (cause) {
case C_FSA_INTERNAL:
case C_CRMD_STATUS_CALLBACK:
case C_IPC_MESSAGE:
case C_HA_MESSAGE:
crm_trace("Copying %s data from %s as a HA msg",
fsa_cause2string(cause), raised_from);
CRM_CHECK(((ha_msg_input_t *) data)->msg != NULL,
crm_err("Bogus data from %s", raised_from));
fsa_data->data = copy_ha_msg_input(data);
fsa_data->data_type = fsa_dt_ha_msg;
break;
case C_LRM_OP_CALLBACK:
crm_trace("Copying %s data from %s as lrmd_event_data_t",
fsa_cause2string(cause), raised_from);
fsa_data->data = lrmd_copy_event((lrmd_event_data_t *) data);
fsa_data->data_type = fsa_dt_lrm;
break;
case C_CCM_CALLBACK:
case C_SUBSYSTEM_CONNECT:
case C_LRM_MONITOR_CALLBACK:
case C_TIMER_POPPED:
case C_SHUTDOWN:
case C_HEARTBEAT_FAILED:
case C_HA_DISCONNECT:
case C_ILLEGAL:
case C_UNKNOWN:
case C_STARTUP:
crm_err("Copying %s data (from %s)"
" not yet implemented", fsa_cause2string(cause), raised_from);
crmd_exit(1);
break;
}
crm_trace("%s data copied", fsa_cause2string(fsa_data->fsa_cause));
}
/* make sure to free it properly later */
if (prepend) {
crm_trace("Prepending input");
fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data);
} else {
fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
}
crm_trace("Queue len: %d", g_list_length(fsa_message_queue));
/* fsa_dump_queue(LOG_DEBUG_2); */
if (old_len == g_list_length(fsa_message_queue)) {
crm_err("Couldnt add message to the queue");
}
if (fsa_source && input != I_WAIT_FOR_EVENT) {
crm_trace("Triggering FSA: %s", __FUNCTION__);
mainloop_set_trigger(fsa_source);
}
return last_data_id;
}
enum crmd_fsa_state
s_crmd_fsa(enum crmd_fsa_cause cause)
{
fsa_data_t *fsa_data = NULL;
long long register_copy = fsa_input_register;
long long new_actions = A_NOTHING;
enum crmd_fsa_state last_state;
crm_trace("FSA invoked with Cause: %s\tState: %s",
fsa_cause2string(cause), fsa_state2string(fsa_state));
fsa_dump_actions(fsa_actions, "Initial");
do_fsa_stall = FALSE;
if (is_message() == FALSE && fsa_actions != A_NOTHING) {
/* fake the first message so we can get into the loop */
fsa_data = calloc(1, sizeof(fsa_data_t));
fsa_data->fsa_input = I_NULL;
fsa_data->fsa_cause = C_FSA_INTERNAL;
fsa_data->origin = __FUNCTION__;
fsa_data->data_type = fsa_dt_none;
fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
fsa_data = NULL;
}
while (is_message() && do_fsa_stall == FALSE) {
crm_trace("Checking messages (%d remaining)", g_list_length(fsa_message_queue));
fsa_data = get_message();
if(fsa_data == NULL) {
continue;
}
log_fsa_input(fsa_data);
/* add any actions back to the queue */
fsa_actions |= fsa_data->actions;
fsa_dump_actions(fsa_data->actions, "Restored actions");
/* get the next batch of actions */
new_actions = crmd_fsa_actions[fsa_data->fsa_input][fsa_state];
fsa_actions |= new_actions;
fsa_dump_actions(new_actions, "New actions");
if (fsa_data->fsa_input != I_NULL && fsa_data->fsa_input != I_ROUTER) {
crm_debug("Processing %s: [ state=%s cause=%s origin=%s ]",
fsa_input2string(fsa_data->fsa_input),
fsa_state2string(fsa_state),
fsa_cause2string(fsa_data->fsa_cause), fsa_data->origin);
}
/* logging : *before* the state is changed */
if (is_set(fsa_actions, A_ERROR)) {
do_fsa_action(fsa_data, A_ERROR, do_log);
}
if (is_set(fsa_actions, A_WARN)) {
do_fsa_action(fsa_data, A_WARN, do_log);
}
if (is_set(fsa_actions, A_LOG)) {
do_fsa_action(fsa_data, A_LOG, do_log);
}
/* update state variables */
last_state = fsa_state;
fsa_state = crmd_fsa_state[fsa_data->fsa_input][fsa_state];
/*
* Remove certain actions during shutdown
*/
if (fsa_state == S_STOPPING || ((fsa_input_register & R_SHUTDOWN) == R_SHUTDOWN)) {
clear_bit(fsa_actions, startup_actions);
}
/*
* Hook for change of state.
* Allows actions to be added or removed when entering a state
*/
if (last_state != fsa_state) {
fsa_actions = do_state_transition(fsa_actions, last_state, fsa_state, fsa_data);
} else {
do_dot_log(DOT_PREFIX "\t// FSA input: State=%s \tCause=%s"
" \tInput=%s \tOrigin=%s() \tid=%d",
fsa_state2string(fsa_state),
fsa_cause2string(fsa_data->fsa_cause),
fsa_input2string(fsa_data->fsa_input), fsa_data->origin, fsa_data->id);
}
/* start doing things... */
s_crmd_fsa_actions(fsa_data);
delete_fsa_input(fsa_data);
fsa_data = NULL;
}
if (g_list_length(fsa_message_queue) > 0 || fsa_actions != A_NOTHING || do_fsa_stall) {
crm_debug("Exiting the FSA: queue=%d, fsa_actions=0x%llx, stalled=%s",
g_list_length(fsa_message_queue), fsa_actions, do_fsa_stall ? "true" : "false");
} else {
crm_trace("Exiting the FSA");
}
/* cleanup inputs? */
if (register_copy != fsa_input_register) {
long long same = register_copy & fsa_input_register;
fsa_dump_inputs(LOG_DEBUG, "Added", fsa_input_register ^ same);
fsa_dump_inputs(LOG_DEBUG, "Removed", register_copy ^ same);
}
fsa_dump_actions(fsa_actions, "Remaining");
fsa_dump_queue(LOG_DEBUG);
return fsa_state;
}
int
register_fsa_input_adv(
enum crmd_fsa_cause cause, enum crmd_fsa_input input,
void *data, long long with_actions,
gboolean prepend, const char *raised_from)
{
unsigned old_len = g_list_length(fsa_message_queue);
fsa_data_t *fsa_data = NULL;
last_data_id++;
CRM_CHECK(raised_from != NULL, raised_from = "<unknown>");
crm_debug("%s %s FSA input %d (%s) (cause=%s) %s data",
raised_from, prepend?"prepended":"appended",last_data_id, fsa_input2string(input),
fsa_cause2string(cause), data?"with":"without");
if(input == I_WAIT_FOR_EVENT) {
do_fsa_stall = TRUE;
crm_debug("Stalling the FSA pending further input: cause=%s",
fsa_cause2string(cause));
if(old_len > 0) {
crm_warn("%s stalled the FSA with pending inputs",
raised_from);
fsa_dump_queue(LOG_DEBUG);
}
if(data == NULL) {
set_bit_inplace(fsa_actions, with_actions);
with_actions = A_NOTHING;
return 0;
}
crm_err("%s stalled the FSA with data - this may be broken",
raised_from);
}
if(old_len == 0) {
last_was_vote = FALSE;
}
if(input == I_NULL && with_actions == A_NOTHING /* && data == NULL */) {
/* no point doing anything */
crm_err("Cannot add entry to queue: no input and no action");
return 0;
} else if(data == NULL) {
last_was_vote = FALSE;
#if 0
} else if(last_was_vote && cause == C_HA_MESSAGE && input == I_ROUTER) {
const char *op = cl_get_string(
((ha_msg_input_t*)data)->msg, F_CRM_TASK);
if(safe_str_eq(op, CRM_OP_VOTE)) {
/* It is always safe to treat N successive votes as
* a single one
*
* If all the discarded votes are more "loosing" than
* the first then the result is accurate
* (win or loose).
*
* If any of the discarded votes are less "loosing"
* than the first then we will cast our vote and the
* eventual winner will vote us down again (which
* even in the case that N=2, is no worse than if we
* had not disarded the vote).
*/
crm_debug_2("Vote compression: %d", old_len);
return 0;
}
#endif
} else if (cause == C_HA_MESSAGE && input == I_ROUTER) {
const char *op = cl_get_string(
((ha_msg_input_t*)data)->msg, F_CRM_TASK);
if(safe_str_eq(op, CRM_OP_VOTE)) {
last_was_vote = TRUE;
crm_debug_3("Added vote: %d", old_len);
}
} else {
last_was_vote = FALSE;
}
crm_malloc0(fsa_data, sizeof(fsa_data_t));
fsa_data->id = last_data_id;
fsa_data->fsa_input = input;
fsa_data->fsa_cause = cause;
fsa_data->origin = raised_from;
fsa_data->data = NULL;
fsa_data->data_type = fsa_dt_none;
fsa_data->actions = with_actions;
if(with_actions != A_NOTHING) {
crm_debug_3("Adding actions %.16llx to input", with_actions);
}
if(data != NULL) {
switch(cause) {
case C_FSA_INTERNAL:
case C_CRMD_STATUS_CALLBACK:
case C_IPC_MESSAGE:
case C_HA_MESSAGE:
crm_debug_3("Copying %s data from %s as a HA msg",
fsa_cause2string(cause),
raised_from);
fsa_data->data = copy_ha_msg_input(data);
fsa_data->data_type = fsa_dt_ha_msg;
break;
case C_LRM_OP_CALLBACK:
crm_debug_3("Copying %s data from %s as lrm_op_t",
fsa_cause2string(cause),
raised_from);
fsa_data->data = copy_lrm_op((lrm_op_t*)data);
fsa_data->data_type = fsa_dt_lrm;
break;
case C_CCM_CALLBACK:
crm_debug_3("Copying %s data from %s as CCM data",
fsa_cause2string(cause),
raised_from);
fsa_data->data = copy_ccm_data(data);
fsa_data->data_type = fsa_dt_ccm;
break;
case C_SUBSYSTEM_CONNECT:
case C_LRM_MONITOR_CALLBACK:
case C_TIMER_POPPED:
case C_SHUTDOWN:
case C_HEARTBEAT_FAILED:
case C_HA_DISCONNECT:
case C_ILLEGAL:
case C_UNKNOWN:
case C_STARTUP:
crm_err("Copying %s data (from %s)"
" not yet implemented",
fsa_cause2string(cause), raised_from);
exit(1);
break;
}
crm_debug_4("%s data copied",
fsa_cause2string(fsa_data->fsa_cause));
}
/* make sure to free it properly later */
if(prepend) {
crm_debug_2("Prepending input");
fsa_message_queue = g_list_prepend(fsa_message_queue, fsa_data);
} else {
fsa_message_queue = g_list_append(fsa_message_queue, fsa_data);
}
crm_debug_2("Queue len: %d", g_list_length(fsa_message_queue));
fsa_dump_queue(LOG_DEBUG_2);
if(old_len == g_list_length(fsa_message_queue)) {
crm_err("Couldnt add message to the queue");
}
if(fsa_source) {
crm_debug_3("Triggering FSA: %s", __FUNCTION__);
G_main_set_trigger(fsa_source);
}
return last_data_id;
}
