static gboolean
te_crm_command(crm_graph_t *graph, crm_action_t *action)
{
char *value = NULL;
char *counter = NULL;
HA_Message *cmd = NULL;
const char *id = NULL;
const char *task = NULL;
const char *on_node = NULL;
gboolean ret = TRUE;
id = ID(action->xml);
task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
te_log_action(LOG_ERR, "Corrupted command (id=%s) %s: no node",
crm_str(id), crm_str(task));
return FALSE);
te_log_action(LOG_INFO, "Executing crm-event (%s): %s on %s",
crm_str(id), crm_str(task), on_node);
cmd = create_request(task, NULL, on_node, CRM_SYSTEM_CRMD,
CRM_SYSTEM_TENGINE, NULL);
counter = generate_transition_key(
transition_graph->id, action->id, te_uuid);
crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
ret = send_ipc_message(crm_ch, cmd);
crm_free(counter);
crm_msg_del(cmd);
value = g_hash_table_lookup(action->params, crm_meta_name(XML_ATTR_TE_NOWAIT));
if(ret == FALSE) {
crm_err("Action %d failed: send", action->id);
return FALSE;
} else if(crm_is_true(value)) {
crm_info("Skipping wait for %d", action->id);
action->confirmed = TRUE;
update_graph(graph, action);
trigger_graph();
} else if(ret && action->timeout > 0) {
crm_debug("Setting timer for action %d",action->id);
action->timer->reason = timeout_action_warn;
te_start_action_timer(action);
}
return TRUE;
}
void
notify_crmd(crm_graph_t * graph)
{
const char *type = "unknown";
enum crmd_fsa_input event = I_NULL;
crm_debug("Processing transition completion in state %s", fsa_state2string(fsa_state));
CRM_CHECK(graph->complete, graph->complete = TRUE);
switch (graph->completion_action) {
case tg_stop:
type = "stop";
/* fall through */
case tg_done:
type = "done";
if (fsa_state == S_TRANSITION_ENGINE) {
event = I_TE_SUCCESS;
}
break;
case tg_restart:
type = "restart";
if (fsa_state == S_TRANSITION_ENGINE) {
if (transition_timer->period_ms > 0) {
crm_timer_start(transition_timer);
} else {
event = I_PE_CALC;
}
} else if (fsa_state == S_POLICY_ENGINE) {
register_fsa_action(A_PE_INVOKE);
}
break;
case tg_shutdown:
type = "shutdown";
if (is_set(fsa_input_register, R_SHUTDOWN)) {
event = I_STOP;
} else {
crm_err("We didn't ask to be shut down, yet our" " PE is telling us too.");
event = I_TERMINATE;
}
}
te_log_action(LOG_DEBUG, "Transition %d status: %s - %s",
graph->id, type, crm_str(graph->abort_reason));
graph->abort_reason = NULL;
graph->completion_action = tg_done;
clear_bit_inplace(fsa_input_register, R_IN_TRANSITION);
if (event != I_NULL) {
register_fsa_input(C_FSA_INTERNAL, event, NULL);
} else if (fsa_source) {
mainloop_set_trigger(fsa_source);
}
}
void
notify_crmd(crm_graph_t *graph)
{
HA_Message *cmd = NULL;
int log_level = LOG_DEBUG;
const char *op = CRM_OP_TEABORT;
int pending_callbacks = num_cib_op_callbacks();
stop_te_timer(transition_timer);
if(pending_callbacks != 0) {
crm_warn("Delaying completion until all CIB updates complete");
return;
}
CRM_CHECK(graph->complete, graph->complete = TRUE);
switch(graph->completion_action) {
case tg_stop:
op = CRM_OP_TECOMPLETE;
log_level = LOG_INFO;
break;
case tg_abort:
case tg_restart:
op = CRM_OP_TEABORT;
break;
case tg_shutdown:
crm_info("Exiting after transition");
if (mainloop != NULL && g_main_is_running(mainloop)) {
g_main_quit(mainloop);
return;
}
exit(LSB_EXIT_OK);
}
te_log_action(log_level, "Transition %d status: %s - %s",
graph->id, op, crm_str(graph->abort_reason));
print_graph(LOG_DEBUG_3, graph);
cmd = create_request(
op, NULL, NULL, CRM_SYSTEM_DC, CRM_SYSTEM_TENGINE, NULL);
if(graph->abort_reason != NULL) {
ha_msg_add(cmd, "message", graph->abort_reason);
}
send_ipc_message(crm_ch, cmd);
crm_msg_del(cmd);
graph->abort_reason = NULL;
graph->completion_action = tg_restart;
}
static gboolean
te_fence_node(crm_graph_t *graph, crm_action_t *action)
{
const char *id = NULL;
const char *uuid = NULL;
const char *target = NULL;
const char *type = NULL;
stonith_ops_t * st_op = NULL;
id = ID(action->xml);
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
type = g_hash_table_lookup(action->params, crm_meta_name("stonith_action"));
CRM_CHECK(id != NULL,
crm_log_xml_warn(action->xml, "BadAction");
return FALSE);
CRM_CHECK(uuid != NULL,
crm_log_xml_warn(action->xml, "BadAction");
return FALSE);
CRM_CHECK(type != NULL,
crm_log_xml_warn(action->xml, "BadAction");
return FALSE);
CRM_CHECK(target != NULL,
crm_log_xml_warn(action->xml, "BadAction");
return FALSE);
te_log_action(LOG_INFO,
"Executing %s fencing operation (%s) on %s (timeout=%d)",
type, id, target,
transition_graph->transition_timeout / 2);
/* Passing NULL means block until we can connect... */
te_connect_stonith(NULL);
crm_malloc0(st_op, sizeof(stonith_ops_t));
if(safe_str_eq(type, "poweroff")) {
st_op->optype = POWEROFF;
} else {
st_op->optype = RESET;
}
st_op->timeout = transition_graph->transition_timeout / 2;
st_op->node_name = crm_strdup(target);
st_op->node_uuid = crm_strdup(uuid);
st_op->private_data = generate_transition_key(
transition_graph->id, action->id, te_uuid);
CRM_ASSERT(stonithd_input_IPC_channel() != NULL);
if (ST_OK != stonithd_node_fence( st_op )) {
crm_err("Cannot fence %s: stonithd_node_fence() call failed ",
target);
}
return TRUE;
}
static gboolean
te_fence_node(crm_graph_t * graph, crm_action_t * action)
{
int rc = 0;
const char *id = NULL;
const char *uuid = NULL;
const char *target = NULL;
const char *type = NULL;
gboolean invalid_action = FALSE;
enum stonith_call_options options = st_opt_none;
id = ID(action->xml);
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
type = crm_meta_value(action->params, "stonith_action");
CRM_CHECK(id != NULL, invalid_action = TRUE);
CRM_CHECK(uuid != NULL, invalid_action = TRUE);
CRM_CHECK(type != NULL, invalid_action = TRUE);
CRM_CHECK(target != NULL, invalid_action = TRUE);
if (invalid_action) {
crm_log_xml_warn(action->xml, "BadAction");
return FALSE;
}
te_log_action(LOG_INFO,
"Executing %s fencing operation (%s) on %s (timeout=%d)",
type, id, target, transition_graph->stonith_timeout);
/* Passing NULL means block until we can connect... */
te_connect_stonith(NULL);
if (finalized_nodes && g_hash_table_size(finalized_nodes) == 1) {
options |= st_opt_allow_suicide;
}
rc = stonith_api->cmds->fence(stonith_api, options, target, type,
transition_graph->stonith_timeout / 1000);
stonith_api->cmds->register_callback(stonith_api, rc, transition_graph->stonith_timeout / 1000,
FALSE, generate_transition_key(transition_graph->id,
action->id, 0, te_uuid),
"tengine_stonith_callback", tengine_stonith_callback);
return TRUE;
}
static gboolean
te_rsc_command(crm_graph_t *graph, crm_action_t *action)
{
/* never overwrite stop actions in the CIB with
* anything other than completed results
*
* Writing pending stops makes it look like the
* resource is running again
*/
const char *task = NULL;
const char *on_node = NULL;
action->executed = FALSE;
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
te_log_action(LOG_ERR, "Corrupted command(id=%s) %s: no node",
ID(action->xml), crm_str(task));
return FALSE);
send_rsc_command(action);
return TRUE;
}
static gboolean
initiate_action(crm_graph_t * graph, crm_action_t * action)
{
const char *id = NULL;
CRM_CHECK(action->executed == FALSE, return FALSE);
id = ID(action->xml);
CRM_CHECK(id != NULL, return FALSE);
action->executed = TRUE;
if (action->type == action_type_pseudo) {
crm_trace("Executing pseudo-event: %d", action->id);
return graph_fns->pseudo(graph, action);
} else if (action->type == action_type_rsc) {
crm_trace("Executing rsc-event: %d", action->id);
return graph_fns->rsc(graph, action);
} else if (action->type == action_type_crm) {
const char *task = NULL;
task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
CRM_CHECK(task != NULL, return FALSE);
if (safe_str_eq(task, CRM_OP_FENCE)) {
crm_trace("Executing STONITH-event: %d", action->id);
return graph_fns->stonith(graph, action);
}
crm_trace("Executing crm-event: %d", action->id);
return graph_fns->crmd(graph, action);
}
te_log_action(LOG_ERR,
"Failed on unsupported command type: %s (id=%s)",
crm_element_name(action->xml), id);
return FALSE;
}
static gboolean
te_rsc_command(crm_graph_t * graph, crm_action_t * action)
{
/* never overwrite stop actions in the CIB with
* anything other than completed results
*
* Writing pending stops makes it look like the
* resource is running again
*/
xmlNode *cmd = NULL;
xmlNode *rsc_op = NULL;
gboolean rc = TRUE;
gboolean no_wait = FALSE;
gboolean is_local = FALSE;
char *counter = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
const char *task_uuid = NULL;
CRM_ASSERT(action != NULL);
CRM_ASSERT(action->xml != NULL);
action->executed = FALSE;
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
te_log_action(LOG_ERR, "Corrupted command(id=%s) %s: no node",
ID(action->xml), crm_str(task));
return FALSE);
rsc_op = action->xml;
task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
on_node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
counter =
generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
if (safe_str_eq(on_node, fsa_our_uname)) {
is_local = TRUE;
}
value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
if (crm_is_true(value)) {
no_wait = TRUE;
}
crm_info("Initiating action %d: %s %s on %s%s%s",
action->id, task, task_uuid, on_node,
is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, on_node,
CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
if (is_local) {
/* shortcut local resource commands */
ha_msg_input_t data = {
.msg = cmd,
.xml = rsc_op,
};
fsa_data_t msg = {
.id = 0,
.data = &data,
.data_type = fsa_dt_ha_msg,
.fsa_input = I_NULL,
.fsa_cause = C_FSA_INTERNAL,
.actions = A_LRM_INVOKE,
.origin = __FUNCTION__,
};
do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
} else {
rc = send_cluster_message(on_node, crm_msg_lrmd, cmd, TRUE);
}
crm_free(counter);
free_xml(cmd);
action->executed = TRUE;
if (rc == FALSE) {
crm_err("Action %d failed: send", action->id);
return FALSE;
} else if (no_wait) {
action->confirmed = TRUE;
update_graph(transition_graph, action);
trigger_graph();
} else {
if (action->timeout <= 0) {
crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %dms instead",
action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
action->timeout = graph->network_delay;
}
te_start_action_timer(graph, action);
}
value = crm_meta_value(action->params, XML_OP_ATTR_PENDING);
if (crm_is_true(value)) {
/* write a "pending" entry to the CIB, inhibit notification */
crm_info("Recording pending op %s in the CIB", task_uuid);
cib_action_update(action, LRM_OP_PENDING, EXECRA_STATUS_UNKNOWN);
}
return TRUE;
}
static gboolean
te_crm_command(crm_graph_t * graph, crm_action_t * action)
{
char *counter = NULL;
xmlNode *cmd = NULL;
gboolean is_local = FALSE;
const char *id = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
gboolean rc = TRUE;
gboolean no_wait = FALSE;
id = ID(action->xml);
task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
te_log_action(LOG_ERR, "Corrupted command (id=%s) %s: no node",
crm_str(id), crm_str(task));
return FALSE);
te_log_action(LOG_INFO, "Executing crm-event (%s): %s on %s%s%s",
crm_str(id), crm_str(task), on_node,
is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
if (safe_str_eq(on_node, fsa_our_uname)) {
is_local = TRUE;
}
value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
if (crm_is_true(value)) {
no_wait = TRUE;
}
if (is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
/* defer until everything else completes */
te_log_action(LOG_INFO, "crm-event (%s) is a local shutdown", crm_str(id));
graph->completion_action = tg_shutdown;
graph->abort_reason = "local shutdown";
action->confirmed = TRUE;
update_graph(graph, action);
trigger_graph();
return TRUE;
}
cmd = create_request(task, action->xml, on_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
counter =
generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
rc = send_cluster_message(on_node, crm_msg_crmd, cmd, TRUE);
crm_free(counter);
free_xml(cmd);
if (rc == FALSE) {
crm_err("Action %d failed: send", action->id);
return FALSE;
} else if (no_wait) {
action->confirmed = TRUE;
update_graph(graph, action);
trigger_graph();
} else {
if (action->timeout <= 0) {
crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %dms instead",
action->id, task, on_node, action->timeout, graph->network_delay);
action->timeout = graph->network_delay;
}
te_start_action_timer(graph, action);
}
return TRUE;
}
/*
* returns the ID of the action if a match is found
* returns -1 if a match was not found
* returns -2 if a match was found but the action failed (and was
* not allowed to)
*/
int
match_graph_event(int action_id, xmlNode * event, const char *event_node,
int op_status, int op_rc, int target_rc)
{
const char *target = NULL;
const char *allow_fail = NULL;
const char *this_event = NULL;
crm_action_t *action = NULL;
action = get_action(action_id, FALSE);
if (action == NULL) {
return -1;
}
op_status = status_from_rc(action, op_status, op_rc, target_rc);
if (op_status != LRM_OP_DONE) {
update_failcount(event, event_node, op_rc, target_rc, FALSE);
}
/* Process OP status */
switch (op_status) {
case LRM_OP_PENDING:
crm_debug("Ignoring pending operation");
return action->id;
break;
case LRM_OP_DONE:
break;
case LRM_OP_ERROR:
case LRM_OP_TIMEOUT:
case LRM_OP_NOTSUPPORTED:
action->failed = TRUE;
break;
case LRM_OP_CANCELLED:
/* do nothing?? */
crm_err("Dont know what to do for cancelled ops yet");
break;
default:
action->failed = TRUE;
crm_err("Unsupported action result: %d", op_status);
}
/* stop this event's timer if it had one */
stop_te_timer(action->timer);
action->confirmed = TRUE;
update_graph(transition_graph, action);
trigger_graph();
if (action->failed) {
allow_fail = crm_meta_value(action->params, XML_ATTR_TE_ALLOWFAIL);
if (crm_is_true(allow_fail)) {
action->failed = FALSE;
}
}
if (action->failed) {
abort_transition(action->synapse->priority + 1, tg_restart, "Event failed", event);
}
this_event = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
te_log_action(LOG_INFO, "Action %s (%d) confirmed on %s (rc=%d)",
crm_str(this_event), action->id, crm_str(target), op_status);
return action->id;
}
