static gboolean
pseudo_action_dummy(crm_graph_t * graph, crm_action_t * action)
{
static int fail = -1;
if (fail < 0) {
char *fail_s = getenv("PE_fail");
if (fail_s) {
fail = crm_int_helper(fail_s, NULL);
} else {
fail = 0;
}
}
crm_trace("Dummy event handler: action %d executed", action->id);
if (action->id == fail) {
crm_err("Dummy event handler: pretending action %d failed", action->id);
action->failed = TRUE;
graph->abort_priority = INFINITY;
}
action->confirmed = TRUE;
update_graph(graph, action);
return TRUE;
}
static gboolean
pseudo_action_dummy(crm_graph_t *graph, crm_action_t *action)
{
crm_debug_2("Dummy event handler: action %d executed", action->id);
action->confirmed = TRUE;
update_graph(graph, action);
return TRUE;
}
static void region_update_graph(chan_info *cp)
{
if (current_region == -1) return;
rsp->nchans = region_chans(region_list_position_to_id(current_region));
if (rsp->nchans == 0) return;
update_graph(cp);
rsp->nchans = 1;
}
static gboolean
te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
{
crm_info("Pseudo action %d fired and confirmed", pseudo->id);
pseudo->confirmed = TRUE;
update_graph(graph, pseudo);
trigger_graph();
return TRUE;
}
static gboolean
ttest_pseudo_command(crm_graph_t *graph, crm_action_t *pseudo)
{
crm_debug("Event handler: action %d executed", pseudo->id);
pseudo->confirmed = TRUE;
update_graph(graph, pseudo);
trigger_graph();
return TRUE;
}
static gboolean
te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
{
crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
te_action_confirmed(pseudo);
update_graph(graph, pseudo);
trigger_graph();
return TRUE;
}
void run_tasks()
{
if(task_flags & task_15ms_flag)
{
Page_Event_Handler();
led_state_machine();
run_oven_state_machine();
task_flags &=~task_15ms_flag;
}
if(task_flags & task_32ms_flag)
{
task_flags &=~task_32ms_flag;
update_temp();
}
if(task_flags & task_62ms_flag)
{
if(Get_LED_Timer_Status())
{
GLED_Sequence();
RLED_Sequence();
BLED_Sequence();
task_flags &=~task_62ms_flag;
}
}
if(task_flags & task_250ms_flag)
{
task_flags &=~task_250ms_flag;
}
if(task_flags & task_500ms_flag)
{
setColor16(COLOR_16_WHITE);
get_system_time_string(text_buffer);
drawString(10,160,text_buffer);
print_oven_data();
if(task_flags & graph_update)
{
update_graph();
task_flags &= ~graph_update;
}
task_flags &=~task_500ms_flag;
//RLED_OUT ^= RLED;
}
}
static gboolean
te_crm_command(crm_graph_t *graph, crm_action_t *action)
{
char *value = NULL;
char *counter = NULL;
HA_Message *cmd = NULL;
const char *id = NULL;
const char *task = NULL;
const char *on_node = NULL;
gboolean ret = TRUE;
id = ID(action->xml);
task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
te_log_action(LOG_ERR, "Corrupted command (id=%s) %s: no node",
crm_str(id), crm_str(task));
return FALSE);
te_log_action(LOG_INFO, "Executing crm-event (%s): %s on %s",
crm_str(id), crm_str(task), on_node);
cmd = create_request(task, NULL, on_node, CRM_SYSTEM_CRMD,
CRM_SYSTEM_TENGINE, NULL);
counter = generate_transition_key(
transition_graph->id, action->id, te_uuid);
crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
ret = send_ipc_message(crm_ch, cmd);
crm_free(counter);
crm_msg_del(cmd);
value = g_hash_table_lookup(action->params, crm_meta_name(XML_ATTR_TE_NOWAIT));
if(ret == FALSE) {
crm_err("Action %d failed: send", action->id);
return FALSE;
} else if(crm_is_true(value)) {
crm_info("Skipping wait for %d", action->id);
action->confirmed = TRUE;
update_graph(graph, action);
trigger_graph();
} else if(ret && action->timeout > 0) {
crm_debug("Setting timer for action %d",action->id);
action->timer->reason = timeout_action_warn;
te_start_action_timer(action);
}
return TRUE;
}
/*!
* \internal
* \brief Confirm action and update transition graph, aborting transition on failures
*
* \param[in/out] action CRM action instance of this operation
* \param[in] event Event instance of this operation
* \param[in] orig_status Original reported operation status
* \param[in] op_rc Actual operation return code
* \param[in] target_rc Expected operation return code
* \param[in] ignore_failures Whether to ignore operation failures
*
* \note This assumes that PCMK_LRM_OP_PENDING operations have already been
* filtered (otherwise they may be treated as failures).
*/
static void
match_graph_event(crm_action_t *action, xmlNode *event, int op_status,
int op_rc, int target_rc, gboolean ignore_failures)
{
const char *target = NULL;
const char *this_event = NULL;
const char *ignore_s = "";
/* Remap operation status based on return code */
op_status = status_from_rc(action, op_status, op_rc, target_rc);
/* Process OP status */
switch (op_status) {
case PCMK_LRM_OP_DONE:
break;
case PCMK_LRM_OP_ERROR:
case PCMK_LRM_OP_TIMEOUT:
case PCMK_LRM_OP_NOTSUPPORTED:
if (ignore_failures) {
ignore_s = ", ignoring failure";
} else {
action->failed = TRUE;
}
break;
case PCMK_LRM_OP_CANCELLED:
/* do nothing?? */
crm_err("Don't know what to do for cancelled ops yet");
break;
default:
/*
PCMK_LRM_OP_ERROR_HARD,
PCMK_LRM_OP_ERROR_FATAL,
PCMK_LRM_OP_NOT_INSTALLED
*/
action->failed = TRUE;
crm_err("Unsupported action result: %d", op_status);
}
/* stop this event's timer if it had one */
stop_te_timer(action->timer);
te_action_confirmed(action);
update_graph(transition_graph, action);
trigger_graph();
if (action->failed) {
abort_transition(action->synapse->priority + 1, tg_restart, "Event failed", event);
}
this_event = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
crm_info("Action %s (%d) confirmed on %s (rc=%d%s)",
crm_str(this_event), action->id, crm_str(target), op_rc, ignore_s);
}
static gboolean
exec_crmd_action(crm_graph_t *graph, crm_action_t *action)
{
qb_enter();
action->confirmed = TRUE;
update_graph(graph, action);
graph_updated = TRUE;
qb_leave();
return TRUE;
}
/*!
* \brief Handle a timeout in node-to-node communication
*
* \param[in] data Pointer to action timer
*
* \return FALSE (indicating that source should be not be re-added)
*/
gboolean
action_timer_callback(gpointer data)
{
crm_action_timer_t *timer = NULL;
const char *task = NULL;
const char *on_node = NULL;
const char *via_node = NULL;
CRM_CHECK(data != NULL, return FALSE);
timer = (crm_action_timer_t *) data;
stop_te_timer(timer);
CRM_CHECK(timer->action != NULL, return FALSE);
task = crm_element_value(timer->action->xml, XML_LRM_ATTR_TASK);
on_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_TARGET);
via_node = crm_element_value(timer->action->xml, XML_LRM_ATTR_ROUTER_NODE);
if (transition_graph->complete) {
crm_notice("Node %s did not send %s result (via %s) within %dms "
"(ignoring because transition not in progress)",
(on_node? on_node : ""), (task? task : "unknown action"),
(via_node? via_node : "controller"), timer->timeout);
} else {
/* fail the action */
crm_err("Node %s did not send %s result (via %s) within %dms "
"(action timeout plus cluster-delay)",
(on_node? on_node : ""), (task? task : "unknown action"),
(via_node? via_node : "controller"), timer->timeout);
print_action(LOG_ERR, "Aborting transition, action lost: ", timer->action);
timer->action->failed = TRUE;
te_action_confirmed(timer->action);
abort_transition(INFINITY, tg_restart, "Action lost", NULL);
update_graph(transition_graph, timer->action);
trigger_graph();
// Record timeout in the CIB if appropriate
if ((timer->action->type == action_type_rsc)
&& controld_action_is_recordable(task)) {
controld_record_action_timeout(timer->action);
}
}
return FALSE;
}
void simulation_render(float delta_time)
{
if(cells != NULL)
{
int i = 0, j = 0;
for(i = 0; i < MAP_HEIGHT; i++)
{
for(j = 0; j < MAP_WIDTH; j++)
{
cell_draw(j, i);
}
}
}
update_graph();
}
static gboolean
fail_incompletable_stonith(crm_graph_t * graph)
{
GListPtr lpc = NULL;
const char *task = NULL;
xmlNode *last_action = NULL;
if (graph == NULL) {
return FALSE;
}
for (lpc = graph->synapses; lpc != NULL; lpc = lpc->next) {
GListPtr lpc2 = NULL;
synapse_t *synapse = (synapse_t *) lpc->data;
if (synapse->confirmed) {
continue;
}
for (lpc2 = synapse->actions; lpc2 != NULL; lpc2 = lpc2->next) {
crm_action_t *action = (crm_action_t *) lpc2->data;
if (action->type != action_type_crm || action->confirmed) {
continue;
}
task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
if (task && safe_str_eq(task, CRM_OP_FENCE)) {
action->failed = TRUE;
last_action = action->xml;
update_graph(graph, action);
crm_notice("Failing action %d (%s): STONITHd terminated",
action->id, ID(action->xml));
}
}
}
if (last_action != NULL) {
crm_warn("STONITHd failure resulted in un-runnable actions");
abort_transition(INFINITY, tg_restart, "Stonith failure", last_action);
return TRUE;
}
return FALSE;
}
static gboolean
exec_stonith_action(crm_graph_t *graph, crm_action_t *action)
{
char *target = crm_element_value_copy(action->xml, XML_LRM_ATTR_TARGET);
qb_enter();
qb_log(LOG_WARNING, "Skipping STONITH %s op (not fencing %s)\n",
crm_element_value(action->xml, "operation"), target);
crm_free(target);
action->confirmed = TRUE;
update_graph(graph, action);
graph_updated = TRUE;
qb_leave();
return TRUE;
}
void queue_debug::init_gui()
{
auto* layout = new QGridLayout{this};
setLayout(layout);
_name = new QLabel{_name_s.c_str()};
layout->addWidget(_name, 0,0,2,1);
_in_graph = new QGraphicsView;
_out_graph = new QGraphicsView;
layout->addWidget(_in_graph, 0,1,1,3);
layout->addWidget(_out_graph, 1,1,1,3);
init_scene(*_in_graph);
init_scene(*_out_graph);
auto *t = new QTimer(this);
connect(t, SIGNAL(timeout()), this, SLOT(update_graph()));
t->start(UPDATE_GRAPH);
}
static void
process_op_deletion(const char *xpath, xmlNode *change)
{
char *mutable_key = strdup(xpath);
char *key;
char *node_uuid;
crm_action_t *cancel = NULL;
// Extract the part of xpath between last pair of single quotes
key = strrchr(mutable_key, '\'');
if (key != NULL) {
*key = '\0';
key = strrchr(mutable_key, '\'');
}
if (key == NULL) {
crm_warn("Ignoring malformed CIB update (resource deletion of %s)",
xpath);
free(mutable_key);
return;
}
++key;
node_uuid = extract_node_uuid(xpath);
cancel = get_cancel_action(key, node_uuid);
if (cancel) {
crm_info("Cancellation of %s on %s confirmed (%d)",
key, node_uuid, cancel->id);
stop_te_timer(cancel->timer);
te_action_confirmed(cancel);
update_graph(transition_graph, cancel);
trigger_graph();
} else {
abort_transition(INFINITY, tg_restart, "Resource operation removal",
change);
}
free(mutable_key);
free(node_uuid);
}
void
pe_resource_completed(struct pe_operation *op, uint32_t return_code)
{
crm_graph_t *graph = op->graph;
crm_action_t *action = op->action;
qb_enter();
if (working_set == NULL) {
qb_leave();
return;
}
if (return_code != op->target_outcome) {
action->failed = TRUE;
graph->abort_priority = INFINITY;
}
action->confirmed = TRUE;
update_graph(graph, action);
graph_updated = TRUE;
qb_leave();
}
static gboolean
te_pseudo_action(crm_graph_t * graph, crm_action_t * pseudo)
{
const char *task = crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK);
/* send to peers as well? */
if (safe_str_eq(task, CRM_OP_MAINTENANCE_NODES)) {
GHashTableIter iter;
crm_node_t *node = NULL;
g_hash_table_iter_init(&iter, crm_peer_cache);
while (g_hash_table_iter_next(&iter, NULL, (gpointer *) &node)) {
xmlNode *cmd = NULL;
if (safe_str_eq(fsa_our_uname, node->uname)) {
continue;
}
cmd = create_request(task, pseudo->xml, node->uname,
CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
send_cluster_message(node, crm_msg_crmd, cmd, FALSE);
free_xml(cmd);
}
remote_ra_process_maintenance_nodes(pseudo->xml);
} else {
/* Check action for Pacemaker Remote node side effects */
remote_ra_process_pseudo(pseudo->xml);
}
crm_debug("Pseudo-action %d (%s) fired and confirmed", pseudo->id,
crm_element_value(pseudo->xml, XML_LRM_ATTR_TASK_KEY));
te_action_confirmed(pseudo);
update_graph(graph, pseudo);
trigger_graph();
return TRUE;
}
void
peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
uint32_t old = 0;
uint32_t changed = 0;
bool appeared = FALSE;
const char *status = NULL;
set_bit(fsa_input_register, R_PEER_DATA);
if (node->uname == NULL) {
return;
}
switch (type) {
case crm_status_uname:
/* If we've never seen the node, then it also wont be in the status section */
crm_info("%s is now %s", node->uname, node->state);
return;
case crm_status_rstate:
crm_info("Remote node %s is now %s (was %s)", node->uname, node->state, (const char *)data);
/* Keep going */
case crm_status_nstate:
crm_info("%s is now %s (was %s)", node->uname, node->state, (const char *)data);
if (safe_str_eq(data, node->state)) {
/* State did not change */
return;
} else if(safe_str_eq(CRM_NODE_MEMBER, node->state)) {
appeared = TRUE;
}
break;
case crm_status_processes:
if (data) {
old = *(const uint32_t *)data;
changed = node->processes ^ old;
}
/* crmd_proc_update(node, proc_flags); */
status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
node->uname, peer2text(proc_flags), status,
AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);
if ((changed & proc_flags) == 0) {
/* Peer process did not change */
crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
return;
} else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
crm_trace("Not connected");
return;
} else if (fsa_state == S_STOPPING) {
crm_trace("Stopping");
return;
}
appeared = (node->processes & proc_flags) != 0;
if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
/* Did we get evicted? */
crm_notice("Our peer connection failed");
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
} else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
/* Did the DC leave us? */
crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
} else if(AM_I_DC && appeared == FALSE) {
crm_info("Peer %s left us", node->uname);
/* crm_update_peer_join(__FUNCTION__, node, crm_join_none); */
}
break;
}
if (AM_I_DC) {
xmlNode *update = NULL;
int flags = node_update_peer;
gboolean alive = crm_is_peer_active(node);
crm_action_t *down = match_down_event(0, node->uuid, NULL, appeared);
crm_trace("Alive=%d, appear=%d, down=%p", alive, appeared, down);
if (alive && type == crm_status_processes) {
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
if (down) {
const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
if (alive && safe_str_eq(task, CRM_OP_FENCE)) {
crm_info("Node return implies stonith of %s (action %d) completed", node->uname,
down->id);
erase_status_tag(node->uname, XML_CIB_TAG_LRM, cib_scope_local);
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
/* down->confirmed = TRUE; Only stonith-ng returning should imply completion */
down->sent_update = TRUE; /* Prevent tengine_stonith_callback() from calling send_stonith_update() */
} else if (safe_str_eq(task, CRM_OP_FENCE)) {
crm_trace("Waiting for stonithd to report the fencing of %s is complete", node->uname); /* via tengine_stonith_callback() */
} else if (alive == FALSE) {
crm_notice("%s of %s (op %d) is complete", task, node->uname, down->id);
/* down->confirmed = TRUE; Only stonith-ng returning should imply completion */
stop_te_timer(down->timer);
flags |= node_update_join | node_update_expected;
crmd_peer_down(node, FALSE);
check_join_state(fsa_state, __FUNCTION__);
update_graph(transition_graph, down);
trigger_graph();
} else {
crm_trace("Other %p", down);
}
} else if (appeared == FALSE) {
crm_notice("Stonith/shutdown of %s not matched", node->uname);
crm_update_peer_join(__FUNCTION__, node, crm_join_none);
check_join_state(fsa_state, __FUNCTION__);
abort_transition(INFINITY, tg_restart, "Node failure", NULL);
fail_incompletable_actions(transition_graph, node->uuid);
} else {
crm_trace("Other %p", down);
}
update = do_update_node_cib(node, flags, NULL, __FUNCTION__);
fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
cib_scope_local | cib_quorum_override | cib_can_create);
free_xml(update);
}
trigger_fsa(fsa_source);
}
static gboolean
te_crm_command(crm_graph_t * graph, crm_action_t * action)
{
char *counter = NULL;
xmlNode *cmd = NULL;
gboolean is_local = FALSE;
const char *id = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
gboolean rc = TRUE;
gboolean no_wait = FALSE;
id = ID(action->xml);
task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
te_log_action(LOG_ERR, "Corrupted command (id=%s) %s: no node",
crm_str(id), crm_str(task));
return FALSE);
te_log_action(LOG_INFO, "Executing crm-event (%s): %s on %s%s%s",
crm_str(id), crm_str(task), on_node,
is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
if (safe_str_eq(on_node, fsa_our_uname)) {
is_local = TRUE;
}
value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
if (crm_is_true(value)) {
no_wait = TRUE;
}
if (is_local && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
/* defer until everything else completes */
te_log_action(LOG_INFO, "crm-event (%s) is a local shutdown", crm_str(id));
graph->completion_action = tg_shutdown;
graph->abort_reason = "local shutdown";
action->confirmed = TRUE;
update_graph(graph, action);
trigger_graph();
return TRUE;
}
cmd = create_request(task, action->xml, on_node, CRM_SYSTEM_CRMD, CRM_SYSTEM_TENGINE, NULL);
counter =
generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
crm_xml_add(cmd, XML_ATTR_TRANSITION_KEY, counter);
rc = send_cluster_message(on_node, crm_msg_crmd, cmd, TRUE);
crm_free(counter);
free_xml(cmd);
if (rc == FALSE) {
crm_err("Action %d failed: send", action->id);
return FALSE;
} else if (no_wait) {
action->confirmed = TRUE;
update_graph(graph, action);
trigger_graph();
} else {
if (action->timeout <= 0) {
crm_err("Action %d: %s on %s had an invalid timeout (%dms). Using %dms instead",
action->id, task, on_node, action->timeout, graph->network_delay);
action->timeout = graph->network_delay;
}
te_start_action_timer(graph, action);
}
return TRUE;
}
static gboolean
te_rsc_command(crm_graph_t * graph, crm_action_t * action)
{
/* never overwrite stop actions in the CIB with
* anything other than completed results
*
* Writing pending stops makes it look like the
* resource is running again
*/
xmlNode *cmd = NULL;
xmlNode *rsc_op = NULL;
gboolean rc = TRUE;
gboolean no_wait = FALSE;
gboolean is_local = FALSE;
char *counter = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
const char *task_uuid = NULL;
CRM_ASSERT(action != NULL);
CRM_ASSERT(action->xml != NULL);
action->executed = FALSE;
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
te_log_action(LOG_ERR, "Corrupted command(id=%s) %s: no node",
ID(action->xml), crm_str(task));
return FALSE);
rsc_op = action->xml;
task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
on_node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
counter =
generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
if (safe_str_eq(on_node, fsa_our_uname)) {
is_local = TRUE;
}
value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
if (crm_is_true(value)) {
no_wait = TRUE;
}
crm_info("Initiating action %d: %s %s on %s%s%s",
action->id, task, task_uuid, on_node,
is_local ? " (local)" : "", no_wait ? " - no waiting" : "");
cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, on_node,
CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
if (is_local) {
/* shortcut local resource commands */
ha_msg_input_t data = {
.msg = cmd,
.xml = rsc_op,
};
fsa_data_t msg = {
.id = 0,
.data = &data,
.data_type = fsa_dt_ha_msg,
.fsa_input = I_NULL,
.fsa_cause = C_FSA_INTERNAL,
.actions = A_LRM_INVOKE,
.origin = __FUNCTION__,
};
do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
} else {
rc = send_cluster_message(on_node, crm_msg_lrmd, cmd, TRUE);
}
crm_free(counter);
free_xml(cmd);
action->executed = TRUE;
if (rc == FALSE) {
crm_err("Action %d failed: send", action->id);
return FALSE;
} else if (no_wait) {
action->confirmed = TRUE;
update_graph(transition_graph, action);
trigger_graph();
} else {
if (action->timeout <= 0) {
crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %dms instead",
action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
action->timeout = graph->network_delay;
}
te_start_action_timer(graph, action);
}
value = crm_meta_value(action->params, XML_OP_ATTR_PENDING);
if (crm_is_true(value)) {
/* write a "pending" entry to the CIB, inhibit notification */
crm_info("Recording pending op %s in the CIB", task_uuid);
cib_action_update(action, LRM_OP_PENDING, EXECRA_STATUS_UNKNOWN);
}
return TRUE;
}
gboolean
fail_incompletable_actions(crm_graph_t * graph, const char *down_node)
{
const char *target = NULL;
xmlNode *last_action = NULL;
GListPtr gIter = NULL;
GListPtr gIter2 = NULL;
if (graph == NULL || graph->complete) {
return FALSE;
}
gIter = graph->synapses;
for (; gIter != NULL; gIter = gIter->next) {
synapse_t *synapse = (synapse_t *) gIter->data;
if (synapse->confirmed) {
continue;
}
gIter2 = synapse->actions;
for (; gIter2 != NULL; gIter2 = gIter2->next) {
crm_action_t *action = (crm_action_t *) gIter2->data;
if (action->type == action_type_pseudo || action->confirmed) {
continue;
} else if (action->type == action_type_crm) {
const char *task = crm_element_value(action->xml, XML_LRM_ATTR_TASK);
if (safe_str_eq(task, CRM_OP_FENCE)) {
continue;
}
}
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
if (safe_str_eq(target, down_node)) {
action->failed = TRUE;
synapse->failed = TRUE;
last_action = action->xml;
stop_te_timer(action->timer);
update_graph(graph, action);
if (synapse->executed) {
crm_notice("Action %d (%s) was pending on %s (offline)",
action->id, ID(action->xml), down_node);
} else {
crm_notice("Action %d (%s) is scheduled for %s (offline)",
action->id, ID(action->xml), down_node);
}
}
}
}
if (last_action != NULL) {
crm_warn("Node %s shutdown resulted in un-runnable actions", down_node);
abort_transition(INFINITY, tg_restart, "Node failure", last_action);
return TRUE;
}
return FALSE;
}
static gboolean
te_rsc_command(crm_graph_t * graph, crm_action_t * action)
{
/* never overwrite stop actions in the CIB with
* anything other than completed results
*
* Writing pending stops makes it look like the
* resource is running again
*/
xmlNode *cmd = NULL;
xmlNode *rsc_op = NULL;
gboolean rc = TRUE;
gboolean no_wait = FALSE;
gboolean is_local = FALSE;
char *counter = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
const char *router_node = NULL;
const char *task_uuid = NULL;
CRM_ASSERT(action != NULL);
CRM_ASSERT(action->xml != NULL);
action->executed = FALSE;
on_node = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
CRM_CHECK(on_node != NULL && strlen(on_node) != 0,
crm_err("Corrupted command(id=%s) %s: no node", ID(action->xml), crm_str(task));
return FALSE);
rsc_op = action->xml;
task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
router_node = crm_element_value(rsc_op, XML_LRM_ATTR_ROUTER_NODE);
if (!router_node) {
router_node = on_node;
}
counter =
generate_transition_key(transition_graph->id, action->id, get_target_rc(action), te_uuid);
crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
if (safe_str_eq(router_node, fsa_our_uname)) {
is_local = TRUE;
}
value = crm_meta_value(action->params, XML_ATTR_TE_NOWAIT);
if (crm_is_true(value)) {
no_wait = TRUE;
}
crm_notice("Initiating %s operation %s%s on %s%s "CRM_XS" action %d",
task, task_uuid, (is_local? " locally" : ""), on_node,
(no_wait? " without waiting" : ""), action->id);
cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, router_node,
CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
if (is_local) {
/* shortcut local resource commands */
ha_msg_input_t data = {
.msg = cmd,
.xml = rsc_op,
};
fsa_data_t msg = {
.id = 0,
.data = &data,
.data_type = fsa_dt_ha_msg,
.fsa_input = I_NULL,
.fsa_cause = C_FSA_INTERNAL,
.actions = A_LRM_INVOKE,
.origin = __FUNCTION__,
};
do_lrm_invoke(A_LRM_INVOKE, C_FSA_INTERNAL, fsa_state, I_NULL, &msg);
} else {
rc = send_cluster_message(crm_get_peer(0, router_node), crm_msg_lrmd, cmd, TRUE);
}
free(counter);
free_xml(cmd);
action->executed = TRUE;
if (rc == FALSE) {
crm_err("Action %d failed: send", action->id);
return FALSE;
} else if (no_wait) {
crm_info("Action %d confirmed - no wait", action->id);
action->confirmed = TRUE; /* Just mark confirmed.
* Don't bump the job count only to immediately decrement it
*/
update_graph(transition_graph, action);
trigger_graph();
} else if (action->confirmed == TRUE) {
crm_debug("Action %d: %s %s on %s(timeout %dms) was already confirmed.",
action->id, task, task_uuid, on_node, action->timeout);
} else {
if (action->timeout <= 0) {
crm_err("Action %d: %s %s on %s had an invalid timeout (%dms). Using %dms instead",
action->id, task, task_uuid, on_node, action->timeout, graph->network_delay);
action->timeout = graph->network_delay;
}
te_update_job_count(action, 1);
te_start_action_timer(graph, action);
}
return TRUE;
}
void
tengine_stonith_callback(stonith_t * stonith, stonith_callback_data_t * data)
{
char *uuid = NULL;
int stonith_id = -1;
int transition_id = -1;
crm_action_t *action = NULL;
int call_id = data->call_id;
int rc = data->rc;
char *userdata = data->userdata;
CRM_CHECK(userdata != NULL, return);
crm_notice("Stonith operation %d/%s: %s (%d)", call_id, (char *)userdata,
pcmk_strerror(rc), rc);
if (AM_I_DC == FALSE) {
return;
}
/* crm_info("call=%d, optype=%d, node_name=%s, result=%d, node_list=%s, action=%s", */
/* op->call_id, op->optype, op->node_name, op->op_result, */
/* (char *)op->node_list, op->private_data); */
/* filter out old STONITH actions */
CRM_CHECK(decode_transition_key(userdata, &uuid, &transition_id, &stonith_id, NULL),
goto bail);
if (transition_graph->complete || stonith_id < 0 || safe_str_neq(uuid, te_uuid)
|| transition_graph->id != transition_id) {
crm_info("Ignoring STONITH action initiated outside of the current transition");
goto bail;
}
action = get_action(stonith_id, FALSE);
if (action == NULL) {
crm_err("Stonith action not matched");
goto bail;
}
stop_te_timer(action->timer);
if (rc == pcmk_ok) {
const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
const char *uuid = crm_element_value(action->xml, XML_LRM_ATTR_TARGET_UUID);
const char *op = crm_meta_value(action->params, "stonith_action");
crm_info("Stonith operation %d for %s passed", call_id, target);
if (action->confirmed == FALSE) {
te_action_confirmed(action);
if (safe_str_eq("on", op)) {
const char *value = NULL;
char *now = crm_itoa(time(NULL));
update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE);
free(now);
value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL);
update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE);
value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE);
update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE);
} else if (action->sent_update == FALSE) {
send_stonith_update(action, target, uuid);
action->sent_update = TRUE;
}
}
st_fail_count_reset(target);
} else {
const char *target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
enum transition_action abort_action = tg_restart;
action->failed = TRUE;
crm_notice("Stonith operation %d for %s failed (%s): aborting transition.",
call_id, target, pcmk_strerror(rc));
/* If no fence devices were available, there's no use in immediately
* checking again, so don't start a new transition in that case.
*/
if (rc == -ENODEV) {
crm_warn("No devices found in cluster to fence %s, giving up",
target);
abort_action = tg_stop;
}
/* Increment the fail count now, so abort_for_stonith_failure() can
* check it. Non-DC nodes will increment it in tengine_stonith_notify().
*/
st_fail_count_increment(target);
abort_for_stonith_failure(abort_action, target, NULL);
}
update_graph(transition_graph, action);
trigger_graph();
bail:
free(userdata);
free(uuid);
return;
}
/*
* returns the ID of the action if a match is found
* returns -1 if a match was not found
* returns -2 if a match was found but the action failed (and was
* not allowed to)
*/
int
match_graph_event(int action_id, xmlNode * event, const char *event_node,
int op_status, int op_rc, int target_rc)
{
const char *target = NULL;
const char *allow_fail = NULL;
const char *this_event = NULL;
crm_action_t *action = NULL;
action = get_action(action_id, FALSE);
if (action == NULL) {
return -1;
}
op_status = status_from_rc(action, op_status, op_rc, target_rc);
if (op_status != PCMK_LRM_OP_DONE) {
update_failcount(event, event_node, op_rc, target_rc, FALSE);
}
/* Process OP status */
switch (op_status) {
case PCMK_LRM_OP_PENDING:
crm_debug("Ignoring pending operation");
return action->id;
break;
case PCMK_LRM_OP_DONE:
break;
case PCMK_LRM_OP_ERROR:
case PCMK_LRM_OP_TIMEOUT:
case PCMK_LRM_OP_NOTSUPPORTED:
action->failed = TRUE;
break;
case PCMK_LRM_OP_CANCELLED:
/* do nothing?? */
crm_err("Dont know what to do for cancelled ops yet");
break;
default:
action->failed = TRUE;
crm_err("Unsupported action result: %d", op_status);
}
/* stop this event's timer if it had one */
stop_te_timer(action->timer);
te_action_confirmed(action);
update_graph(transition_graph, action);
trigger_graph();
if (action->failed) {
allow_fail = crm_meta_value(action->params, XML_ATTR_TE_ALLOWFAIL);
if (crm_is_true(allow_fail)) {
action->failed = FALSE;
}
}
if (action->failed) {
abort_transition(action->synapse->priority + 1, tg_restart, "Event failed", event);
}
this_event = crm_element_value(event, XML_LRM_ATTR_TASK_KEY);
target = crm_element_value(action->xml, XML_LRM_ATTR_TARGET);
crm_info("Action %s (%d) confirmed on %s (rc=%d)",
crm_str(this_event), action->id, crm_str(target), op_status);
/* determine if this action affects a remote-node's online/offline status */
process_remote_node_action(action, event);
return action->id;
}
void
te_update_diff(const char *event, xmlNode * msg)
{
int rc = -1;
const char *op = NULL;
xmlNode *diff = NULL;
xmlXPathObject *xpathObj = NULL;
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
CRM_CHECK(msg != NULL, return);
crm_element_value_int(msg, F_CIB_RC, &rc);
if (transition_graph == NULL) {
crm_trace("No graph");
return;
} else if (rc < pcmk_ok) {
crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
return;
} else if (transition_graph->complete == TRUE
&& fsa_state != S_IDLE
&& fsa_state != S_TRANSITION_ENGINE && fsa_state != S_POLICY_ENGINE) {
crm_trace("Filter state=%s, complete=%d", fsa_state2string(fsa_state),
transition_graph->complete);
return;
}
op = crm_element_value(msg, F_CIB_OPERATION);
diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
cib_diff_version_details(diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_debug("Processing diff (%s): %d.%d.%d -> %d.%d.%d (%s)", op,
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates, fsa_state2string(fsa_state));
log_cib_diff(LOG_DEBUG_2, diff, op);
if (cib_config_changed(NULL, NULL, &diff)) {
abort_transition(INFINITY, tg_restart, "Non-status change", diff);
goto bail; /* configuration changed */
}
/* Tickets Attributes - Added/Updated */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Ticket attribute: update", aborted);
goto bail;
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Tickets Attributes - Removed */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Ticket attribute: removal", aborted);
goto bail;
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Transient Attributes - Added/Updated */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
XML_TAG_TRANSIENT_NODEATTRS "//" XML_CIB_TAG_NVPAIR);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
int lpc;
for (lpc = 0; lpc < xpathObj->nodesetval->nodeNr; lpc++) {
xmlNode *attr = getXpathResult(xpathObj, lpc);
const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
const char *value = NULL;
if (safe_str_eq(CRM_OP_PROBED, name)) {
value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
}
if (crm_is_true(value) == FALSE) {
abort_transition(INFINITY, tg_restart, "Transient attribute: update", attr);
crm_log_xml_trace(attr, "Abort");
goto bail;
}
}
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Transient Attributes - Removed */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
XML_TAG_TRANSIENT_NODEATTRS);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
goto bail;
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Check for node state updates... possibly from a shutdown we requested */
xpathObj =
xpath_search(diff, "//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_STATE);
if (xpathObj) {
int lpc = 0, max = xpathObj->nodesetval->nodeNr;
for (lpc = 0; lpc < max; lpc++) {
xmlNode *node = getXpathResult(xpathObj, lpc);
const char *event_node = crm_element_value(node, XML_ATTR_ID);
const char *event_uname = crm_element_value(node, XML_ATTR_UNAME);
const char *is_peer = crm_element_value(node, XML_NODE_IS_PEER);
/* Don't check the value of XML_NODE_IN_CLUSTER, only pacemaker may have been shut down */
if (safe_str_eq(is_peer, XML_BOOLEAN_NO)) {
/* Pacemaker is now stopped/gone
* Was it a shutdown or fencing operation?
*/
crm_action_t *shutdown = match_down_event(0, event_node, NULL);
if (shutdown != NULL) {
const char *task = crm_element_value(shutdown->xml, XML_LRM_ATTR_TASK);
if (safe_str_eq(task, CRM_OP_FENCE)) {
crm_trace("Waiting for stonithd to report the fencing of %s is complete", event_uname); /* via tengine_stonith_callback() */
} else {
crm_debug("%s of %s (op %d) is complete", task, event_uname, shutdown->id);
/* match->confirmed = TRUE; */
stop_te_timer(shutdown->timer);
erase_node_from_join(event_uname);
update_graph(transition_graph, shutdown);
trigger_graph();
}
} else {
crm_info("Stonith/shutdown of %s not matched", event_node);
abort_transition(INFINITY, tg_restart, "Node failure", node);
}
fail_incompletable_actions(transition_graph, event_node);
}
}
xmlXPathFreeObject(xpathObj);
}
/*
* Check for and fast-track the processing of LRM refreshes
* In large clusters this can result in _huge_ speedups
*
* Unfortunately we can only do so when there are no pending actions
* Otherwise we could miss updates we're waiting for and stall
*
*/
xpathObj = NULL;
if (transition_graph->pending == 0) {
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
XML_LRM_TAG_RESOURCE);
}
if (xpathObj) {
int updates = xpathObj->nodesetval->nodeNr;
if (updates > 1) {
/* Updates by, or in response to, TE actions will never contain updates
* for more than one resource at a time
*/
crm_debug("Detected LRM refresh - %d resources updated: Skipping all resource events",
updates);
crm_log_xml_trace(diff, "lrm-refresh");
abort_transition(INFINITY, tg_restart, "LRM Refresh", NULL);
goto bail;
}
xmlXPathFreeObject(xpathObj);
}
/* Process operation updates */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
if (xpathObj) {
process_resource_updates(xpathObj);
xmlXPathFreeObject(xpathObj);
}
/* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
if (xpathObj) {
int lpc = 0, max = xpathObj->nodesetval->nodeNr;
for (lpc = 0; lpc < max; lpc++) {
int max = 0;
const char *op_id = NULL;
char *rsc_op_xpath = NULL;
xmlXPathObject *op_match = NULL;
xmlNode *match = getXpathResult(xpathObj, lpc);
CRM_CHECK(match != NULL, continue);
op_id = ID(match);
max = strlen(rsc_op_template) + strlen(op_id) + 1;
rsc_op_xpath = calloc(1, max);
snprintf(rsc_op_xpath, max, rsc_op_template, op_id);
op_match = xpath_search(diff, rsc_op_xpath);
if (op_match == NULL || op_match->nodesetval->nodeNr == 0) {
/* Prevent false positives by matching cancelations too */
const char *node = get_node_id(match);
crm_action_t *cancelled = get_cancel_action(op_id, node);
if (cancelled == NULL) {
crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
node);
abort_transition(INFINITY, tg_restart, "Resource op removal", match);
if (op_match) {
xmlXPathFreeObject(op_match);
}
free(rsc_op_xpath);
goto bail;
} else {
crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
op_id, node, cancelled->id);
}
}
if (op_match) {
xmlXPathFreeObject(op_match);
}
free(rsc_op_xpath);
}
}
bail:
if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
}
void
te_update_diff(const char *event, xmlNode *msg)
{
int rc = -1;
const char *op = NULL;
xmlNode *diff = NULL;
xmlNode *cib_top = NULL;
xmlXPathObject *xpathObj = NULL;
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
CRM_CHECK(msg != NULL, return);
crm_element_value_int(msg, F_CIB_RC, &rc);
if(transition_graph == NULL) {
crm_debug_3("No graph");
return;
} else if(rc < cib_ok) {
crm_debug_3("Filter rc=%d (%s)", rc, cib_error2string(rc));
return;
} else if(transition_graph->complete == TRUE
&& fsa_state != S_IDLE
&& fsa_state != S_TRANSITION_ENGINE
&& fsa_state != S_POLICY_ENGINE) {
crm_debug_2("Filter state=%s, complete=%d", fsa_state2string(fsa_state), transition_graph->complete);
return;
}
op = crm_element_value(msg, F_CIB_OPERATION);
diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
cib_diff_version_details(
diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_debug("Processing diff (%s): %d.%d.%d -> %d.%d.%d (%s)", op,
diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
diff_add_admin_epoch,diff_add_epoch,diff_add_updates,
fsa_state2string(fsa_state));
log_cib_diff(LOG_DEBUG_2, diff, op);
/* Process anything that was added */
cib_top = get_xpath_object("//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB, diff, LOG_ERR);
if(need_abort(cib_top)) {
goto bail; /* configuration changed */
}
/* Process anything that was removed */
cib_top = get_xpath_object("//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_REMOVED"//"XML_TAG_CIB, diff, LOG_ERR);
if(need_abort(cib_top)) {
goto bail; /* configuration changed */
}
/* Transient Attributes - Added/Updated */
xpathObj = xpath_search(diff,"//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_TAG_TRANSIENT_NODEATTRS"//"XML_CIB_TAG_NVPAIR);
if(xpathObj && xpathObj->nodesetval->nodeNr > 0) {
int lpc;
for(lpc = 0; lpc < xpathObj->nodesetval->nodeNr; lpc++) {
xmlNode *attr = getXpathResult(xpathObj, lpc);
const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
const char *value = NULL;
if(safe_str_eq(CRM_OP_PROBED, name)) {
value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
}
if(crm_is_true(value) == FALSE) {
abort_transition(INFINITY, tg_restart, "Transient attribute: update", attr);
crm_log_xml_debug_2(attr, "Abort");
goto bail;
}
}
} else if(xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Transient Attributes - Removed */
xpathObj = xpath_search(diff,"//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_REMOVED"//"XML_TAG_TRANSIENT_NODEATTRS);
if(xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
goto bail;
} else if(xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Check for node state updates... possibly from a shutdown we requested */
xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_CIB_TAG_STATE);
if(xpathObj) {
int lpc = 0, max = xpathObj->nodesetval->nodeNr;
for(lpc = 0; lpc < max; lpc++) {
xmlNode *node = getXpathResult(xpathObj, lpc);
const char *event_node = crm_element_value(node, XML_ATTR_ID);
const char *ccm_state = crm_element_value(node, XML_CIB_ATTR_INCCM);
const char *ha_state = crm_element_value(node, XML_CIB_ATTR_HASTATE);
const char *shutdown_s = crm_element_value(node, XML_CIB_ATTR_SHUTDOWN);
const char *crmd_state = crm_element_value(node, XML_CIB_ATTR_CRMDSTATE);
if(safe_str_eq(ccm_state, XML_BOOLEAN_FALSE)
|| safe_str_eq(ha_state, DEADSTATUS)
|| safe_str_eq(crmd_state, CRMD_JOINSTATE_DOWN)) {
crm_action_t *shutdown = match_down_event(0, event_node, NULL);
if(shutdown != NULL) {
const char *task = crm_element_value(shutdown->xml, XML_LRM_ATTR_TASK);
if(safe_str_neq(task, CRM_OP_FENCE)) {
/* Wait for stonithd to tell us it is complete via tengine_stonith_callback() */
update_graph(transition_graph, shutdown);
trigger_graph();
}
} else {
crm_info("Stonith/shutdown of %s not matched", event_node);
abort_transition(INFINITY, tg_restart, "Node failure", node);
}
fail_incompletable_actions(transition_graph, event_node);
}
if(shutdown_s) {
int shutdown = crm_parse_int(shutdown_s, NULL);
if(shutdown > 0) {
crm_info("Aborting on "XML_CIB_ATTR_SHUTDOWN" attribute for %s", event_node);
abort_transition(INFINITY, tg_restart, "Shutdown request", node);
}
}
}
xmlXPathFreeObject(xpathObj);
}
/*
* Check for and fast-track the processing of LRM refreshes
* In large clusters this can result in _huge_ speedups
*
* Unfortunately we can only do so when there are no pending actions
* Otherwise we could miss updates we're waiting for and stall
*
*/
xpathObj = NULL;
if(transition_graph->pending == 0) {
xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_LRM_TAG_RESOURCE);
}
if(xpathObj) {
int updates = xpathObj->nodesetval->nodeNr;
if(updates > 1) {
/* Updates by, or in response to, TE actions will never contain updates
* for more than one resource at a time
*/
crm_info("Detected LRM refresh - %d resources updated: Skipping all resource events", updates);
abort_transition(INFINITY, tg_restart, "LRM Refresh", diff);
goto bail;
}
xmlXPathFreeObject(xpathObj);
}
/* Process operation updates */
xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_LRM_TAG_RSC_OP);
if(xpathObj) {
process_resource_updates(xpathObj);
xmlXPathFreeObject(xpathObj);
}
/* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
xpathObj = xpath_search(diff, "//"XML_TAG_DIFF_REMOVED"//"XML_LRM_TAG_RSC_OP);
if(xpathObj) {
int lpc = 0, max = xpathObj->nodesetval->nodeNr;
for(lpc = 0; lpc < max; lpc++) {
int max = 0;
const char *op_id = NULL;
char *rsc_op_xpath = NULL;
xmlXPathObject *op_match = NULL;
xmlNode *match = getXpathResult(xpathObj, lpc);
CRM_CHECK(match != NULL, continue);
op_id = ID(match);
max = strlen(rsc_op_template) + strlen(op_id) + 1;
crm_malloc0(rsc_op_xpath, max);
snprintf(rsc_op_xpath, max, rsc_op_template, op_id);
op_match = xpath_search(diff, rsc_op_xpath);
if(op_match == NULL || op_match->nodesetval->nodeNr == 0) {
/* Prevent false positives by matching cancelations too */
const char *node = get_node_id(match);
crm_action_t *cancelled = get_cancel_action(op_id, node);
if(cancelled == NULL) {
crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id, node);
abort_transition(INFINITY, tg_restart, "Resource op removal", match);
if(op_match) {
xmlXPathFreeObject(op_match);
}
crm_free(rsc_op_xpath);
goto bail;
} else {
crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
op_id, node, cancelled->id);
}
}
if(op_match) {
xmlXPathFreeObject(op_match);
}
crm_free(rsc_op_xpath);
}
}
bail:
if(xpathObj) {
xmlXPathFreeObject(xpathObj);
}
}
void
peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
uint32_t old = 0;
uint32_t changed = 0;
bool appeared = FALSE;
bool is_remote = is_set(node->flags, crm_remote_node);
const char *status = NULL;
/* Crmd waits to receive some information from the membership layer before
* declaring itself operational. If this is being called for a cluster node,
* indicate that we have it.
*/
if (!is_remote) {
set_bit(fsa_input_register, R_PEER_DATA);
}
if (node->uname == NULL) {
return;
}
switch (type) {
case crm_status_uname:
/* If we've never seen the node, then it also won't be in the status section */
crm_info("%s node %s is now %s",
(is_remote? "Remote" : "Cluster"),
node->uname, state_text(node->state));
return;
case crm_status_rstate:
case crm_status_nstate:
/* This callback should not be called unless the state actually
* changed, but here's a failsafe just in case.
*/
CRM_CHECK(safe_str_neq(data, node->state), return);
crm_info("%s node %s is now %s (was %s)",
(is_remote? "Remote" : "Cluster"),
node->uname, state_text(node->state), state_text(data));
if (safe_str_eq(CRM_NODE_MEMBER, node->state)) {
appeared = TRUE;
if (!is_remote) {
remove_stonith_cleanup(node->uname);
}
}
crmd_alert_node_event(node);
break;
case crm_status_processes:
if (data) {
old = *(const uint32_t *)data;
changed = node->processes ^ old;
}
status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
node->uname, peer2text(proc_flags), status,
AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);
if ((changed & proc_flags) == 0) {
/* Peer process did not change */
crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
return;
} else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
crm_trace("Not connected");
return;
} else if (fsa_state == S_STOPPING) {
crm_trace("Stopping");
return;
}
appeared = (node->processes & proc_flags) != 0;
if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
/* Did we get evicted? */
crm_notice("Our peer connection failed");
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
} else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
/* Did the DC leave us? */
crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
/* @COMPAT DC < 1.1.13: If a DC shuts down normally, we don't
* want to fence it. Newer DCs will send their shutdown request
* to all peers, who will update the DC's expected state to
* down, thus avoiding fencing. We can safely erase the DC's
* transient attributes when it leaves in that case. However,
* the only way to avoid fencing older DCs is to leave the
* transient attributes intact until it rejoins.
*/
if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
}
} else if(AM_I_DC && appeared == FALSE) {
crm_info("Peer %s left us", node->uname);
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
}
break;
}
if (AM_I_DC) {
xmlNode *update = NULL;
int flags = node_update_peer;
gboolean alive = is_remote? appeared : crm_is_peer_active(node);
crm_action_t *down = match_down_event(node->uuid, appeared);
crm_trace("Alive=%d, appeared=%d, down=%d",
alive, appeared, (down? down->id : -1));
if (alive && type == crm_status_processes) {
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
if (down) {
const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
if (safe_str_eq(task, CRM_OP_FENCE)) {
/* tengine_stonith_callback() confirms fence actions */
crm_trace("Updating CIB %s stonithd reported fencing of %s complete",
(down->confirmed? "after" : "before"), node->uname);
} else if ((alive == FALSE) && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
crm_notice("%s of peer %s is complete "CRM_XS" op=%d",
task, node->uname, down->id);
/* down->confirmed = TRUE; */
stop_te_timer(down->timer);
if (!is_remote) {
flags |= node_update_join | node_update_expected;
crmd_peer_down(node, FALSE);
check_join_state(fsa_state, __FUNCTION__);
}
update_graph(transition_graph, down);
trigger_graph();
} else {
crm_trace("Node %s is %salive, was expected to %s (op %d)",
node->uname, (alive? "" : "not "), task, down->id);
}
} else if (appeared == FALSE) {
crm_notice("Stonith/shutdown of %s not matched", node->uname);
if (!is_remote) {
crm_update_peer_join(__FUNCTION__, node, crm_join_none);
check_join_state(fsa_state, __FUNCTION__);
}
abort_transition(INFINITY, tg_restart, "Node failure", NULL);
fail_incompletable_actions(transition_graph, node->uuid);
} else {
crm_trace("Node %s came up, was not expected to be down",
node->uname);
}
if (is_remote) {
/* A pacemaker_remote node won't have its cluster status updated
* in the CIB by membership-layer callbacks, so do it here.
*/
flags |= node_update_cluster;
/* Trigger resource placement on newly integrated nodes */
if (appeared) {
abort_transition(INFINITY, tg_restart,
"pacemaker_remote node integrated", NULL);
}
}
/* Update the CIB node state */
update = create_node_state_update(node, flags, NULL, __FUNCTION__);
fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
cib_scope_local | cib_quorum_override | cib_can_create);
free_xml(update);
}
trigger_fsa(fsa_source);
}
void
send_rsc_command(crm_action_t *action)
{
HA_Message *cmd = NULL;
crm_data_t *rsc_op = NULL;
char *counter = NULL;
const char *task = NULL;
const char *value = NULL;
const char *on_node = NULL;
const char *task_uuid = NULL;
CRM_ASSERT(action != NULL);
CRM_ASSERT(action->xml != NULL);
rsc_op = action->xml;
task = crm_element_value(rsc_op, XML_LRM_ATTR_TASK);
task_uuid = crm_element_value(action->xml, XML_LRM_ATTR_TASK_KEY);
on_node = crm_element_value(rsc_op, XML_LRM_ATTR_TARGET);
counter = generate_transition_key(
transition_graph->id, action->id, te_uuid);
crm_xml_add(rsc_op, XML_ATTR_TRANSITION_KEY, counter);
crm_info("Initiating action %d: %s on %s",
action->id, task_uuid, on_node);
crm_free(counter);
if(rsc_op != NULL) {
crm_log_xml_debug_2(rsc_op, "Performing");
}
cmd = create_request(CRM_OP_INVOKE_LRM, rsc_op, on_node,
CRM_SYSTEM_LRMD, CRM_SYSTEM_TENGINE, NULL);
#if 1
send_ipc_message(crm_ch, cmd);
#else
/* test the TE timer/recovery code */
if((action->id % 11) == 0) {
crm_err("Faking lost action %d: %s", action->id, task_uuid);
} else {
send_ipc_message(crm_ch, cmd);
}
#endif
crm_msg_del(cmd);
action->executed = TRUE;
value = g_hash_table_lookup(action->params, crm_meta_name(XML_ATTR_TE_NOWAIT));
if(crm_is_true(value)) {
crm_debug("Skipping wait for %d", action->id);
action->confirmed = TRUE;
update_graph(transition_graph, action);
trigger_graph();
} else if(action->timeout > 0) {
int action_timeout = (2 * action->timeout) + transition_graph->network_delay;
crm_debug_3("Setting timer for action %s", task_uuid);
if(transition_graph->transition_timeout < action_timeout) {
crm_debug("Action %d:"
" Increasing transition %d timeout to %d (2*%d + %d)",
action->id, transition_graph->id, action_timeout,
action->timeout, transition_graph->network_delay);
transition_graph->transition_timeout = action_timeout;
}
te_start_action_timer(action);
}
}
void simulation_newgeneration()
{
int* neighbours;
int preys[8], predators[8], empty[8], rand_array[MAP_WIDTH*MAP_HEIGHT], prey_count, pred_count, empty_count, random, i, p;
prey_count = pred_count = empty_count = random = i = p = 0;
// array of indicies !!!!!!!!
for(i = 0; i < MAP_WIDTH*MAP_HEIGHT; i++)
{
rand_array[i] = i;
}
for(i = 0; i < MAP_WIDTH*MAP_HEIGHT; i++)
{
int randz = rand() % (MAP_WIDTH * MAP_HEIGHT);
rand_array[i] = rand_array[randz]; rand_array[randz] = i;
}
for(i = 0; i < MAP_WIDTH * MAP_HEIGHT; i++)
{
if(cells[rand_array[i]] != NULL)
{
neighbours = simulation_check(rand_array[i]);
for(p = 0; p < 7; p++)
{
if(cells[neighbours[p]] != NULL)
{
if(cells[neighbours[p]]->Type == PREY)
{
preys[prey_count] = neighbours[p];
prey_count++;
}
else if(cells[neighbours[p]]->Type == PREDATOR)
{
predators[pred_count] = neighbours[p];
pred_count++;
}
}
else if(cells[neighbours[p]] == NULL)
{
empty[empty_count] = neighbours[p];
empty_count++;
}
}
switch(cells[rand_array[i]]->Type)
{
case PREDATOR:
if(cells[rand_array[i]]->Organism.predator.starve < 0)
{
free(cells[rand_array[i]]);
cells[rand_array[i]] = NULL;
simulation.total_pred--;
}
else
{
if(prey_count > 0)
{
random = rand() % (prey_count);
free(cells[preys[random]]);
cells[preys[random]] = NULL;
cells[preys[random]] = Cell_create_pred((cells[rand_array[i]]->Organism.predator.breed - 1), simulation.g_pred_starve);
// That cell becomes this predator Starve reset to 0, if multiple of breed, leave offspring
simulation.total_prey--;
if(cells[rand_array[i]]->Organism.predator.breed >= 0)
{
free(cells[rand_array[i]]); // Free / Nullify old cel
cells[rand_array[i]] = NULL;
}
else
{
cells[rand_array[i]]->Organism.predator.breed = simulation.g_pred_breed;
cells[rand_array[i]]->Organism.predator.starve = simulation.g_pred_starve;
simulation.total_pred++;
}
}
else if(empty_count > 0) // No prey nearby
{
random = rand() % (empty_count);
cells[empty[random]] = Cell_create_pred((cells[rand_array[i]]->Organism.predator.breed - 1), (cells[rand_array[i]]->Organism.predator.starve - 1)); // Predator moves to empty cell
if(cells[rand_array[i]]->Organism.predator.breed >= 0)
{
free(cells[rand_array[i]]);
cells[rand_array[i]] = NULL;
}
else
{
cells[rand_array[i]]->Organism.predator.breed = simulation.g_pred_breed;
simulation.total_pred++;
}
}
else if(empty_count == 0) // Surrounded by predators
{
// can't move
cells[rand_array[i]]->Organism.predator.starve -= 1;
}
}
break;
case PREY:
if(empty_count > 0) // empty area near by!
{
random = rand() % (empty_count);
cells[empty[random]] = Cell_create_prey((cells[rand_array[i]]->Organism.prey.breed - 1)); // Prey moves to empty cell
if(cells[rand_array[i]]->Organism.prey.breed >= 0)
{
free(cells[rand_array[i]]);
cells[rand_array[i]] = NULL;
}
else
{
cells[rand_array[i]]->Organism.prey.breed = simulation.g_prey_breed;
simulation.total_prey++;
}
}
break;
}
free(neighbours); prey_count = pred_count = empty_count = random = 0;
int k = 0;
for(k = 0; k < 7; k++)
{
preys[k] = 0;
predators[k] = 0;
empty[k] = 0;
}
}
}
update_graph();
}
