void
create_node_entry(const char *uuid, const char *uname, const char *type)
{
/* make sure a node entry exists for the new node
*
* this will add anyone except the first ever node in the cluster
* since it will also be the DC which doesnt go through the
* join process (with itself). We can include a special case
* later if desired.
*/
xmlNode *tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
crm_trace("Creating node entry for %s", uname);
set_uuid(tmp1, XML_ATTR_UUID, uname);
crm_xml_add(tmp1, XML_ATTR_UNAME, uname);
crm_xml_add(tmp1, XML_ATTR_TYPE, type);
fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1,
cib_scope_local | cib_quorum_override | cib_can_create);
free_xml(tmp1);
}
void
finalize_join_for(gpointer key, gpointer value, gpointer user_data)
{
xmlNode *acknak = NULL;
xmlNode *tmp1 = NULL;
crm_node_t *join_node = value;
const char *join_to = join_node->uname;
if(join_node->join != crm_join_integrated) {
crm_trace("Skipping %s in state %d", join_to, join_node->join);
return;
}
/* make sure a node entry exists for the new node */
crm_trace("Creating node entry for %s", join_to);
tmp1 = create_xml_node(NULL, XML_CIB_TAG_NODE);
set_uuid(tmp1, XML_ATTR_UUID, join_node);
crm_xml_add(tmp1, XML_ATTR_UNAME, join_to);
fsa_cib_anon_update(XML_CIB_TAG_NODES, tmp1,
cib_scope_local | cib_quorum_override | cib_can_create);
free_xml(tmp1);
join_node = crm_get_peer(0, join_to);
if (crm_is_peer_active(join_node) == FALSE) {
/*
* NACK'ing nodes that the membership layer doesn't know about yet
* simply creates more churn
*
* Better to leave them waiting and let the join restart when
* the new membership event comes in
*
* All other NACKs (due to versions etc) should still be processed
*/
crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_PENDING);
return;
}
/* send the ack/nack to the node */
acknak = create_request(CRM_OP_JOIN_ACKNAK, NULL, join_to,
CRM_SYSTEM_CRMD, CRM_SYSTEM_DC, NULL);
crm_xml_add_int(acknak, F_CRM_JOIN_ID, current_join_id);
crm_debug("join-%d: ACK'ing join request from %s",
current_join_id, join_to);
crm_xml_add(acknak, CRM_OP_JOIN_ACKNAK, XML_BOOLEAN_TRUE);
crm_update_peer_join(__FUNCTION__, join_node, crm_join_finalized);
crm_update_peer_expected(__FUNCTION__, join_node, CRMD_JOINSTATE_MEMBER);
send_cluster_message(crm_get_peer(0, join_to), crm_msg_crmd, acknak, TRUE);
free_xml(acknak);
return;
}
enum crmd_fsa_input
handle_shutdown_request(xmlNode * stored_msg)
{
/* handle here to avoid potential version issues
* where the shutdown message/proceedure may have
* been changed in later versions.
*
* This way the DC is always in control of the shutdown
*/
char *now_s = NULL;
time_t now = time(NULL);
xmlNode *node_state = NULL;
const char *host_from = crm_element_value(stored_msg, F_CRM_HOST_FROM);
if (host_from == NULL) {
/* we're shutting down and the DC */
host_from = fsa_our_uname;
}
crm_info("Creating shutdown request for %s (state=%s)", host_from, fsa_state2string(fsa_state));
crm_log_xml(LOG_MSG, "message", stored_msg);
node_state = create_node_state(host_from, NULL, NULL, NULL, NULL,
CRMD_STATE_INACTIVE, FALSE, __FUNCTION__);
fsa_cib_anon_update(XML_CIB_TAG_STATUS, node_state, cib_quorum_override);
crm_log_xml_debug_2(node_state, "Shutdown update");
free_xml(node_state);
now_s = crm_itoa(now);
update_attrd(host_from, XML_CIB_ATTR_SHUTDOWN, now_s, NULL);
crm_free(now_s);
/* will be picked up by the TE as long as its running */
return I_NULL;
}
void
peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
uint32_t old = 0;
uint32_t changed = 0;
bool appeared = FALSE;
bool is_remote = is_set(node->flags, crm_remote_node);
const char *status = NULL;
/* Crmd waits to receive some information from the membership layer before
* declaring itself operational. If this is being called for a cluster node,
* indicate that we have it.
*/
if (!is_remote) {
set_bit(fsa_input_register, R_PEER_DATA);
}
if (node->uname == NULL) {
return;
}
switch (type) {
case crm_status_uname:
/* If we've never seen the node, then it also won't be in the status section */
crm_info("%s node %s is now %s",
(is_remote? "Remote" : "Cluster"),
node->uname, state_text(node->state));
return;
case crm_status_rstate:
case crm_status_nstate:
/* This callback should not be called unless the state actually
* changed, but here's a failsafe just in case.
*/
CRM_CHECK(safe_str_neq(data, node->state), return);
crm_info("%s node %s is now %s (was %s)",
(is_remote? "Remote" : "Cluster"),
node->uname, state_text(node->state), state_text(data));
if (safe_str_eq(CRM_NODE_MEMBER, node->state)) {
appeared = TRUE;
if (!is_remote) {
remove_stonith_cleanup(node->uname);
}
}
crmd_alert_node_event(node);
break;
case crm_status_processes:
if (data) {
old = *(const uint32_t *)data;
changed = node->processes ^ old;
}
status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
node->uname, peer2text(proc_flags), status,
AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);
if ((changed & proc_flags) == 0) {
/* Peer process did not change */
crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
return;
} else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
crm_trace("Not connected");
return;
} else if (fsa_state == S_STOPPING) {
crm_trace("Stopping");
return;
}
appeared = (node->processes & proc_flags) != 0;
if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
/* Did we get evicted? */
crm_notice("Our peer connection failed");
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
} else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
/* Did the DC leave us? */
crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
/* @COMPAT DC < 1.1.13: If a DC shuts down normally, we don't
* want to fence it. Newer DCs will send their shutdown request
* to all peers, who will update the DC's expected state to
* down, thus avoiding fencing. We can safely erase the DC's
* transient attributes when it leaves in that case. However,
* the only way to avoid fencing older DCs is to leave the
* transient attributes intact until it rejoins.
*/
if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
}
} else if(AM_I_DC && appeared == FALSE) {
crm_info("Peer %s left us", node->uname);
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
}
break;
}
if (AM_I_DC) {
xmlNode *update = NULL;
int flags = node_update_peer;
gboolean alive = is_remote? appeared : crm_is_peer_active(node);
crm_action_t *down = match_down_event(node->uuid, appeared);
crm_trace("Alive=%d, appeared=%d, down=%d",
alive, appeared, (down? down->id : -1));
if (alive && type == crm_status_processes) {
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
if (down) {
const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
if (safe_str_eq(task, CRM_OP_FENCE)) {
/* tengine_stonith_callback() confirms fence actions */
crm_trace("Updating CIB %s stonithd reported fencing of %s complete",
(down->confirmed? "after" : "before"), node->uname);
} else if ((alive == FALSE) && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
crm_notice("%s of peer %s is complete "CRM_XS" op=%d",
task, node->uname, down->id);
/* down->confirmed = TRUE; */
stop_te_timer(down->timer);
if (!is_remote) {
flags |= node_update_join | node_update_expected;
crmd_peer_down(node, FALSE);
check_join_state(fsa_state, __FUNCTION__);
}
update_graph(transition_graph, down);
trigger_graph();
} else {
crm_trace("Node %s is %salive, was expected to %s (op %d)",
node->uname, (alive? "" : "not "), task, down->id);
}
} else if (appeared == FALSE) {
crm_notice("Stonith/shutdown of %s not matched", node->uname);
if (!is_remote) {
crm_update_peer_join(__FUNCTION__, node, crm_join_none);
check_join_state(fsa_state, __FUNCTION__);
}
abort_transition(INFINITY, tg_restart, "Node failure", NULL);
fail_incompletable_actions(transition_graph, node->uuid);
} else {
crm_trace("Node %s came up, was not expected to be down",
node->uname);
}
if (is_remote) {
/* A pacemaker_remote node won't have its cluster status updated
* in the CIB by membership-layer callbacks, so do it here.
*/
flags |= node_update_cluster;
/* Trigger resource placement on newly integrated nodes */
if (appeared) {
abort_transition(INFINITY, tg_restart,
"pacemaker_remote node integrated", NULL);
}
}
/* Update the CIB node state */
update = create_node_state_update(node, flags, NULL, __FUNCTION__);
fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
cib_scope_local | cib_quorum_override | cib_can_create);
free_xml(update);
}
trigger_fsa(fsa_source);
}
void
peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
uint32_t old = 0;
uint32_t changed = 0;
bool appeared = FALSE;
const char *status = NULL;
set_bit(fsa_input_register, R_PEER_DATA);
if (node->uname == NULL) {
return;
}
switch (type) {
case crm_status_uname:
/* If we've never seen the node, then it also wont be in the status section */
crm_info("%s is now %s", node->uname, node->state);
return;
case crm_status_rstate:
crm_info("Remote node %s is now %s (was %s)", node->uname, node->state, (const char *)data);
/* Keep going */
case crm_status_nstate:
crm_info("%s is now %s (was %s)", node->uname, node->state, (const char *)data);
if (safe_str_eq(data, node->state)) {
/* State did not change */
return;
} else if(safe_str_eq(CRM_NODE_MEMBER, node->state)) {
appeared = TRUE;
}
break;
case crm_status_processes:
if (data) {
old = *(const uint32_t *)data;
changed = node->processes ^ old;
}
/* crmd_proc_update(node, proc_flags); */
status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
node->uname, peer2text(proc_flags), status,
AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);
if ((changed & proc_flags) == 0) {
/* Peer process did not change */
crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
return;
} else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
crm_trace("Not connected");
return;
} else if (fsa_state == S_STOPPING) {
crm_trace("Stopping");
return;
}
appeared = (node->processes & proc_flags) != 0;
if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
/* Did we get evicted? */
crm_notice("Our peer connection failed");
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);
} else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
/* Did the DC leave us? */
crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);
} else if(AM_I_DC && appeared == FALSE) {
crm_info("Peer %s left us", node->uname);
/* crm_update_peer_join(__FUNCTION__, node, crm_join_none); */
}
break;
}
if (AM_I_DC) {
xmlNode *update = NULL;
int flags = node_update_peer;
gboolean alive = crm_is_peer_active(node);
crm_action_t *down = match_down_event(0, node->uuid, NULL, appeared);
crm_trace("Alive=%d, appear=%d, down=%p", alive, appeared, down);
if (alive && type == crm_status_processes) {
register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
}
if (down) {
const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);
if (alive && safe_str_eq(task, CRM_OP_FENCE)) {
crm_info("Node return implies stonith of %s (action %d) completed", node->uname,
down->id);
erase_status_tag(node->uname, XML_CIB_TAG_LRM, cib_scope_local);
erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
/* down->confirmed = TRUE; Only stonith-ng returning should imply completion */
down->sent_update = TRUE; /* Prevent tengine_stonith_callback() from calling send_stonith_update() */
} else if (safe_str_eq(task, CRM_OP_FENCE)) {
crm_trace("Waiting for stonithd to report the fencing of %s is complete", node->uname); /* via tengine_stonith_callback() */
} else if (alive == FALSE) {
crm_notice("%s of %s (op %d) is complete", task, node->uname, down->id);
/* down->confirmed = TRUE; Only stonith-ng returning should imply completion */
stop_te_timer(down->timer);
flags |= node_update_join | node_update_expected;
crmd_peer_down(node, FALSE);
check_join_state(fsa_state, __FUNCTION__);
update_graph(transition_graph, down);
trigger_graph();
} else {
crm_trace("Other %p", down);
}
} else if (appeared == FALSE) {
crm_notice("Stonith/shutdown of %s not matched", node->uname);
crm_update_peer_join(__FUNCTION__, node, crm_join_none);
check_join_state(fsa_state, __FUNCTION__);
abort_transition(INFINITY, tg_restart, "Node failure", NULL);
fail_incompletable_actions(transition_graph, node->uuid);
} else {
crm_trace("Other %p", down);
}
update = do_update_node_cib(node, flags, NULL, __FUNCTION__);
fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
cib_scope_local | cib_quorum_override | cib_can_create);
free_xml(update);
}
trigger_fsa(fsa_source);
}