static gboolean
send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
{
CRM_ASSERT(msg != NULL);
if (broadcast) {
/* this (successful) call modified the CIB _and_ the
* change needs to be broadcast...
* send via HA to other nodes
*/
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
char *digest = NULL;
cib_diff_version_details(result_diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d",
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates);
crm_xml_add(msg, F_CIB_ISREPLY, originator);
crm_xml_add(msg, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);
crm_xml_add(msg, F_CIB_OPERATION, CIB_OP_APPLY_DIFF);
/* Its safe to always use the latest version since the election
* ensures the software on this node is the oldest node in the cluster
*/
digest = calculate_xml_versioned_digest(the_cib, FALSE, TRUE, CRM_FEATURE_SET);
crm_xml_add(result_diff, XML_ATTR_DIGEST, digest);
crm_log_xml_trace(the_cib, digest);
free(digest);
add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
crm_log_xml_trace(msg, "copy");
return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
} else if (originator != NULL) {
/* send reply via HA to originating node */
crm_trace("Sending request result to originator only");
crm_xml_add(msg, F_CIB_ISREPLY, originator);
return send_cluster_message(originator, crm_msg_cib, msg, FALSE);
}
return FALSE;
}
static gboolean
send_peer_reply(xmlNode * msg, xmlNode * result_diff, const char *originator, gboolean broadcast)
{
CRM_ASSERT(msg != NULL);
if (broadcast) {
/* this (successful) call modified the CIB _and_ the
* change needs to be broadcast...
* send via HA to other nodes
*/
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
const char *digest = NULL;
CRM_LOG_ASSERT(result_diff != NULL);
digest = crm_element_value(result_diff, XML_ATTR_DIGEST);
cib_diff_version_details(result_diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_trace("Sending update diff %d.%d.%d -> %d.%d.%d %s",
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates, digest);
crm_xml_add(msg, F_CIB_ISREPLY, originator);
crm_xml_add(msg, F_CIB_GLOBAL_UPDATE, XML_BOOLEAN_TRUE);
crm_xml_add(msg, F_CIB_OPERATION, CIB_OP_APPLY_DIFF);
CRM_ASSERT(digest != NULL);
add_message_xml(msg, F_CIB_UPDATE_DIFF, result_diff);
crm_log_xml_explicit(msg, "copy");
return send_cluster_message(NULL, crm_msg_cib, msg, TRUE);
} else if (originator != NULL) {
/* send reply via HA to originating node */
crm_trace("Sending request result to %s only", originator);
crm_xml_add(msg, F_CIB_ISREPLY, originator);
return send_cluster_message(crm_get_peer(0, originator), crm_msg_cib, msg, FALSE);
}
return FALSE;
}
void
cib_replace_notify(crm_data_t *update, enum cib_errors result, crm_data_t *diff)
{
const char *origin = NULL;
HA_Message *replace_msg = NULL;
int add_updates = 0;
int add_epoch = 0;
int add_admin_epoch = 0;
int del_updates = 0;
int del_epoch = 0;
int del_admin_epoch = 0;
if(diff == NULL) {
return;
}
cib_diff_version_details(
diff, &add_admin_epoch, &add_epoch, &add_updates,
&del_admin_epoch, &del_epoch, &del_updates);
origin = crm_element_value(update, F_CRM_ORIGIN);
if(add_updates != del_updates) {
crm_info("Replaced: %d.%d.%d -> %d.%d.%d from %s",
del_admin_epoch, del_epoch, del_updates,
add_admin_epoch, add_epoch, add_updates,
crm_str(origin));
} else if(diff != NULL) {
crm_info("Local-only Replace: %d.%d.%d from %s",
add_admin_epoch, add_epoch, add_updates,
crm_str(origin));
}
replace_msg = ha_msg_new(8);
ha_msg_add(replace_msg, F_TYPE, T_CIB_NOTIFY);
ha_msg_add(replace_msg, F_SUBTYPE, T_CIB_REPLACE_NOTIFY);
ha_msg_add(replace_msg, F_CIB_OPERATION, CIB_OP_REPLACE);
ha_msg_add_int(replace_msg, F_CIB_RC, result);
attach_cib_generation(replace_msg, "cib-replace-generation", update);
crm_log_message_adv(LOG_DEBUG_2,"CIB Replaced", replace_msg);
g_hash_table_foreach(client_list, cib_notify_client, replace_msg);
crm_msg_del(replace_msg);
}
void
cib_diff_notify(
int options, const char *client, const char *call_id, const char *op,
crm_data_t *update, enum cib_errors result, crm_data_t *diff)
{
int add_updates = 0;
int add_epoch = 0;
int add_admin_epoch = 0;
int del_updates = 0;
int del_epoch = 0;
int del_admin_epoch = 0;
int log_level = LOG_DEBUG_2;
if(diff == NULL) {
return;
}
if(result != cib_ok) {
log_level = LOG_WARNING;
}
cib_diff_version_details(
diff, &add_admin_epoch, &add_epoch, &add_updates,
&del_admin_epoch, &del_epoch, &del_updates);
if(add_updates != del_updates) {
do_crm_log(log_level,
"Update (client: %s%s%s): %d.%d.%d -> %d.%d.%d (%s)",
client, call_id?", call:":"", call_id?call_id:"",
del_admin_epoch, del_epoch, del_updates,
add_admin_epoch, add_epoch, add_updates,
cib_error2string(result));
} else if(diff != NULL) {
do_crm_log(log_level,
"Local-only Change (client:%s%s%s): %d.%d.%d (%s)",
client, call_id?", call: ":"", call_id?call_id:"",
add_admin_epoch, add_epoch, add_updates,
cib_error2string(result));
}
do_cib_notify(options, op, update, result, diff, T_CIB_DIFF_NOTIFY);
}
void
cib_replace_notify(const char *origin, xmlNode *update, enum cib_errors result, xmlNode *diff)
{
xmlNode *replace_msg = NULL;
int add_updates = 0;
int add_epoch = 0;
int add_admin_epoch = 0;
int del_updates = 0;
int del_epoch = 0;
int del_admin_epoch = 0;
if(diff == NULL) {
return;
}
cib_diff_version_details(
diff, &add_admin_epoch, &add_epoch, &add_updates,
&del_admin_epoch, &del_epoch, &del_updates);
if(add_updates != del_updates) {
crm_info("Replaced: %d.%d.%d -> %d.%d.%d from %s",
del_admin_epoch, del_epoch, del_updates,
add_admin_epoch, add_epoch, add_updates,
crm_str(origin));
} else if(diff != NULL) {
crm_info("Local-only Replace: %d.%d.%d from %s",
add_admin_epoch, add_epoch, add_updates,
crm_str(origin));
}
replace_msg = create_xml_node(NULL, "notify-replace");
crm_xml_add(replace_msg, F_TYPE, T_CIB_NOTIFY);
crm_xml_add(replace_msg, F_SUBTYPE, T_CIB_REPLACE_NOTIFY);
crm_xml_add(replace_msg, F_CIB_OPERATION, CIB_OP_REPLACE);
crm_xml_add_int(replace_msg, F_CIB_RC, result);
attach_cib_generation(replace_msg, "cib-replace-generation", update);
crm_log_xml(LOG_DEBUG_2,"CIB Replaced", replace_msg);
g_hash_table_foreach(client_list, cib_notify_client, replace_msg);
free_xml(replace_msg);
}
void
te_update_diff(const char *event, xmlNode * msg)
{
int rc = -1;
const char *op = NULL;
xmlNode *diff = NULL;
xmlXPathObject *xpathObj = NULL;
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
CRM_CHECK(msg != NULL, return);
crm_element_value_int(msg, F_CIB_RC, &rc);
if (transition_graph == NULL) {
crm_trace("No graph");
return;
} else if (rc < pcmk_ok) {
crm_trace("Filter rc=%d (%s)", rc, pcmk_strerror(rc));
return;
} else if (transition_graph->complete == TRUE
&& fsa_state != S_IDLE
&& fsa_state != S_TRANSITION_ENGINE && fsa_state != S_POLICY_ENGINE) {
crm_trace("Filter state=%s, complete=%d", fsa_state2string(fsa_state),
transition_graph->complete);
return;
}
op = crm_element_value(msg, F_CIB_OPERATION);
diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
cib_diff_version_details(diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_debug("Processing diff (%s): %d.%d.%d -> %d.%d.%d (%s)", op,
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates, fsa_state2string(fsa_state));
log_cib_diff(LOG_DEBUG_2, diff, op);
if (cib_config_changed(NULL, NULL, &diff)) {
abort_transition(INFINITY, tg_restart, "Non-status change", diff);
goto bail; /* configuration changed */
}
/* Tickets Attributes - Added/Updated */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_CIB_TAG_TICKETS);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Ticket attribute: update", aborted);
goto bail;
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Tickets Attributes - Removed */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//" XML_CIB_TAG_TICKETS);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Ticket attribute: removal", aborted);
goto bail;
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Transient Attributes - Added/Updated */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
XML_TAG_TRANSIENT_NODEATTRS "//" XML_CIB_TAG_NVPAIR);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
int lpc;
for (lpc = 0; lpc < xpathObj->nodesetval->nodeNr; lpc++) {
xmlNode *attr = getXpathResult(xpathObj, lpc);
const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
const char *value = NULL;
if (safe_str_eq(CRM_OP_PROBED, name)) {
value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
}
if (crm_is_true(value) == FALSE) {
abort_transition(INFINITY, tg_restart, "Transient attribute: update", attr);
crm_log_xml_trace(attr, "Abort");
goto bail;
}
}
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Transient Attributes - Removed */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_REMOVED "//"
XML_TAG_TRANSIENT_NODEATTRS);
if (xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
goto bail;
} else if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/*
* Check for and fast-track the processing of LRM refreshes
* In large clusters this can result in _huge_ speedups
*
* Unfortunately we can only do so when there are no pending actions
* Otherwise we could miss updates we're waiting for and stall
*
*/
xpathObj = NULL;
if (transition_graph->pending == 0) {
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//"
XML_LRM_TAG_RESOURCE);
}
if (xpathObj) {
int updates = xpathObj->nodesetval->nodeNr;
if (updates > 1) {
/* Updates by, or in response to, TE actions will never contain updates
* for more than one resource at a time
*/
crm_debug("Detected LRM refresh - %d resources updated: Skipping all resource events",
updates);
crm_log_xml_trace(diff, "lrm-refresh");
abort_transition(INFINITY, tg_restart, "LRM Refresh", NULL);
goto bail;
}
xmlXPathFreeObject(xpathObj);
}
/* Process operation updates */
xpathObj =
xpath_search(diff,
"//" F_CIB_UPDATE_RESULT "//" XML_TAG_DIFF_ADDED "//" XML_LRM_TAG_RSC_OP);
if (xpathObj) {
process_resource_updates(xpathObj);
xmlXPathFreeObject(xpathObj);
}
/* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
xpathObj = xpath_search(diff, "//" XML_TAG_DIFF_REMOVED "//" XML_LRM_TAG_RSC_OP);
if (xpathObj) {
int lpc = 0, max = xpathObj->nodesetval->nodeNr;
for (lpc = 0; lpc < max; lpc++) {
int max = 0;
const char *op_id = NULL;
char *rsc_op_xpath = NULL;
xmlXPathObject *op_match = NULL;
xmlNode *match = getXpathResult(xpathObj, lpc);
CRM_CHECK(match != NULL, continue);
op_id = ID(match);
max = strlen(rsc_op_template) + strlen(op_id) + 1;
rsc_op_xpath = calloc(1, max);
snprintf(rsc_op_xpath, max, rsc_op_template, op_id);
op_match = xpath_search(diff, rsc_op_xpath);
if (op_match == NULL || op_match->nodesetval->nodeNr == 0) {
/* Prevent false positives by matching cancelations too */
const char *node = get_node_id(match);
crm_action_t *cancelled = get_cancel_action(op_id, node);
if (cancelled == NULL) {
crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id,
node);
abort_transition(INFINITY, tg_restart, "Resource op removal", match);
if (op_match) {
xmlXPathFreeObject(op_match);
}
free(rsc_op_xpath);
goto bail;
} else {
crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
op_id, node, cancelled->id);
}
}
if (op_match) {
xmlXPathFreeObject(op_match);
}
free(rsc_op_xpath);
}
}
bail:
if (xpathObj) {
xmlXPathFreeObject(xpathObj);
}
}
enum cib_errors
cib_process_diff(const char *op, int options, const char *section, xmlNode * req, xmlNode * input,
xmlNode * existing_cib, xmlNode ** result_cib, xmlNode ** answer)
{
unsigned int log_level = LOG_DEBUG;
const char *reason = NULL;
gboolean apply_diff = TRUE;
enum cib_errors result = cib_ok;
int this_updates = 0;
int this_epoch = 0;
int this_admin_epoch = 0;
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
crm_debug_2("Processing \"%s\" event", op);
cib_diff_version_details(input,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_element_value_int(existing_cib, XML_ATTR_GENERATION, &this_epoch);
crm_element_value_int(existing_cib, XML_ATTR_NUMUPDATES, &this_updates);
crm_element_value_int(existing_cib, XML_ATTR_GENERATION_ADMIN, &this_admin_epoch);
if (this_epoch < 0) {
this_epoch = 0;
}
if (this_updates < 0) {
this_updates = 0;
}
if (this_admin_epoch < 0) {
this_admin_epoch = 0;
}
if (diff_del_admin_epoch == diff_add_admin_epoch
&& diff_del_epoch == diff_add_epoch && diff_del_updates == diff_add_updates) {
if (options & cib_force_diff) {
apply_diff = FALSE;
log_level = LOG_ERR;
reason = "+ and - versions in the diff did not change in global update";
crm_log_xml_warn(input, "Bad global update");
} else if (diff_add_admin_epoch == -1 && diff_add_epoch == -1 && diff_add_updates == -1) {
diff_add_epoch = this_epoch;
diff_add_updates = this_updates + 1;
diff_add_admin_epoch = this_admin_epoch;
diff_del_epoch = this_epoch;
diff_del_updates = this_updates;
diff_del_admin_epoch = this_admin_epoch;
} else {
apply_diff = FALSE;
log_level = LOG_ERR;
reason = "+ and - versions in the diff did not change";
log_cib_diff(LOG_ERR, input, __FUNCTION__);
}
}
if (apply_diff && diff_del_admin_epoch > this_admin_epoch) {
result = cib_diff_resync;
apply_diff = FALSE;
log_level = LOG_INFO;
reason = "current \"" XML_ATTR_GENERATION_ADMIN "\" is less than required";
} else if (apply_diff && diff_del_admin_epoch < this_admin_epoch) {
apply_diff = FALSE;
log_level = LOG_WARNING;
reason = "current \"" XML_ATTR_GENERATION_ADMIN "\" is greater than required";
} else if (apply_diff && diff_del_epoch > this_epoch) {
result = cib_diff_resync;
apply_diff = FALSE;
log_level = LOG_INFO;
reason = "current \"" XML_ATTR_GENERATION "\" is less than required";
} else if (apply_diff && diff_del_epoch < this_epoch) {
apply_diff = FALSE;
log_level = LOG_WARNING;
reason = "current \"" XML_ATTR_GENERATION "\" is greater than required";
} else if (apply_diff && diff_del_updates > this_updates) {
result = cib_diff_resync;
apply_diff = FALSE;
log_level = LOG_INFO;
reason = "current \"" XML_ATTR_NUMUPDATES "\" is less than required";
} else if (apply_diff && diff_del_updates < this_updates) {
apply_diff = FALSE;
log_level = LOG_WARNING;
reason = "current \"" XML_ATTR_NUMUPDATES "\" is greater than required";
}
if (apply_diff) {
free_xml(*result_cib);
*result_cib = NULL;
if (apply_xml_diff(existing_cib, input, result_cib) == FALSE) {
log_level = LOG_NOTICE;
reason = "Failed application of an update diff";
if (options & cib_force_diff) {
result = cib_diff_resync;
}
}
}
if (reason != NULL) {
do_crm_log(log_level,
"Diff %d.%d.%d -> %d.%d.%d not applied to %d.%d.%d: %s",
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates,
this_admin_epoch, this_epoch, this_updates, reason);
crm_log_xml_trace(input, "Discarded diff");
if (result == cib_ok) {
result = cib_diff_failed;
}
} else if (apply_diff) {
crm_debug_2("Diff %d.%d.%d -> %d.%d.%d was applied to %d.%d.%d",
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates,
this_admin_epoch, this_epoch, this_updates);
}
return result;
}
void
abort_transition_graph(int abort_priority, enum transition_action abort_action,
const char *abort_text, xmlNode * reason, const char *fn, int line)
{
int log_level = LOG_INFO;
const char *magic = NULL;
CRM_CHECK(transition_graph != NULL, return);
if (reason) {
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
xmlNode *diff = get_xpath_object("//" F_CIB_UPDATE_RESULT "//diff", reason, LOG_DEBUG_2);
magic = crm_element_value(reason, XML_ATTR_TRANSITION_MAGIC);
if (diff) {
cib_diff_version_details(diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
if (crm_str_eq(TYPE(reason), XML_CIB_TAG_NVPAIR, TRUE)) {
do_crm_log(log_level,
"%s:%d - Triggered transition abort (complete=%d, tag=%s, id=%s, name=%s, value=%s, magic=%s, cib=%d.%d.%d) : %s",
fn, line, transition_graph->complete, TYPE(reason), ID(reason),
NAME(reason), VALUE(reason), magic ? magic : "NA", diff_add_admin_epoch,
diff_add_epoch, diff_add_updates, abort_text);
} else {
do_crm_log(log_level,
"%s:%d - Triggered transition abort (complete=%d, tag=%s, id=%s, magic=%s, cib=%d.%d.%d) : %s",
fn, line, transition_graph->complete, TYPE(reason), ID(reason),
magic ? magic : "NA", diff_add_admin_epoch, diff_add_epoch,
diff_add_updates, abort_text);
}
} else {
do_crm_log(log_level,
"%s:%d - Triggered transition abort (complete=%d, tag=%s, id=%s, magic=%s) : %s",
fn, line, transition_graph->complete, TYPE(reason), ID(reason),
magic ? magic : "NA", abort_text);
}
} else {
do_crm_log(log_level,
"%s:%d - Triggered transition abort (complete=%d) : %s",
fn, line, transition_graph->complete, abort_text);
}
switch (fsa_state) {
case S_STARTING:
case S_PENDING:
case S_NOT_DC:
case S_HALT:
case S_ILLEGAL:
case S_STOPPING:
case S_TERMINATE:
do_crm_log(log_level,
"Abort suppressed: state=%s (complete=%d)",
fsa_state2string(fsa_state), transition_graph->complete);
return;
default:
break;
}
if (magic == NULL && reason != NULL) {
crm_log_xml(log_level + 1, "Cause", reason);
}
/* Make sure any queued calculations are discarded ASAP */
crm_free(fsa_pe_ref);
fsa_pe_ref = NULL;
if (transition_graph->complete) {
if (transition_timer->period_ms > 0) {
crm_timer_start(transition_timer);
} else {
register_fsa_input(C_FSA_INTERNAL, I_PE_CALC, NULL);
}
return;
}
update_abort_priority(transition_graph, abort_priority, abort_action, abort_text);
mainloop_set_trigger(transition_trigger);
}
void
te_update_diff(const char *event, xmlNode *msg)
{
int rc = -1;
const char *op = NULL;
xmlNode *diff = NULL;
xmlNode *cib_top = NULL;
xmlXPathObject *xpathObj = NULL;
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
CRM_CHECK(msg != NULL, return);
crm_element_value_int(msg, F_CIB_RC, &rc);
if(transition_graph == NULL) {
crm_debug_3("No graph");
return;
} else if(rc < cib_ok) {
crm_debug_3("Filter rc=%d (%s)", rc, cib_error2string(rc));
return;
} else if(transition_graph->complete == TRUE
&& fsa_state != S_IDLE
&& fsa_state != S_TRANSITION_ENGINE
&& fsa_state != S_POLICY_ENGINE) {
crm_debug_2("Filter state=%s, complete=%d", fsa_state2string(fsa_state), transition_graph->complete);
return;
}
op = crm_element_value(msg, F_CIB_OPERATION);
diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
cib_diff_version_details(
diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_debug("Processing diff (%s): %d.%d.%d -> %d.%d.%d (%s)", op,
diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
diff_add_admin_epoch,diff_add_epoch,diff_add_updates,
fsa_state2string(fsa_state));
log_cib_diff(LOG_DEBUG_2, diff, op);
/* Process anything that was added */
cib_top = get_xpath_object("//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_TAG_CIB, diff, LOG_ERR);
if(need_abort(cib_top)) {
goto bail; /* configuration changed */
}
/* Process anything that was removed */
cib_top = get_xpath_object("//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_REMOVED"//"XML_TAG_CIB, diff, LOG_ERR);
if(need_abort(cib_top)) {
goto bail; /* configuration changed */
}
/* Transient Attributes - Added/Updated */
xpathObj = xpath_search(diff,"//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_TAG_TRANSIENT_NODEATTRS"//"XML_CIB_TAG_NVPAIR);
if(xpathObj && xpathObj->nodesetval->nodeNr > 0) {
int lpc;
for(lpc = 0; lpc < xpathObj->nodesetval->nodeNr; lpc++) {
xmlNode *attr = getXpathResult(xpathObj, lpc);
const char *name = crm_element_value(attr, XML_NVPAIR_ATTR_NAME);
const char *value = NULL;
if(safe_str_eq(CRM_OP_PROBED, name)) {
value = crm_element_value(attr, XML_NVPAIR_ATTR_VALUE);
}
if(crm_is_true(value) == FALSE) {
abort_transition(INFINITY, tg_restart, "Transient attribute: update", attr);
crm_log_xml_debug_2(attr, "Abort");
goto bail;
}
}
} else if(xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Transient Attributes - Removed */
xpathObj = xpath_search(diff,"//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_REMOVED"//"XML_TAG_TRANSIENT_NODEATTRS);
if(xpathObj && xpathObj->nodesetval->nodeNr > 0) {
xmlNode *aborted = getXpathResult(xpathObj, 0);
abort_transition(INFINITY, tg_restart, "Transient attribute: removal", aborted);
goto bail;
} else if(xpathObj) {
xmlXPathFreeObject(xpathObj);
}
/* Check for node state updates... possibly from a shutdown we requested */
xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_CIB_TAG_STATE);
if(xpathObj) {
int lpc = 0, max = xpathObj->nodesetval->nodeNr;
for(lpc = 0; lpc < max; lpc++) {
xmlNode *node = getXpathResult(xpathObj, lpc);
const char *event_node = crm_element_value(node, XML_ATTR_ID);
const char *ccm_state = crm_element_value(node, XML_CIB_ATTR_INCCM);
const char *ha_state = crm_element_value(node, XML_CIB_ATTR_HASTATE);
const char *shutdown_s = crm_element_value(node, XML_CIB_ATTR_SHUTDOWN);
const char *crmd_state = crm_element_value(node, XML_CIB_ATTR_CRMDSTATE);
if(safe_str_eq(ccm_state, XML_BOOLEAN_FALSE)
|| safe_str_eq(ha_state, DEADSTATUS)
|| safe_str_eq(crmd_state, CRMD_JOINSTATE_DOWN)) {
crm_action_t *shutdown = match_down_event(0, event_node, NULL);
if(shutdown != NULL) {
const char *task = crm_element_value(shutdown->xml, XML_LRM_ATTR_TASK);
if(safe_str_neq(task, CRM_OP_FENCE)) {
/* Wait for stonithd to tell us it is complete via tengine_stonith_callback() */
update_graph(transition_graph, shutdown);
trigger_graph();
}
} else {
crm_info("Stonith/shutdown of %s not matched", event_node);
abort_transition(INFINITY, tg_restart, "Node failure", node);
}
fail_incompletable_actions(transition_graph, event_node);
}
if(shutdown_s) {
int shutdown = crm_parse_int(shutdown_s, NULL);
if(shutdown > 0) {
crm_info("Aborting on "XML_CIB_ATTR_SHUTDOWN" attribute for %s", event_node);
abort_transition(INFINITY, tg_restart, "Shutdown request", node);
}
}
}
xmlXPathFreeObject(xpathObj);
}
/*
* Check for and fast-track the processing of LRM refreshes
* In large clusters this can result in _huge_ speedups
*
* Unfortunately we can only do so when there are no pending actions
* Otherwise we could miss updates we're waiting for and stall
*
*/
xpathObj = NULL;
if(transition_graph->pending == 0) {
xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_LRM_TAG_RESOURCE);
}
if(xpathObj) {
int updates = xpathObj->nodesetval->nodeNr;
if(updates > 1) {
/* Updates by, or in response to, TE actions will never contain updates
* for more than one resource at a time
*/
crm_info("Detected LRM refresh - %d resources updated: Skipping all resource events", updates);
abort_transition(INFINITY, tg_restart, "LRM Refresh", diff);
goto bail;
}
xmlXPathFreeObject(xpathObj);
}
/* Process operation updates */
xpathObj = xpath_search(diff, "//"F_CIB_UPDATE_RESULT"//"XML_TAG_DIFF_ADDED"//"XML_LRM_TAG_RSC_OP);
if(xpathObj) {
process_resource_updates(xpathObj);
xmlXPathFreeObject(xpathObj);
}
/* Detect deleted (as opposed to replaced or added) actions - eg. crm_resource -C */
xpathObj = xpath_search(diff, "//"XML_TAG_DIFF_REMOVED"//"XML_LRM_TAG_RSC_OP);
if(xpathObj) {
int lpc = 0, max = xpathObj->nodesetval->nodeNr;
for(lpc = 0; lpc < max; lpc++) {
int max = 0;
const char *op_id = NULL;
char *rsc_op_xpath = NULL;
xmlXPathObject *op_match = NULL;
xmlNode *match = getXpathResult(xpathObj, lpc);
CRM_CHECK(match != NULL, continue);
op_id = ID(match);
max = strlen(rsc_op_template) + strlen(op_id) + 1;
crm_malloc0(rsc_op_xpath, max);
snprintf(rsc_op_xpath, max, rsc_op_template, op_id);
op_match = xpath_search(diff, rsc_op_xpath);
if(op_match == NULL || op_match->nodesetval->nodeNr == 0) {
/* Prevent false positives by matching cancelations too */
const char *node = get_node_id(match);
crm_action_t *cancelled = get_cancel_action(op_id, node);
if(cancelled == NULL) {
crm_debug("No match for deleted action %s (%s on %s)", rsc_op_xpath, op_id, node);
abort_transition(INFINITY, tg_restart, "Resource op removal", match);
if(op_match) {
xmlXPathFreeObject(op_match);
}
crm_free(rsc_op_xpath);
goto bail;
} else {
crm_debug("Deleted lrm_rsc_op %s on %s was for graph event %d",
op_id, node, cancelled->id);
}
}
if(op_match) {
xmlXPathFreeObject(op_match);
}
crm_free(rsc_op_xpath);
}
}
bail:
if(xpathObj) {
xmlXPathFreeObject(xpathObj);
}
}
int
cib_server_process_diff(const char *op, int options, const char *section, xmlNode * req,
xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
xmlNode ** answer)
{
int rc = pcmk_ok;
if (cib_is_master) {
/* the master is never waiting for a resync */
sync_in_progress = 0;
}
if (sync_in_progress > MAX_DIFF_RETRY) {
/* request another full-sync,
* the last request may have been lost
*/
sync_in_progress = 0;
}
if (sync_in_progress) {
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
cib_diff_version_details(input,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
sync_in_progress++;
crm_notice("Not applying diff %d.%d.%d -> %d.%d.%d (sync in progress)",
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates);
return -pcmk_err_diff_resync;
}
rc = cib_process_diff(op, options, section, req, input, existing_cib, result_cib, answer);
crm_trace("result: %s (%d), %s", pcmk_strerror(rc), rc, cib_is_master?"master":"slave");
if (rc == -pcmk_err_diff_resync && cib_is_master == FALSE) {
free_xml(*result_cib);
*result_cib = NULL;
send_sync_request(NULL);
} else if (rc == -pcmk_err_diff_resync) {
rc = -pcmk_err_diff_failed;
if (options & cib_force_diff) {
crm_warn("Not requesting full refresh in R/W mode");
}
} else if(rc != pcmk_ok && cib_legacy_mode()) {
crm_warn("Something went wrong in compatibility mode, requesting full refresh");
xml_log_patchset(LOG_INFO, __FUNCTION__, input);
free_xml(*result_cib);
*result_cib = NULL;
send_sync_request(NULL);
}
return rc;
}
void
te_update_diff(const char *event, HA_Message *msg)
{
int rc = -1;
const char *op = NULL;
crm_data_t *diff = NULL;
crm_data_t *aborted = NULL;
const char *set_name = NULL;
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
if(msg == NULL) {
crm_err("NULL update");
return;
}
ha_msg_value_int(msg, F_CIB_RC, &rc);
op = cl_get_string(msg, F_CIB_OPERATION);
if(rc < cib_ok) {
crm_debug_2("Ignoring failed %s operation: %s",
op, cib_error2string(rc));
return;
}
diff = get_message_xml(msg, F_CIB_UPDATE_RESULT);
cib_diff_version_details(
diff,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
crm_debug("Processing diff (%s): %d.%d.%d -> %d.%d.%d", op,
diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
diff_add_admin_epoch,diff_add_epoch,diff_add_updates);
log_cib_diff(LOG_DEBUG_2, diff, op);
set_name = "diff-added";
if(diff != NULL) {
crm_data_t *section = NULL;
crm_data_t *change_set = find_xml_node(diff, set_name, FALSE);
change_set = find_xml_node(change_set, XML_TAG_CIB, FALSE);
if(change_set != NULL) {
crm_debug_2("Checking status changes");
section=get_object_root(XML_CIB_TAG_STATUS,change_set);
}
if(section != NULL) {
extract_event(section);
}
crm_debug_2("Checking change set: %s", set_name);
aborted = need_abort(change_set);
}
set_name = "diff-removed";
if(diff != NULL && aborted == NULL) {
crm_data_t *attrs = NULL;
crm_data_t *status = NULL;
crm_data_t *change_set = find_xml_node(diff, set_name, FALSE);
change_set = find_xml_node(change_set, XML_TAG_CIB, FALSE);
crm_debug_2("Checking change set: %s", set_name);
aborted = need_abort(change_set);
if(aborted == NULL && change_set != NULL) {
status = get_object_root(XML_CIB_TAG_STATUS, change_set);
xml_child_iter_filter(
status, node_state, XML_CIB_TAG_STATE,
attrs = find_xml_node(
node_state, XML_TAG_TRANSIENT_NODEATTRS, FALSE);
if(attrs != NULL) {
crm_info("Aborting on "XML_TAG_TRANSIENT_NODEATTRS" deletions");
abort_transition(INFINITY, tg_restart,
XML_TAG_TRANSIENT_NODEATTRS, attrs);
}
);
int
cib_server_process_diff(const char *op, int options, const char *section, xmlNode * req,
xmlNode * input, xmlNode * existing_cib, xmlNode ** result_cib,
xmlNode ** answer)
{
int rc = pcmk_ok;
if (cib_is_master) {
/* the master is never waiting for a resync */
sync_in_progress = 0;
}
if (sync_in_progress > MAX_DIFF_RETRY) {
/* request another full-sync,
* the last request may have been lost
*/
sync_in_progress = 0;
}
if (sync_in_progress) {
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
cib_diff_version_details(input,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
sync_in_progress++;
crm_notice("Not applying diff %d.%d.%d -> %d.%d.%d (sync in progress)",
diff_del_admin_epoch, diff_del_epoch, diff_del_updates,
diff_add_admin_epoch, diff_add_epoch, diff_add_updates);
return -pcmk_err_diff_resync;
}
rc = cib_process_diff(op, options, section, req, input, existing_cib, result_cib, answer);
if (rc == -pcmk_err_diff_resync && cib_is_master == FALSE) {
xmlNode *sync_me = create_xml_node(NULL, "sync-me");
free_xml(*result_cib);
*result_cib = NULL;
crm_info("Requesting re-sync from peer");
sync_in_progress++;
crm_xml_add(sync_me, F_TYPE, "cib");
crm_xml_add(sync_me, F_CIB_OPERATION, CIB_OP_SYNC_ONE);
crm_xml_add(sync_me, F_CIB_DELEGATED, cib_our_uname);
if (send_cluster_message(NULL, crm_msg_cib, sync_me, FALSE) == FALSE) {
rc = -ENOTCONN;
}
free_xml(sync_me);
} else if (rc == -pcmk_err_diff_resync) {
rc = -pcmk_err_diff_failed;
if (options & cib_force_diff) {
crm_warn("Not requesting full refresh in R/W mode");
}
}
return rc;
}
enum cib_errors
cib_process_diff(
const char *op, int options, const char *section, crm_data_t *input,
crm_data_t *existing_cib, crm_data_t **result_cib, crm_data_t **answer)
{
unsigned int log_level = LOG_DEBUG;
const char *value = NULL;
const char *reason = NULL;
gboolean apply_diff = TRUE;
gboolean do_resync = FALSE;
enum cib_errors result = cib_ok;
int this_updates = 0;
int this_epoch = 0;
int this_admin_epoch = 0;
int diff_add_updates = 0;
int diff_add_epoch = 0;
int diff_add_admin_epoch = 0;
int diff_del_updates = 0;
int diff_del_epoch = 0;
int diff_del_admin_epoch = 0;
crm_debug_2("Processing \"%s\" event", op);
if(cib_is_master) {
/* the master is never waiting for a resync */
sync_in_progress = 0;
}
cib_diff_version_details(
input,
&diff_add_admin_epoch, &diff_add_epoch, &diff_add_updates,
&diff_del_admin_epoch, &diff_del_epoch, &diff_del_updates);
if(sync_in_progress > MAX_DIFF_RETRY) {
/* request another full-sync,
* the last request may have been lost
*/
sync_in_progress = 0;
}
if(sync_in_progress) {
sync_in_progress++;
crm_warn("Not applying diff %d.%d.%d -> %d.%d.%d (sync in progress)",
diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
diff_add_admin_epoch,diff_add_epoch,diff_add_updates);
return cib_diff_resync;
}
value = crm_element_value(existing_cib, XML_ATTR_GENERATION);
this_epoch = atoi(value?value:"0");
value = crm_element_value(existing_cib, XML_ATTR_NUMUPDATES);
this_updates = atoi(value?value:"0");
value = crm_element_value(existing_cib, XML_ATTR_GENERATION_ADMIN);
this_admin_epoch = atoi(value?value:"0");
if(diff_del_admin_epoch == diff_add_admin_epoch
&& diff_del_epoch == diff_add_epoch
&& diff_del_updates == diff_add_updates) {
if(diff_add_admin_epoch == -1 && diff_add_epoch == -1 && diff_add_updates == -1) {
diff_add_epoch = this_epoch;
diff_add_updates = this_updates + 1;
diff_add_admin_epoch = this_admin_epoch;
diff_del_epoch = this_epoch;
diff_del_updates = this_updates;
diff_del_admin_epoch = this_admin_epoch;
} else {
apply_diff = FALSE;
log_level = LOG_ERR;
reason = "+ and - versions in the diff did not change";
log_cib_diff(LOG_ERR, input, __FUNCTION__);
}
}
if(apply_diff && diff_del_admin_epoch > this_admin_epoch) {
do_resync = TRUE;
apply_diff = FALSE;
log_level = LOG_INFO;
reason = "current \""XML_ATTR_GENERATION_ADMIN"\" is less than required";
} else if(apply_diff && diff_del_admin_epoch < this_admin_epoch) {
apply_diff = FALSE;
log_level = LOG_WARNING;
reason = "current \""XML_ATTR_GENERATION_ADMIN"\" is greater than required";
}
if(apply_diff && diff_del_epoch > this_epoch) {
do_resync = TRUE;
apply_diff = FALSE;
log_level = LOG_INFO;
reason = "current \""XML_ATTR_GENERATION"\" is less than required";
} else if(apply_diff && diff_del_epoch < this_epoch) {
apply_diff = FALSE;
log_level = LOG_WARNING;
reason = "current \""XML_ATTR_GENERATION"\" is greater than required";
}
if(apply_diff && diff_del_updates > this_updates) {
do_resync = TRUE;
apply_diff = FALSE;
log_level = LOG_INFO;
reason = "current \""XML_ATTR_NUMUPDATES"\" is less than required";
} else if(apply_diff && diff_del_updates < this_updates) {
apply_diff = FALSE;
log_level = LOG_WARNING;
reason = "current \""XML_ATTR_NUMUPDATES"\" is greater than required";
}
if(apply_diff) {
free_xml(*result_cib);
*result_cib = NULL;
if(apply_xml_diff(existing_cib, input, result_cib) == FALSE) {
log_level = LOG_WARNING;
reason = "Failed application of an update diff";
if(options & cib_force_diff) {
if(cib_is_master == FALSE) {
log_level = LOG_INFO;
reason = "Failed application of a global update."
" Requesting full refresh.";
do_resync = TRUE;
} else {
reason = "Failed application of a global update."
" Not requesting full refresh.";
}
}
} else if((options & cib_force_diff) && !validate_with_dtd(
*result_cib, FALSE, HA_NOARCHDATAHBDIR"/crm.dtd")) {
if(cib_is_master == FALSE) {
log_level = LOG_INFO;
reason = "Failed DTD validation of a global update."
" Requesting full refresh.";
do_resync = TRUE;
} else {
log_level = LOG_WARNING;
reason = "Failed DTD validation of a global update."
" Not requesting full refresh.";
}
}
}
if(reason != NULL) {
do_crm_log(
log_level,
"Diff %d.%d.%d -> %d.%d.%d not applied to %d.%d.%d: %s",
diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
diff_add_admin_epoch,diff_add_epoch,diff_add_updates,
this_admin_epoch,this_epoch,this_updates, reason);
result = cib_diff_failed;
} else if(apply_diff) {
crm_debug_2("Diff %d.%d.%d -> %d.%d.%d was applied",
diff_del_admin_epoch,diff_del_epoch,diff_del_updates,
diff_add_admin_epoch,diff_add_epoch,diff_add_updates);
}
if(do_resync && cib_is_master == FALSE) {
HA_Message *sync_me = ha_msg_new(3);
free_xml(*result_cib);
*result_cib = NULL;
result = cib_diff_resync;
crm_info("Requesting re-sync from peer: %s", reason);
sync_in_progress++;
ha_msg_add(sync_me, F_TYPE, "cib");
ha_msg_add(sync_me, F_CIB_OPERATION, CIB_OP_SYNC_ONE);
ha_msg_add(sync_me, F_CIB_DELEGATED, cib_our_uname);
if(send_ha_message(hb_conn, sync_me, NULL, FALSE) == FALSE) {
result = cib_not_connected;
}
ha_msg_del(sync_me);
} else if(do_resync) {
crm_err("Not resyncing in master mode");
}
return result;
}
