Ejemplo n.º 1
0
static void
reap_dead_nodes(gpointer key, gpointer value, gpointer user_data)
{
    crm_node_t *node = value;

    if (crm_is_peer_active(node) == FALSE) {
        crm_update_peer_join(__FUNCTION__, node, crm_join_none);

        if(node && node->uname) {
            election_remove(fsa_election, node->uname);

            if (safe_str_eq(fsa_our_uname, node->uname)) {
                crm_err("We're not part of the cluster anymore");
                register_fsa_input(C_FSA_INTERNAL, I_ERROR, NULL);

            } else if (AM_I_DC == FALSE && safe_str_eq(node->uname, fsa_our_dc)) {
                crm_warn("Our DC node (%s) left the cluster", node->uname);
                register_fsa_input(C_FSA_INTERNAL, I_ELECTION, NULL);
            }
        }

        if (fsa_state == S_INTEGRATION || fsa_state == S_FINALIZE_JOIN) {
            check_join_state(fsa_state, __FUNCTION__);
        }
        fail_incompletable_actions(transition_graph, node->uuid);
    }
}
Ejemplo n.º 2
0
void
finalize_sync_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
    CRM_LOG_ASSERT(-EPERM != rc);
    clear_bit(fsa_input_register, R_CIB_ASKED);
    if (rc != pcmk_ok) {
        do_crm_log((rc == -pcmk_err_old_data ? LOG_WARNING : LOG_ERR),
                   "Sync from %s failed: %s", (char *)user_data, pcmk_strerror(rc));

        /* restart the whole join process */
        register_fsa_error_adv(C_FSA_INTERNAL, I_ELECTION_DC, NULL, NULL, __FUNCTION__);

    } else if (AM_I_DC && fsa_state == S_FINALIZE_JOIN) {
        set_bit(fsa_input_register, R_HAVE_CIB);
        clear_bit(fsa_input_register, R_CIB_ASKED);

        /* make sure dc_uuid is re-set to us */
        if (check_join_state(fsa_state, __FUNCTION__) == FALSE) {
            crm_debug("Notifying %d clients of join-%d results",
                      crmd_join_phase_count(crm_join_integrated), current_join_id);
            g_hash_table_foreach(crm_peer_cache, finalize_join_for, NULL);
        }

    } else {
        crm_debug("No longer the DC in S_FINALIZE_JOIN: %s/%s",
                  AM_I_DC ? "DC" : "CRMd", fsa_state2string(fsa_state));
    }
}
Ejemplo n.º 3
0
/*	A_DC_JOIN_FINALIZE	*/
void
do_dc_join_finalize(long long action,
                    enum crmd_fsa_cause cause,
                    enum crmd_fsa_state cur_state,
                    enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
    char *sync_from = NULL;
    int rc = pcmk_ok;

    /* This we can do straight away and avoid clients timing us out
     *  while we compute the latest CIB
     */
    crm_debug("Finializing join-%d for %d clients",
              current_join_id, crmd_join_phase_count(crm_join_integrated));

    crmd_join_phase_log(LOG_INFO);
    if (crmd_join_phase_count(crm_join_welcomed) != 0) {
        crm_info("Waiting for %d more nodes", crmd_join_phase_count(crm_join_welcomed));
        /* crmd_fsa_stall(FALSE); Needed? */
        return;

    } else if (crmd_join_phase_count(crm_join_integrated) == 0) {
        /* Nothing to do */
        check_join_state(fsa_state, __FUNCTION__);
        return;
    }

    clear_bit(fsa_input_register, R_HAVE_CIB);
    if (max_generation_from == NULL || safe_str_eq(max_generation_from, fsa_our_uname)) {
        set_bit(fsa_input_register, R_HAVE_CIB);
    }

    if (is_set(fsa_input_register, R_IN_TRANSITION)) {
        crm_warn("Delaying response to cluster join offer while transition in progress "
                 CRM_XS " join-%d", current_join_id);
        crmd_fsa_stall(FALSE);
        return;
    }

    if (max_generation_from && is_set(fsa_input_register, R_HAVE_CIB) == FALSE) {
        /* ask for the agreed best CIB */
        sync_from = strdup(max_generation_from);
        set_bit(fsa_input_register, R_CIB_ASKED);
        crm_notice("Syncing the Cluster Information Base from %s to rest of cluster "
                   CRM_XS " join-%d", sync_from, current_join_id);
        crm_log_xml_notice(max_generation_xml, "Requested version");

    } else {
        /* Send _our_ CIB out to everyone */
        sync_from = strdup(fsa_our_uname);
        crm_info("join-%d: Syncing our CIB to the rest of the cluster",
                 current_join_id);
        crm_log_xml_debug(max_generation_xml, "Requested version");
    }


    rc = fsa_cib_conn->cmds->sync_from(fsa_cib_conn, sync_from, NULL, cib_quorum_override);
    fsa_register_cib_callback(rc, FALSE, sync_from, finalize_sync_callback);
}
Ejemplo n.º 4
0
static void
join_update_complete_callback(xmlNode * msg, int call_id, int rc, xmlNode * output, void *user_data)
{
    fsa_data_t *msg_data = NULL;

    if (rc == pcmk_ok) {
        crm_debug("Join update %d complete", call_id);
        check_join_state(fsa_state, __FUNCTION__);

    } else {
        crm_err("Join update %d failed", call_id);
        crm_log_xml_debug(msg, "failed");
        register_fsa_error(C_FSA_INTERNAL, I_ERROR, NULL);
    }
}
/*
	CIBへのsync_from処理のコールバック処理
		CRM_OP_JOIN_ACKNAKメッセージを送信する
*/
void
finalize_sync_callback(xmlNode *msg, int call_id, int rc,
		       xmlNode *output, void *user_data) 
{
	CRM_DEV_ASSERT(cib_not_master != rc);
	clear_bit_inplace(fsa_input_register, R_CIB_ASKED);
	if(rc != cib_ok) {
		do_crm_log((rc==cib_old_data?LOG_WARNING:LOG_ERR),
			      "Sync from %s resulted in an error: %s",
			      (char*)user_data, cib_error2string(rc));

		/* restart the whole join process */
		/* コールバックがcib_okではない場合は、I_ELECTION_DCに戻る */
		register_fsa_error_adv(
			C_FSA_INTERNAL, I_ELECTION_DC,NULL,NULL,__FUNCTION__);

	} else if(AM_I_DC && fsa_state == S_FINALIZE_JOIN) {
		/* 自ノードがDCノードで、fsa_stateがS_FINALIZE_JOINの場合 */
	    set_bit_inplace(fsa_input_register, R_HAVE_CIB);
	    clear_bit_inplace(fsa_input_register, R_CIB_ASKED);
	    
	    /* make sure dc_uuid is re-set to us */
	    /* JOIN状態チェック処理 */
	    if(check_join_state(fsa_state, __FUNCTION__) == FALSE) {
			crm_debug("Notifying %d clients of join-%d results",
			  g_hash_table_size(integrated_nodes), current_join_id);
			
			/*
				ノードをCIBのnodeエントリに追加して、CRM_OP_JOIN_ACKNAKメッセージを送信する
				また、クラスタメンバーとして認識したノードは、finalized_nodesハッシュテーブルに追加する	
				そして、integrated_nodesハッシュテーブルから削除する
			*/
			g_hash_table_foreach_remove(
		    	integrated_nodes, finalize_join_for, NULL);
	    }
		
	} else {
		crm_debug("No longer the DC in S_FINALIZE_JOIN: %s/%s",
			  AM_I_DC?"DC":"CRMd", fsa_state2string(fsa_state));
	}
	
	crm_free(user_data);
}
/*
	A_DC_JOIN_PROCESS_REQアクション処理(CRM_OP_JOIN_REQUESTをDCノードが受信した時の処理)
*/
void
do_dc_join_filter_offer(long long action,
	       enum crmd_fsa_cause cause,
	       enum crmd_fsa_state cur_state,
	       enum crmd_fsa_input current_input,
	       fsa_data_t *msg_data)
{
	xmlNode *generation = NULL;

	int cmp = 0;
	int join_id = -1;
	gboolean ack_nack_bool = TRUE;
	const char *ack_nack = CRMD_JOINSTATE_MEMBER;
	ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);

	const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
	const char *ref       = crm_element_value(join_ack->msg, XML_ATTR_REFERENCE);
	
	crm_node_t *join_node = crm_get_peer(0, join_from);

	crm_debug("Processing req from %s", join_from);
	
	generation = join_ack->xml;
	crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);

	if(max_generation_xml != NULL && generation != NULL) {
	    int lpc = 0;
	    const char *attributes[] = {
		XML_ATTR_GENERATION_ADMIN,
		XML_ATTR_GENERATION,
		XML_ATTR_NUMUPDATES,
	    };
	    
	    for(lpc = 0; cmp == 0 && lpc < DIMOF(attributes); lpc++) {
			cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
	    }
	}
	
	if(join_id != current_join_id) {
		crm_debug("Invalid response from %s: join-%d vs. join-%d",
			  join_from, join_id, current_join_id);
		/* JOIN状態チェックを行う */
		check_join_state(cur_state, __FUNCTION__);
		return;
		
	} else if(join_node == NULL || crm_is_member_active(join_node) == FALSE) {
		crm_err("Node %s is not a member", join_from);
		ack_nack_bool = FALSE;
		
	} else if(generation == NULL) {
		crm_err("Generation was NULL");
		ack_nack_bool = FALSE;

	} else if(max_generation_xml == NULL) {
		max_generation_xml = copy_xml(generation);
		max_generation_from = crm_strdup(join_from);

	} else if(cmp < 0
		  || (cmp == 0 && safe_str_eq(join_from, fsa_our_uname))) {
		crm_debug("%s has a better generation number than"
			  " the current max %s",
			  join_from, max_generation_from);
		if(max_generation_xml) {
			crm_log_xml_debug(max_generation_xml, "Max generation");
		}
		crm_log_xml_debug(generation, "Their generation");
		
		crm_free(max_generation_from);
		free_xml(max_generation_xml);
		
		max_generation_from = crm_strdup(join_from);
		max_generation_xml = copy_xml(join_ack->xml);
	}
	
	if(ack_nack_bool == FALSE) {
		/* NACK this client */
		ack_nack = CRMD_JOINSTATE_NACK;
		crm_err("join-%d: NACK'ing node %s (ref %s)",
			join_id, join_from, ref);
	} else {
		crm_debug("join-%d: Welcoming node %s (ref %s)",
			  join_id, join_from, ref);
	}
	
	/* add them to our list of CRMD_STATE_ACTIVE nodes */
	/* integrated_nodesハッシュテーブルにCRM_OP_JOIN_REQUESTを送信して来たノードを追加する */
	g_hash_table_insert(
		integrated_nodes, crm_strdup(join_from), crm_strdup(ack_nack));

	/* integrated_nodesハッシュテーブルサイズと、CRM_OP_JOIN_REQUESTを送信して来たノードjoin_idをログ出力する */
	crm_debug("%u nodes have been integrated into join-%d",
		    g_hash_table_size(integrated_nodes), join_id);
	
	/* welcomed_nodesハッシュテーブルからCRM_OP_JOIN_REQUESTを送信して来たノードを削除する */
	g_hash_table_remove(welcomed_nodes, join_from);

	/* JOIN状態チェックを行う */
	if(check_join_state(cur_state, __FUNCTION__) == FALSE) {
		/* dont waste time by invoking the PE yet; */
		crm_debug("join-%d: Still waiting on %d outstanding offers",
			  join_id, g_hash_table_size(welcomed_nodes));
	}
}
Ejemplo n.º 7
0
/*	 A_DC_JOIN_PROCESS_REQ	*/
void
do_dc_join_filter_offer(long long action,
                        enum crmd_fsa_cause cause,
                        enum crmd_fsa_state cur_state,
                        enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
    xmlNode *generation = NULL;

    int cmp = 0;
    int join_id = -1;
    gboolean ack_nack_bool = TRUE;
    const char *ack_nack = CRMD_JOINSTATE_MEMBER;
    ha_msg_input_t *join_ack = fsa_typed_data(fsa_dt_ha_msg);

    const char *join_from = crm_element_value(join_ack->msg, F_CRM_HOST_FROM);
    const char *ref = crm_element_value(join_ack->msg, F_CRM_REFERENCE);

    crm_node_t *join_node = crm_get_peer(0, join_from);

    crm_debug("Processing req from %s", join_from);

    generation = join_ack->xml;
    crm_element_value_int(join_ack->msg, F_CRM_JOIN_ID, &join_id);

    if (max_generation_xml != NULL && generation != NULL) {
        int lpc = 0;

        const char *attributes[] = {
            XML_ATTR_GENERATION_ADMIN,
            XML_ATTR_GENERATION,
            XML_ATTR_NUMUPDATES,
        };

        for (lpc = 0; cmp == 0 && lpc < DIMOF(attributes); lpc++) {
            cmp = compare_int_fields(max_generation_xml, generation, attributes[lpc]);
        }
    }

    if (join_id != current_join_id) {
        crm_debug("Invalid response from %s: join-%d vs. join-%d",
                  join_from, join_id, current_join_id);
        check_join_state(cur_state, __FUNCTION__);
        return;

    } else if (join_node == NULL || crm_is_peer_active(join_node) == FALSE) {
        crm_err("Node %s is not a member", join_from);
        ack_nack_bool = FALSE;

    } else if (generation == NULL) {
        crm_err("Generation was NULL");
        ack_nack_bool = FALSE;

    } else if (max_generation_xml == NULL) {
        max_generation_xml = copy_xml(generation);
        max_generation_from = strdup(join_from);

    } else if (cmp < 0 || (cmp == 0 && safe_str_eq(join_from, fsa_our_uname))) {
        crm_debug("%s has a better generation number than"
                  " the current max %s", join_from, max_generation_from);
        if (max_generation_xml) {
            crm_log_xml_debug(max_generation_xml, "Max generation");
        }
        crm_log_xml_debug(generation, "Their generation");

        free(max_generation_from);
        free_xml(max_generation_xml);

        max_generation_from = strdup(join_from);
        max_generation_xml = copy_xml(join_ack->xml);
    }

    if (ack_nack_bool == FALSE) {
        /* NACK this client */
        ack_nack = CRMD_JOINSTATE_NACK;
        crm_update_peer_join(__FUNCTION__, join_node, crm_join_nack);
        crm_err("Rejecting cluster join request from %s " CRM_XS
                " NACK join-%d ref=%s", join_from, join_id, ref);

    } else {
        crm_debug("join-%d: Welcoming node %s (ref %s)", join_id, join_from, ref);
        crm_update_peer_join(__FUNCTION__, join_node, crm_join_integrated);
    }

    crm_update_peer_expected(__FUNCTION__, join_node, ack_nack);

    crm_debug("%u nodes have been integrated into join-%d",
              crmd_join_phase_count(crm_join_integrated), join_id);


    if (check_join_state(cur_state, __FUNCTION__) == FALSE) {
        /* don't waste time by invoking the PE yet; */
        crm_debug("join-%d: Still waiting on %d outstanding offers",
                  join_id, crmd_join_phase_count(crm_join_welcomed));
    }
}
Ejemplo n.º 8
0
/*	 A_DC_JOIN_OFFER_ONE	*/
void
do_dc_join_offer_one(long long action,
                     enum crmd_fsa_cause cause,
                     enum crmd_fsa_state cur_state,
                     enum crmd_fsa_input current_input, fsa_data_t * msg_data)
{
    crm_node_t *member;
    ha_msg_input_t *welcome = NULL;

    const char *op = NULL;
    const char *join_to = NULL;

    if (msg_data->data) {
        welcome = fsa_typed_data(fsa_dt_ha_msg);

    } else {
        crm_info("An unknown node joined - (re-)offer to any unconfirmed nodes");
        g_hash_table_foreach(crm_peer_cache, join_make_offer, &member);
        check_join_state(cur_state, __FUNCTION__);
        return;
    }

    if (welcome == NULL) {
        crm_err("Attempt to send welcome message without a message to reply to!");
        return;
    }

    join_to = crm_element_value(welcome->msg, F_CRM_HOST_FROM);
    if (join_to == NULL) {
        crm_err("Attempt to send welcome message without a host to reply to!");
        return;
    }

    member = crm_get_peer(0, join_to);
    op = crm_element_value(welcome->msg, F_CRM_TASK);
    if (join_to != NULL && (cur_state == S_INTEGRATION || cur_state == S_FINALIZE_JOIN)) {
        /* note: it _is_ possible that a node will have been
         *  sick or starting up when the original offer was made.
         *  however, it will either re-announce itself in due course
         *  _or_ we can re-store the original offer on the client.
         */
        crm_trace("(Re-)offering membership to %s...", join_to);
    }

    crm_info("join-%d: Processing %s request from %s in state %s",
             current_join_id, op, join_to, fsa_state2string(cur_state));

    crm_update_peer_join(__FUNCTION__, member, crm_join_none);
    join_make_offer(NULL, member, NULL);

    /* always offer to the DC (ourselves)
     * this ensures the correct value for max_generation_from
     */
    member = crm_get_peer(0, fsa_our_uname);
    join_make_offer(NULL, member, NULL);

    /* this was a genuine join request, cancel any existing
     * transition and invoke the PE
     */
    abort_transition(INFINITY, tg_restart, "Node join", NULL);

    /* don't waste time by invoking the PE yet; */
    crm_debug("Waiting on %d outstanding join acks for join-%d",
              crmd_join_phase_count(crm_join_welcomed), current_join_id);
}
Ejemplo n.º 9
0
void
peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
    uint32_t old = 0;
    uint32_t changed = 0;
    bool appeared = FALSE;
    bool is_remote = is_set(node->flags, crm_remote_node);
    const char *status = NULL;

    /* Crmd waits to receive some information from the membership layer before
     * declaring itself operational. If this is being called for a cluster node,
     * indicate that we have it.
     */
    if (!is_remote) {
        set_bit(fsa_input_register, R_PEER_DATA);
    }

    if (node->uname == NULL) {
        return;
    }

    switch (type) {
        case crm_status_uname:
            /* If we've never seen the node, then it also won't be in the status section */
            crm_info("%s node %s is now %s",
                     (is_remote? "Remote" : "Cluster"),
                     node->uname, state_text(node->state));
            return;

        case crm_status_rstate:
        case crm_status_nstate:
            /* This callback should not be called unless the state actually
             * changed, but here's a failsafe just in case.
             */
            CRM_CHECK(safe_str_neq(data, node->state), return);

            crm_info("%s node %s is now %s (was %s)",
                     (is_remote? "Remote" : "Cluster"),
                     node->uname, state_text(node->state), state_text(data));

            if (safe_str_eq(CRM_NODE_MEMBER, node->state)) {
                appeared = TRUE;
                if (!is_remote) {
                    remove_stonith_cleanup(node->uname);
                }
            }

            crmd_alert_node_event(node);
            break;

        case crm_status_processes:
            if (data) {
                old = *(const uint32_t *)data;
                changed = node->processes ^ old;
            }

            status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
            crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
                     node->uname, peer2text(proc_flags), status,
                     AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);

            if ((changed & proc_flags) == 0) {
                /* Peer process did not change */
                crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
                return;
            } else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
                crm_trace("Not connected");
                return;
            } else if (fsa_state == S_STOPPING) {
                crm_trace("Stopping");
                return;
            }

            appeared = (node->processes & proc_flags) != 0;
            if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
                /* Did we get evicted? */
                crm_notice("Our peer connection failed");
                register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);

            } else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
                /* Did the DC leave us? */
                crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
                register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);

                /* @COMPAT DC < 1.1.13: If a DC shuts down normally, we don't
                 * want to fence it. Newer DCs will send their shutdown request
                 * to all peers, who will update the DC's expected state to
                 * down, thus avoiding fencing. We can safely erase the DC's
                 * transient attributes when it leaves in that case. However,
                 * the only way to avoid fencing older DCs is to leave the
                 * transient attributes intact until it rejoins.
                 */
                if (compare_version(fsa_our_dc_version, "3.0.9") > 0) {
                    erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
                }

            } else if(AM_I_DC && appeared == FALSE) {
                crm_info("Peer %s left us", node->uname);
                erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
            }
            break;
    }

    if (AM_I_DC) {
        xmlNode *update = NULL;
        int flags = node_update_peer;
        gboolean alive = is_remote? appeared : crm_is_peer_active(node);
        crm_action_t *down = match_down_event(node->uuid, appeared);

        crm_trace("Alive=%d, appeared=%d, down=%d",
                  alive, appeared, (down? down->id : -1));

        if (alive && type == crm_status_processes) {
            register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
        }

        if (down) {
            const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);

            if (safe_str_eq(task, CRM_OP_FENCE)) {

                /* tengine_stonith_callback() confirms fence actions */
                crm_trace("Updating CIB %s stonithd reported fencing of %s complete",
                          (down->confirmed? "after" : "before"), node->uname);

            } else if ((alive == FALSE) && safe_str_eq(task, CRM_OP_SHUTDOWN)) {
                crm_notice("%s of peer %s is complete "CRM_XS" op=%d",
                           task, node->uname, down->id);

                /* down->confirmed = TRUE; */
                stop_te_timer(down->timer);

                if (!is_remote) {
                    flags |= node_update_join | node_update_expected;
                    crmd_peer_down(node, FALSE);
                    check_join_state(fsa_state, __FUNCTION__);
                }

                update_graph(transition_graph, down);
                trigger_graph();

            } else {
                crm_trace("Node %s is %salive, was expected to %s (op %d)",
                          node->uname, (alive? "" : "not "), task, down->id);
            }

        } else if (appeared == FALSE) {
            crm_notice("Stonith/shutdown of %s not matched", node->uname);

            if (!is_remote) {
                crm_update_peer_join(__FUNCTION__, node, crm_join_none);
                check_join_state(fsa_state, __FUNCTION__);
            }

            abort_transition(INFINITY, tg_restart, "Node failure", NULL);
            fail_incompletable_actions(transition_graph, node->uuid);

        } else {
            crm_trace("Node %s came up, was not expected to be down",
                      node->uname);
        }

        if (is_remote) {
            /* A pacemaker_remote node won't have its cluster status updated
             * in the CIB by membership-layer callbacks, so do it here.
             */
            flags |= node_update_cluster;

            /* Trigger resource placement on newly integrated nodes */
            if (appeared) {
                abort_transition(INFINITY, tg_restart,
                                 "pacemaker_remote node integrated", NULL);
            }
        }

        /* Update the CIB node state */
        update = create_node_state_update(node, flags, NULL, __FUNCTION__);
        fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
                            cib_scope_local | cib_quorum_override | cib_can_create);
        free_xml(update);
    }

    trigger_fsa(fsa_source);
}
Ejemplo n.º 10
0
void
peer_update_callback(enum crm_status_type type, crm_node_t * node, const void *data)
{
    uint32_t old = 0;
    uint32_t changed = 0;
    bool appeared = FALSE;
    const char *status = NULL;

    set_bit(fsa_input_register, R_PEER_DATA);
    if (node->uname == NULL) {
        return;
    }

    switch (type) {
        case crm_status_uname:
            /* If we've never seen the node, then it also wont be in the status section */
            crm_info("%s is now %s", node->uname, node->state);
            return;
        case crm_status_rstate:
            crm_info("Remote node %s is now %s (was %s)", node->uname, node->state, (const char *)data);
            /* Keep going */
        case crm_status_nstate:
            crm_info("%s is now %s (was %s)", node->uname, node->state, (const char *)data);
            if (safe_str_eq(data, node->state)) {
                /* State did not change */
                return;
            } else if(safe_str_eq(CRM_NODE_MEMBER, node->state)) {
                appeared = TRUE;
            }
            break;
        case crm_status_processes:
            if (data) {
                old = *(const uint32_t *)data;
                changed = node->processes ^ old;
            }

            /* crmd_proc_update(node, proc_flags); */
            status = (node->processes & proc_flags) ? ONLINESTATUS : OFFLINESTATUS;
            crm_info("Client %s/%s now has status [%s] (DC=%s, changed=%6x)",
                     node->uname, peer2text(proc_flags), status,
                     AM_I_DC ? "true" : crm_str(fsa_our_dc), changed);

            if ((changed & proc_flags) == 0) {
                /* Peer process did not change */
                crm_trace("No change %6x %6x %6x", old, node->processes, proc_flags);
                return;
            } else if (is_not_set(fsa_input_register, R_CIB_CONNECTED)) {
                crm_trace("Not connected");
                return;
            } else if (fsa_state == S_STOPPING) {
                crm_trace("Stopping");
                return;
            }

            appeared = (node->processes & proc_flags) != 0;
            if (safe_str_eq(node->uname, fsa_our_uname) && (node->processes & proc_flags) == 0) {
                /* Did we get evicted? */
                crm_notice("Our peer connection failed");
                register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ERROR, NULL);

            } else if (safe_str_eq(node->uname, fsa_our_dc) && crm_is_peer_active(node) == FALSE) {
                /* Did the DC leave us? */
                crm_notice("Our peer on the DC (%s) is dead", fsa_our_dc);
                register_fsa_input(C_CRMD_STATUS_CALLBACK, I_ELECTION, NULL);

            } else if(AM_I_DC && appeared == FALSE) {
                crm_info("Peer %s left us", node->uname);
                /* crm_update_peer_join(__FUNCTION__, node, crm_join_none); */
            }
            break;
    }

    if (AM_I_DC) {
        xmlNode *update = NULL;
        int flags = node_update_peer;
        gboolean alive = crm_is_peer_active(node);
        crm_action_t *down = match_down_event(0, node->uuid, NULL, appeared);

        crm_trace("Alive=%d, appear=%d, down=%p", alive, appeared, down);

        if (alive && type == crm_status_processes) {
            register_fsa_input_before(C_FSA_INTERNAL, I_NODE_JOIN, NULL);
        }

        if (down) {
            const char *task = crm_element_value(down->xml, XML_LRM_ATTR_TASK);

            if (alive && safe_str_eq(task, CRM_OP_FENCE)) {
                crm_info("Node return implies stonith of %s (action %d) completed", node->uname,
                         down->id);
                erase_status_tag(node->uname, XML_CIB_TAG_LRM, cib_scope_local);
                erase_status_tag(node->uname, XML_TAG_TRANSIENT_NODEATTRS, cib_scope_local);
                /* down->confirmed = TRUE; Only stonith-ng returning should imply completion */
                down->sent_update = TRUE;       /* Prevent tengine_stonith_callback() from calling send_stonith_update() */

            } else if (safe_str_eq(task, CRM_OP_FENCE)) {
                crm_trace("Waiting for stonithd to report the fencing of %s is complete", node->uname); /* via tengine_stonith_callback() */

            } else if (alive == FALSE) {
                crm_notice("%s of %s (op %d) is complete", task, node->uname, down->id);
                /* down->confirmed = TRUE; Only stonith-ng returning should imply completion */
                stop_te_timer(down->timer);

                flags |= node_update_join | node_update_expected;
                crmd_peer_down(node, FALSE);
                check_join_state(fsa_state, __FUNCTION__);

                update_graph(transition_graph, down);
                trigger_graph();

            } else {
                crm_trace("Other %p", down);
            }

        } else if (appeared == FALSE) {
            crm_notice("Stonith/shutdown of %s not matched", node->uname);

            crm_update_peer_join(__FUNCTION__, node, crm_join_none);
            check_join_state(fsa_state, __FUNCTION__);

            abort_transition(INFINITY, tg_restart, "Node failure", NULL);
            fail_incompletable_actions(transition_graph, node->uuid);

        } else {
            crm_trace("Other %p", down);
        }

        update = do_update_node_cib(node, flags, NULL, __FUNCTION__);
        fsa_cib_anon_update(XML_CIB_TAG_STATUS, update,
                            cib_scope_local | cib_quorum_override | cib_can_create);
        free_xml(update);
    }

    trigger_fsa(fsa_source);
}