static void
node_hash_update_one(GHashTable * hash, node_t * other, const char *attr, int score)
{
GHashTableIter iter;
node_t *node = NULL;
const char *value = NULL;
if (other == NULL) {
return;
} else if (attr == NULL) {
attr = "#" XML_ATTR_UNAME;
}
value = g_hash_table_lookup(other->details->attrs, attr);
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
const char *tmp = g_hash_table_lookup(node->details->attrs, attr);
if (safe_str_eq(value, tmp)) {
crm_trace("%s: %d + %d", node->details->uname, node->weight, other->weight);
node->weight = merge_weights(node->weight, score);
}
}
}
/* any node in list1 or list2 and not in the other gets a score of -INFINITY */
void
node_list_exclude(GHashTable * hash, GListPtr list, gboolean merge_scores)
{
GHashTable *result = hash;
node_t *other_node = NULL;
GListPtr gIter = list;
GHashTableIter iter;
node_t *node = NULL;
g_hash_table_iter_init(&iter, hash);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
other_node = pe_find_node_id(list, node->details->id);
if (other_node == NULL) {
node->weight = -INFINITY;
} else if (merge_scores) {
node->weight = merge_weights(node->weight, other_node->weight);
}
}
for (; gIter != NULL; gIter = gIter->next) {
node_t *node = (node_t *) gIter->data;
other_node = pe_hash_table_lookup(result, node->details->id);
if (other_node == NULL) {
node_t *new_node = node_copy(node);
new_node->weight = -INFINITY;
g_hash_table_insert(result, (gpointer) new_node->details->id, new_node);
}
}
}
static void
apply_master_prefs(resource_t * rsc)
{
int score, new_score;
GListPtr gIter = rsc->children;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
if (clone_data->applied_master_prefs) {
/* Make sure we only do this once */
return;
}
clone_data->applied_master_prefs = TRUE;
for (; gIter != NULL; gIter = gIter->next) {
GHashTableIter iter;
node_t *node = NULL;
resource_t *child_rsc = (resource_t *) gIter->data;
g_hash_table_iter_init(&iter, child_rsc->allowed_nodes);
while (g_hash_table_iter_next(&iter, NULL, (void **)&node)) {
if (can_run_resources(node) == FALSE) {
/* This node will never be promoted to master,
* so don't apply the master score as that may
* lead to clone shuffling
*/
continue;
}
score = master_score(child_rsc, node, 0);
if (score > 0) {
new_score = merge_weights(node->weight, score);
if (new_score != node->weight) {
crm_trace("\t%s: Updating preference for %s (%d->%d)",
child_rsc->id, node->details->uname, node->weight, new_score);
node->weight = new_score;
}
}
new_score = max(child_rsc->priority, score);
if (new_score != child_rsc->priority) {
crm_trace("\t%s: Updating priority (%d->%d)",
child_rsc->id, child_rsc->priority, new_score);
child_rsc->priority = new_score;
}
}
}
}
void sync_layer(network *nets, int n, int j)
{
//printf("Syncing layer %d\n", j);
int i;
network net = nets[0];
layer base = net.layers[j];
cuda_set_device(net.gpu_index);
pull_weights(base);
for (i = 1; i < n; ++i) {
cuda_set_device(nets[i].gpu_index);
layer l = nets[i].layers[j];
pull_weights(l);
merge_weights(l, base);
}
scale_weights(base, 1./n);
for (i = 0; i < n; ++i) {
cuda_set_device(nets[i].gpu_index);
layer l = nets[i].layers[j];
distribute_weights(l, base);
}
//printf("Done syncing layer %d\n", j);
}
void
master_rsc_colocation_rh(resource_t * rsc_lh, resource_t * rsc_rh, rsc_colocation_t * constraint)
{
GListPtr gIter = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc_rh);
CRM_CHECK(rsc_rh != NULL, return);
if (is_set(rsc_rh->flags, pe_rsc_provisional)) {
return;
} else if (constraint->role_rh == RSC_ROLE_UNKNOWN) {
crm_trace("Handling %s as a clone colocation", constraint->id);
clone_rsc_colocation_rh(rsc_lh, rsc_rh, constraint);
return;
}
CRM_CHECK(rsc_lh != NULL, return);
CRM_CHECK(rsc_lh->variant == pe_native, return);
crm_trace("Processing constraint %s: %d", constraint->id, constraint->score);
if (constraint->role_rh == RSC_ROLE_UNKNOWN) {
gIter = rsc_rh->children;
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child_rsc = (resource_t *) gIter->data;
child_rsc->cmds->rsc_colocation_rh(rsc_lh, child_rsc, constraint);
}
} else if (is_set(rsc_lh->flags, pe_rsc_provisional)) {
GListPtr rhs = NULL;
gIter = rsc_rh->children;
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child_rsc = (resource_t *) gIter->data;
node_t *chosen = child_rsc->fns->location(child_rsc, NULL, FALSE);
enum rsc_role_e next_role = child_rsc->fns->state(child_rsc, FALSE);
crm_trace("Processing: %s", child_rsc->id);
if (chosen != NULL && next_role == constraint->role_rh) {
crm_trace("Applying: %s %s %s %d", child_rsc->id,
role2text(next_role), chosen->details->uname, constraint->score);
if (constraint->score < INFINITY) {
node_hash_update_one(rsc_lh->allowed_nodes, chosen,
constraint->node_attribute, constraint->score);
}
rhs = g_list_prepend(rhs, chosen);
}
}
/* Only do this if its not a master-master colocation
* Doing this unconditionally would prevent the slaves from being started
*/
if (constraint->role_lh != RSC_ROLE_MASTER || constraint->role_rh != RSC_ROLE_MASTER) {
if (constraint->score >= INFINITY) {
node_list_exclude(rsc_lh->allowed_nodes, rhs, TRUE);
}
}
g_list_free(rhs);
} else if (constraint->role_lh == RSC_ROLE_MASTER) {
resource_t *rh_child = find_compatible_child(rsc_lh, rsc_rh, constraint->role_rh, FALSE);
if (rh_child == NULL && constraint->score >= INFINITY) {
crm_trace("%s can't be promoted %s", rsc_lh->id, constraint->id);
rsc_lh->priority = -INFINITY;
} else if (rh_child != NULL) {
int new_priority = merge_weights(rsc_lh->priority, constraint->score);
crm_debug("Applying %s to %s", constraint->id, rsc_lh->id);
crm_debug("\t%s: %d->%d", rsc_lh->id, rsc_lh->priority, new_priority);
rsc_lh->priority = new_priority;
}
}
return;
}
static void
master_promotion_order(resource_t * rsc, pe_working_set_t * data_set)
{
GListPtr gIter = NULL;
node_t *node = NULL;
node_t *chosen = NULL;
clone_variant_data_t *clone_data = NULL;
get_clone_variant_data(clone_data, rsc);
if (clone_data->merged_master_weights) {
return;
}
clone_data->merged_master_weights = TRUE;
crm_trace("Merging weights for %s", rsc->id);
set_bit(rsc->flags, pe_rsc_merging);
gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child = (resource_t *) gIter->data;
crm_trace("Sort index: %s = %d", child->id, child->sort_index);
}
dump_node_scores(LOG_DEBUG_3, rsc, "Before", rsc->allowed_nodes);
gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child = (resource_t *) gIter->data;
chosen = child->fns->location(child, NULL, FALSE);
if (chosen == NULL || child->sort_index < 0) {
crm_trace("Skipping %s", child->id);
continue;
}
node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id);
CRM_ASSERT(node != NULL);
/* adds in master preferences and rsc_location.role=Master */
crm_trace("Adding %s to %s from %s", score2char(child->sort_index), node->details->uname,
child->id);
node->weight = merge_weights(child->sort_index, node->weight);
}
dump_node_scores(LOG_DEBUG_3, rsc, "Middle", rsc->allowed_nodes);
gIter = rsc->rsc_cons;
for (; gIter != NULL; gIter = gIter->next) {
rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
/* (re-)adds location preferences of resources that the
* master instance should/must be colocated with
*/
if (constraint->role_lh == RSC_ROLE_MASTER) {
crm_trace("RHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id,
constraint->score);
rsc->allowed_nodes =
constraint->rsc_rh->cmds->merge_weights(constraint->rsc_rh, rsc->id,
rsc->allowed_nodes,
constraint->node_attribute,
constraint->score / INFINITY,
constraint->score ==
INFINITY ? FALSE : TRUE, FALSE);
}
}
gIter = rsc->rsc_cons_lhs;
for (; gIter != NULL; gIter = gIter->next) {
rsc_colocation_t *constraint = (rsc_colocation_t *) gIter->data;
/* (re-)adds location preferences of resource that wish to be
* colocated with the master instance
*/
if (constraint->role_rh == RSC_ROLE_MASTER) {
crm_trace("LHS: %s with %s: %d", constraint->rsc_lh->id, constraint->rsc_rh->id,
constraint->score);
rsc->allowed_nodes =
constraint->rsc_lh->cmds->merge_weights(constraint->rsc_lh, rsc->id,
rsc->allowed_nodes,
constraint->node_attribute,
constraint->score / INFINITY, TRUE, TRUE);
}
}
gIter = rsc->rsc_tickets;
for (; gIter != NULL; gIter = gIter->next) {
rsc_ticket_t *rsc_ticket = (rsc_ticket_t *) gIter->data;
if (rsc_ticket->role_lh == RSC_ROLE_MASTER && rsc_ticket->ticket->granted == FALSE) {
resource_location(rsc, NULL, -INFINITY, "__stateful_without_ticket__", data_set);
}
}
dump_node_scores(LOG_DEBUG_3, rsc, "After", rsc->allowed_nodes);
/* write them back and sort */
gIter = rsc->children;
for (; gIter != NULL; gIter = gIter->next) {
resource_t *child = (resource_t *) gIter->data;
chosen = child->fns->location(child, NULL, FALSE);
if (is_not_set(child->flags, pe_rsc_managed) && child->next_role == RSC_ROLE_MASTER) {
child->sort_index = INFINITY;
} else if (chosen == NULL || child->sort_index < 0) {
crm_trace("%s: %d", child->id, child->sort_index);
} else {
node = (node_t *) pe_hash_table_lookup(rsc->allowed_nodes, chosen->details->id);
CRM_ASSERT(node != NULL);
child->sort_index = node->weight;
}
crm_trace("Set sort index: %s = %d", child->id, child->sort_index);
}
rsc->children = g_list_sort_with_data(rsc->children, sort_master_instance, data_set);
clear_bit(rsc->flags, pe_rsc_merging);
}
int
pe_get_failcount(node_t *node, resource_t *rsc, time_t *last_failure,
uint32_t flags, xmlNode *xml_op, pe_working_set_t *data_set)
{
char *key = NULL;
const char *value = NULL;
regex_t failcount_re, lastfailure_re;
int failcount = 0;
time_t last = 0;
GHashTableIter iter;
generate_fail_regexes(rsc, data_set, &failcount_re, &lastfailure_re);
/* Resource fail count is sum of all matching operation fail counts */
g_hash_table_iter_init(&iter, node->details->attrs);
while (g_hash_table_iter_next(&iter, (gpointer *) &key, (gpointer *) &value)) {
if (regexec(&failcount_re, key, 0, NULL, 0) == 0) {
failcount = merge_weights(failcount, char2score(value));
} else if (regexec(&lastfailure_re, key, 0, NULL, 0) == 0) {
last = QB_MAX(last, crm_int_helper(value, NULL));
}
}
regfree(&failcount_re);
regfree(&lastfailure_re);
if ((failcount > 0) && (last > 0) && (last_failure != NULL)) {
*last_failure = last;
}
/* If failure blocks the resource, disregard any failure timeout */
if ((failcount > 0) && rsc->failure_timeout
&& block_failure(node, rsc, xml_op, data_set)) {
pe_warn("Ignoring failure timeout %d for %s because it conflicts with on-fail=block",
rsc->id, rsc->failure_timeout);
rsc->failure_timeout = 0;
}
/* If all failures have expired, ignore fail count */
if (is_set(flags, pe_fc_effective) && (failcount > 0) && (last > 0)
&& rsc->failure_timeout) {
time_t now = get_effective_time(data_set);
if (now > (last + rsc->failure_timeout)) {
crm_debug("Failcount for %s on %s expired after %ds",
rsc->id, node->details->uname, rsc->failure_timeout);
failcount = 0;
}
}
if (is_set(flags, pe_fc_fillers) && rsc->fillers) {
GListPtr gIter = NULL;
for (gIter = rsc->fillers; gIter != NULL; gIter = gIter->next) {
resource_t *filler = (resource_t *) gIter->data;
time_t filler_last_failure = 0;
failcount += pe_get_failcount(node, filler, &filler_last_failure,
flags, xml_op, data_set);
if (last_failure && filler_last_failure > *last_failure) {
*last_failure = filler_last_failure;
}
}
if (failcount > 0) {
char *score = score2char(failcount);
crm_info("Container %s and the resources within it have failed %s times on %s",
rsc->id, score, node->details->uname);
free(score);
}
} else if (failcount > 0) {
char *score = score2char(failcount);
crm_info("%s has failed %s times on %s",
rsc->id, score, node->details->uname);
free(score);
}
return failcount;
}