static int send_data(rmcast_base_send_t *snd, orte_rmcast_channel_t channel) { char *bytes=NULL; int32_t sz; int rc; opal_buffer_t *buf=NULL; rmcast_base_channel_t *chan; rmcast_send_log_t *log, *lg; if (!comm_enabled) { return ORTE_ERR_COMM_DISABLED; } OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output, "%s transmitting data for channel %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), channel)); /* setup the message for xmission */ if (ORTE_SUCCESS != (rc = orte_rmcast_base_queue_xmit(snd, channel, &buf, &chan))) { ORTE_ERROR_LOG(rc); goto CLEANUP; } /* store the working buf in the send ring buffer in case we * need to retransmit it later */ log = OBJ_NEW(rmcast_send_log_t); log->channel = chan->channel; log->seq_num = chan->seq_num; opal_dss.copy_payload(log->buf, buf); if (NULL != (lg = (rmcast_send_log_t*)opal_ring_buffer_push(&chan->cache, log))) { /* release the old message */ OPAL_OUTPUT_VERBOSE((5, orte_rmcast_base.rmcast_output, "%s releasing message %d channel %d from log", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), lg->seq_num, lg->channel)); OBJ_RELEASE(lg); } /* unload the working buf to obtain the payload */ if (ORTE_SUCCESS != (rc = opal_dss.unload(buf, (void**)&bytes, &sz))) { ORTE_ERROR_LOG(rc); goto CLEANUP; } OPAL_OUTPUT_VERBOSE((2, orte_rmcast_base.rmcast_output, "%s rmcast:udp multicasting %d bytes to network %03d.%03d.%03d.%03d port %d tag %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), sz, OPAL_IF_FORMAT_ADDR(chan->network), (int)chan->port, (int)snd->tag)); if (sz != (rc = sendto(chan->xmit, bytes, sz, 0, (struct sockaddr *)&(chan->addr), sizeof(struct sockaddr_in)))) { /* didn't get the message out */ opal_output(0, "%s failed to send message to multicast network %03d.%03d.%03d.%03d on\n\terror %s(%d)", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), OPAL_IF_FORMAT_ADDR(chan->network), strerror(errno), errno); rc = ORTE_ERR_COMM_FAILURE; } else { rc = ORTE_SUCCESS; } if (NULL != snd->buf) { /* call the cbfunc if required */ if (NULL != snd->cbfunc_buffer) { snd->cbfunc_buffer(rc, chan->channel, chan->seq_num, snd->tag, ORTE_PROC_MY_NAME, snd->buf, snd->cbdata); } } else { /* call the cbfunc if required */ if (NULL != snd->cbfunc_iovec) { snd->cbfunc_iovec(rc, chan->channel, chan->seq_num, snd->tag, ORTE_PROC_MY_NAME, snd->iovec_array, snd->iovec_count, snd->cbdata); } } CLEANUP: if (NULL != buf) { OBJ_RELEASE(buf); } if (NULL != bytes) { free(bytes); } return rc; }
static void sample(orcm_sensor_sampler_t *sampler) { opal_pstats_t *stats; opal_node_stats_t *nstats; int rc, i; orte_proc_t *child; opal_buffer_t buf, *bptr; char *comp; OPAL_OUTPUT_VERBOSE((1, orcm_sensor_base_framework.framework_output, "sample:resusage sampling resource usage")); /* setup a buffer for our stats */ OBJ_CONSTRUCT(&buf, opal_buffer_t); /* pack our name */ comp = strdup("resusage"); if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &comp, 1, OPAL_STRING))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(&buf); return; } free(comp); /* update stats on ourself and the node */ stats = OBJ_NEW(opal_pstats_t); nstats = OBJ_NEW(opal_node_stats_t); if (ORCM_SUCCESS != (rc = opal_pstat.query(orte_process_info.pid, stats, nstats))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(stats); OBJ_RELEASE(nstats); OBJ_DESTRUCT(&buf); return; } /* the stats framework can't know nodename or rank */ strncpy(stats->node, orte_process_info.nodename, (OPAL_PSTAT_MAX_STRING_LEN - 1)); stats->rank = ORTE_PROC_MY_NAME->vpid; #if 0 /* locally save the stats */ if (NULL != (st = (opal_pstats_t*)opal_ring_buffer_push(&my_proc->stats, stats))) { OBJ_RELEASE(st); } if (NULL != (nst = (opal_node_stats_t*)opal_ring_buffer_push(&my_node->stats, nstats))) { /* release the popped value */ OBJ_RELEASE(nst); } #endif /* pack them */ if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &orte_process_info.nodename, 1, OPAL_STRING))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(&buf); return; } if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &nstats, 1, OPAL_NODE_STAT))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(&buf); return; } if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &stats, 1, OPAL_PSTAT))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(&buf); return; } /* loop through our children and update their stats */ if (NULL != orte_local_children) { for (i=0; i < orte_local_children->size; i++) { if (NULL == (child = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i))) { continue; } if (!ORTE_FLAG_TEST(child, ORTE_PROC_FLAG_ALIVE)) { continue; } if (0 == child->pid) { /* race condition */ continue; } stats = OBJ_NEW(opal_pstats_t); if (ORCM_SUCCESS != opal_pstat.query(child->pid, stats, NULL)) { /* may hit a race condition where the process has * terminated, so just ignore any error */ OBJ_RELEASE(stats); continue; } /* the stats framework can't know nodename or rank */ strncpy(stats->node, orte_process_info.nodename, (OPAL_PSTAT_MAX_STRING_LEN - 1)); stats->rank = child->name.vpid; #if 0 /* store it */ if (NULL != (st = (opal_pstats_t*)opal_ring_buffer_push(&child->stats, stats))) { OBJ_RELEASE(st); } #endif /* pack them */ if (OPAL_SUCCESS != (rc = opal_dss.pack(&buf, &stats, 1, OPAL_PSTAT))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(&buf); return; } } } /* xfer any data for transmission */ if (0 < buf.bytes_used) { bptr = &buf; if (OPAL_SUCCESS != (rc = opal_dss.pack(&sampler->bucket, &bptr, 1, OPAL_BUFFER))) { ORTE_ERROR_LOG(rc); OBJ_DESTRUCT(&buf); return; } } OBJ_DESTRUCT(&buf); #if 0 /* are there any issues with node-level usage? */ nst = (opal_node_stats_t*)opal_ring_buffer_poke(&my_node->stats, -1); if (NULL != nst && 0.0 < mca_sensor_resusage_component.node_memory_limit) { OPAL_OUTPUT_VERBOSE((2, orcm_sensor_base_framework.framework_output, "%s CHECKING NODE MEM", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* compute the percentage of node memory in-use */ in_use = 1.0 - (nst->free_mem / nst->total_mem); OPAL_OUTPUT_VERBOSE((2, orcm_sensor_base_framework.framework_output, "%s PERCENT USED: %f LIMIT: %f", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), in_use, mca_sensor_resusage_component.node_memory_limit)); if (mca_sensor_resusage_component.node_memory_limit <= in_use) { /* loop through our children and find the biggest hog */ hog = NULL; max_mem = 0.0; for (i=0; i < orte_local_children->size; i++) { if (NULL == (child = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i))) { continue; } if (!ORTE_FLAG_TEST(child, ORTE_PROC_IS_ALIVE)) { continue; } if (0 == child->pid) { /* race condition */ continue; } if (NULL == (st = (opal_pstats_t*)opal_ring_buffer_poke(&child->stats, -1))) { continue; } OPAL_OUTPUT_VERBOSE((5, orcm_sensor_base_framework.framework_output, "%s PROC %s AT VSIZE %f", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&child->name), st->vsize)); if (max_mem < st->vsize) { hog = child; max_mem = st->vsize; } } if (NULL == hog) { /* if all children dead and we are still too big, * then we must be the culprit - abort */ OPAL_OUTPUT_VERBOSE((2, orcm_sensor_base_framework.framework_output, "%s NO CHILD: COMMITTING SUICIDE", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); orte_errmgr.abort(ORCM_ERR_MEM_LIMIT_EXCEEDED, NULL); } else { /* report the problem */ OPAL_OUTPUT_VERBOSE((2, orcm_sensor_base_framework.framework_output, "%s REPORTING %s TO ERRMGR FOR EXCEEDING LIMITS", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&hog->name))); ORTE_ACTIVATE_PROC_STATE(&hog->name, ORTE_PROC_STATE_SENSOR_BOUND_EXCEEDED); } /* since we have ordered someone to die, we've done enough for this * time around - don't check proc limits as well */ return; } } /* check proc limits */ if (0.0 < mca_sensor_resusage_component.proc_memory_limit) { OPAL_OUTPUT_VERBOSE((2, orcm_sensor_base_framework.framework_output, "%s CHECKING PROC MEM", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* check my children first */ for (i=0; i < orte_local_children->size; i++) { if (NULL == (child = (orte_proc_t*)opal_pointer_array_get_item(orte_local_children, i))) { continue; } if (!ORTE_FLAG_TEST(child, ORTE_PROC_IS_ALIVE)) { continue; } if (0 == child->pid) { /* race condition */ continue; } if (NULL == (st = (opal_pstats_t*)opal_ring_buffer_poke(&child->stats, -1))) { continue; } OPAL_OUTPUT_VERBOSE((5, orcm_sensor_base_framework.framework_output, "%s PROC %s AT VSIZE %f", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&child->name), st->vsize)); if (mca_sensor_resusage_component.proc_memory_limit <= st->vsize) { /* report the problem */ ORTE_ACTIVATE_PROC_STATE(&child->name, ORTE_PROC_STATE_SENSOR_BOUND_EXCEEDED); } } } #endif }