static void rcd_allgather_process_data(scon_collectives_tracker_t *coll,
uint32_t distance)
{
/* Communication step:
At every step i, rank r:
- exchanges message containing all data collected so far with rank peer = (r ^ 2^i).
*/
uint32_t log2nprocs = (uint32_t) log2(coll->sig->nprocs);
scon_proc_t peer;
int rc, peer_index;
while (distance < log2nprocs) {
scon_output_verbose(2, scon_collectives_base_framework.framework_output,
"%s rcd: allgather process distance =%d",
SCON_PRINT_PROC(SCON_PROC_MY_NAME),
distance);
/* first send my current contents */
peer_index = coll->my_rank ^ (1 << distance);
assert (peer_index < (int)coll->sig->nprocs);
peer = coll->sig->procs[peer_index];
rcd_allgather_send_dist(coll, &peer, distance);
/* check whether data for next distance is available */
if (NULL == coll->buffers || NULL == coll->buffers[distance]) {
break;
}
scon_output_verbose(2, scon_collectives_base_framework.framework_output,
"%s rcd: allgather DATA found for distance =%d",
SCON_PRINT_PROC(SCON_PROC_MY_NAME),
distance);
if (SCON_SUCCESS != (rc = scon_bfrop.copy_payload(&coll->bucket, coll->buffers[distance]))) {
SCON_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
coll->nreported += 1 << distance;
scon_collectives_base_mark_distance_recv(coll, distance);
SCON_RELEASE(coll->buffers[distance]);
coll->buffers[distance] = NULL;
++distance;
}
scon_output_verbose(2, scon_collectives_base_framework.framework_output,
"%s collectives rcd allgather reported %lu process from %lu",
SCON_PRINT_PROC(SCON_PROC_MY_NAME), (unsigned long)coll->nreported,
(unsigned long)coll->sig->nprocs);
/* if we are done, then complete things */
if (coll->nreported == coll->sig->nprocs) {
rcd_finalize_coll(coll, SCON_SUCCESS);
}
}
static void rcd_allgather_recv_dist(int status, orte_process_name_t* sender,
opal_buffer_t* buffer, orte_rml_tag_t tag,
void* cbdata)
{
int32_t cnt, num_remote;
int rc;
orte_grpcomm_signature_t *sig;
orte_grpcomm_coll_t *coll;
orte_vpid_t distance, new_distance;
OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base_framework.framework_output,
"%s grpcomm:coll:recdub received data",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* unpack the signature */
cnt = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &sig, &cnt, ORTE_SIGNATURE))) {
ORTE_ERROR_LOG(rc);
return;
}
/* check for the tracker and create it if not found */
if (NULL == (coll = orte_grpcomm_base_get_tracker(sig, true))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
OBJ_RELEASE(sig);
return;
}
/* unpack the distance */
distance = 1;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &distance, &cnt, OPAL_INT32))) {
OBJ_RELEASE(sig);
ORTE_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
/* unpack number of reported */
num_remote = 0;
if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &num_remote, &cnt, OPAL_INT32))) {
OBJ_RELEASE(sig);
ORTE_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
coll->nreported += num_remote;
/* capture any provided content */
if (OPAL_SUCCESS != (rc = opal_dss.copy_payload(&coll->bucket, buffer))) {
OBJ_RELEASE(sig);
ORTE_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
//update distance and send
new_distance = distance <<= 1;
if (new_distance < coll->ndmns) {
rcd_allgather_send_dist(coll, new_distance);
} else {
rcd_finalize_coll(coll, ORTE_SUCCESS);
}
OBJ_RELEASE(sig);
return;
}
static void rcd_allgather_recv_dist(int status,
scon_handle_t handle,
scon_proc_t* sender,
scon_buffer_t* buffer,
scon_msg_tag_t tag,
void* cbdata)
{
int32_t cnt;
int rc;
scon_collectives_signature_t *sig;
scon_collectives_tracker_t *coll;
uint32_t distance;
scon_output_verbose(2, scon_collectives_base_framework.framework_output,
"%s rcd: allgather receiving from %s",
SCON_PRINT_PROC(SCON_PROC_MY_NAME),
SCON_PRINT_PROC(sender));
/* unpack the signature */
cnt = 1;
if (SCON_SUCCESS != (rc = scon_bfrop.unpack(buffer, &sig, &cnt, SCON_COLLECTIVES_SIGNATURE))) {
SCON_ERROR_LOG(rc);
return;
}
/* check for the tracker and create it if not found */
if (NULL == (coll = scon_collectives_base_get_tracker (sig, true))) {
SCON_ERROR_LOG(SCON_ERR_NOT_FOUND);
SCON_RELEASE(sig);
return;
}
/* unpack the distance */
distance = -1;
cnt = 1;
if (SCON_SUCCESS != (rc = scon_bfrop.unpack(buffer, &distance, &cnt, SCON_INT32))) {
SCON_RELEASE(sig);
SCON_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
assert(0 == scon_collectives_base_check_distance_recv(coll, distance));
/* Check whether we can process next distance */
if (coll->nreported && (!distance || scon_collectives_base_check_distance_recv(coll, distance - 1))) {
scon_output_verbose(2, scon_collectives_base_framework.framework_output,
"%s grpcomm:coll:rcd data from %d distance received, "
"Process the next distance.",
SCON_PRINT_PROC(SCON_PROC_MY_NAME), distance);
/* capture any provided content */
if (SCON_SUCCESS != (rc = scon_bfrop.copy_payload(&coll->bucket, buffer))) {
SCON_RELEASE(sig);
SCON_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
coll->nreported += (1 << distance);
scon_collectives_base_mark_distance_recv(coll, distance);
rcd_allgather_process_data(coll, distance + 1);
} else {
scon_output_verbose(2, scon_collectives_base_framework.framework_output,
"%s grpcomm:coll:rcd data from %d distance received, "
"still waiting for data.",
SCON_PRINT_PROC(SCON_PROC_MY_NAME), distance);
if (NULL == coll->buffers) {
if (NULL == (coll->buffers = (scon_buffer_t **) calloc ((uint32_t) log2 (coll->sig->nprocs) , sizeof(scon_buffer_t *)))) {
rc = SCON_ERR_OUT_OF_RESOURCE;
SCON_RELEASE(sig);
SCON_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
}
//if (NULL == (coll->buffers[distance] = SCON_NEW(scon_buffer_t))) {
if (NULL == (coll->buffers[distance] = (scon_buffer_t*) malloc (sizeof(scon_buffer_t)))) {
rc = SCON_ERR_OUT_OF_RESOURCE;
SCON_RELEASE(sig);
SCON_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
if (SCON_SUCCESS != (rc = scon_bfrop.copy_payload(coll->buffers[distance], buffer))) {
SCON_RELEASE(sig);
SCON_ERROR_LOG(rc);
rcd_finalize_coll(coll, rc);
return;
}
}
SCON_RELEASE(sig);
}
