static int allgather(orte_grpcomm_coll_t *coll, opal_buffer_t *sendbuf) { OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base_framework.framework_output, "%s grpcomm:coll:recdub algo employed for %d processes", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (int)coll->ndmns)); /* if we only have one proc participating, just copy the data across and return */ if (!((coll->ndmns != 0) && ((coll->ndmns & (coll->ndmns - 1)) == 0))) { OPAL_OUTPUT((orte_grpcomm_base_framework.framework_output, "%s grpcomm:coll:recdub number of participating daemons (%d) is not power 2", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), (int)coll->ndmns )); return ORTE_ERROR; } /* start by seeding the collection with our own data */ opal_dss.copy_payload(&coll->bucket, sendbuf); /* Communication step: At every step i, rank r: - exchanges message containing all data collected so far with rank peer = (r ^ 2^i). */ rcd_allgather_send_dist(coll, 1); return ORTE_SUCCESS; }
static void rcd_allgather_process_data(scon_collectives_tracker_t *coll, uint32_t distance) { /* Communication step: At every step i, rank r: - exchanges message containing all data collected so far with rank peer = (r ^ 2^i). */ uint32_t log2nprocs = (uint32_t) log2(coll->sig->nprocs); scon_proc_t peer; int rc, peer_index; while (distance < log2nprocs) { scon_output_verbose(2, scon_collectives_base_framework.framework_output, "%s rcd: allgather process distance =%d", SCON_PRINT_PROC(SCON_PROC_MY_NAME), distance); /* first send my current contents */ peer_index = coll->my_rank ^ (1 << distance); assert (peer_index < (int)coll->sig->nprocs); peer = coll->sig->procs[peer_index]; rcd_allgather_send_dist(coll, &peer, distance); /* check whether data for next distance is available */ if (NULL == coll->buffers || NULL == coll->buffers[distance]) { break; } scon_output_verbose(2, scon_collectives_base_framework.framework_output, "%s rcd: allgather DATA found for distance =%d", SCON_PRINT_PROC(SCON_PROC_MY_NAME), distance); if (SCON_SUCCESS != (rc = scon_bfrop.copy_payload(&coll->bucket, coll->buffers[distance]))) { SCON_ERROR_LOG(rc); rcd_finalize_coll(coll, rc); return; } coll->nreported += 1 << distance; scon_collectives_base_mark_distance_recv(coll, distance); SCON_RELEASE(coll->buffers[distance]); coll->buffers[distance] = NULL; ++distance; } scon_output_verbose(2, scon_collectives_base_framework.framework_output, "%s collectives rcd allgather reported %lu process from %lu", SCON_PRINT_PROC(SCON_PROC_MY_NAME), (unsigned long)coll->nreported, (unsigned long)coll->sig->nprocs); /* if we are done, then complete things */ if (coll->nreported == coll->sig->nprocs) { rcd_finalize_coll(coll, SCON_SUCCESS); } }
static void rcd_allgather_recv_dist(int status, orte_process_name_t* sender, opal_buffer_t* buffer, orte_rml_tag_t tag, void* cbdata) { int32_t cnt, num_remote; int rc; orte_grpcomm_signature_t *sig; orte_grpcomm_coll_t *coll; orte_vpid_t distance, new_distance; OPAL_OUTPUT_VERBOSE((5, orte_grpcomm_base_framework.framework_output, "%s grpcomm:coll:recdub received data", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME))); /* unpack the signature */ cnt = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &sig, &cnt, ORTE_SIGNATURE))) { ORTE_ERROR_LOG(rc); return; } /* check for the tracker and create it if not found */ if (NULL == (coll = orte_grpcomm_base_get_tracker(sig, true))) { ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND); OBJ_RELEASE(sig); return; } /* unpack the distance */ distance = 1; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &distance, &cnt, OPAL_INT32))) { OBJ_RELEASE(sig); ORTE_ERROR_LOG(rc); rcd_finalize_coll(coll, rc); return; } /* unpack number of reported */ num_remote = 0; if (OPAL_SUCCESS != (rc = opal_dss.unpack(buffer, &num_remote, &cnt, OPAL_INT32))) { OBJ_RELEASE(sig); ORTE_ERROR_LOG(rc); rcd_finalize_coll(coll, rc); return; } coll->nreported += num_remote; /* capture any provided content */ if (OPAL_SUCCESS != (rc = opal_dss.copy_payload(&coll->bucket, buffer))) { OBJ_RELEASE(sig); ORTE_ERROR_LOG(rc); rcd_finalize_coll(coll, rc); return; } //update distance and send new_distance = distance <<= 1; if (new_distance < coll->ndmns) { rcd_allgather_send_dist(coll, new_distance); } else { rcd_finalize_coll(coll, ORTE_SUCCESS); } OBJ_RELEASE(sig); return; }