int memheap_oob_init(mca_memheap_map_t *map)
{
int rc = OSHMEM_SUCCESS;
int i;
oob_comm_request_t *r;
memheap_map = map;
OBJ_CONSTRUCT(&memheap_oob.lck, opal_mutex_t);
OBJ_CONSTRUCT(&memheap_oob.cond, opal_condition_t);
OBJ_CONSTRUCT(&memheap_oob.req_list, opal_list_t);
for (i = 0; i < MEMHEAP_RECV_REQS_MAX; i++) {
r = &memheap_oob.req_pool[i];
rc = PMPI_Recv_init(r->buf, sizeof(r->buf), MPI_BYTE,
MPI_ANY_SOURCE, 0,
oshmem_comm_world,
&r->recv_req);
if (MPI_SUCCESS != rc) {
MEMHEAP_ERROR("Failed to created recv request %d", rc);
return rc;
}
rc = PMPI_Start(&r->recv_req);
if (MPI_SUCCESS != rc) {
MEMHEAP_ERROR("Failed to post recv request %d", rc);
return rc;
}
opal_list_append(&memheap_oob.req_list, &r->super);
}
opal_progress_register(oshmem_mkey_recv_cb);
memheap_oob.is_inited = 1;
return rc;
}
static int oshmem_mkey_recv_cb(void)
{
MPI_Status status;
int flag;
int n;
int rc;
opal_buffer_t *msg;
int32_t size;
void *tmp_buf;
oob_comm_request_t *r;
n = 0;
r = (oob_comm_request_t *)opal_list_get_first(&memheap_oob.req_list);
assert(r);
while(r != (oob_comm_request_t *)opal_list_get_end(&memheap_oob.req_list)) {
my_MPI_Test(&r->recv_req, &flag, &status);
if (OPAL_LIKELY(0 == flag)) {
return n;
}
PMPI_Get_count(&status, MPI_BYTE, &size);
MEMHEAP_VERBOSE(5, "OOB request from PE: %d, size %d", status.MPI_SOURCE, size);
n++;
opal_list_remove_first(&memheap_oob.req_list);
/* to avoid deadlock we must start request
* before processing it. Data are copied to
* the tmp buffer
*/
tmp_buf = malloc(size);
if (NULL == tmp_buf) {
MEMHEAP_ERROR("not enough memory");
ORTE_ERROR_LOG(0);
return n;
} else {
memcpy(tmp_buf, (void*)&r->buf, size);
msg = OBJ_NEW(opal_buffer_t);
if (NULL == msg) {
MEMHEAP_ERROR("not enough memory");
ORTE_ERROR_LOG(0);
free(tmp_buf);
return n;
}
opal_dss.load(msg, (void*)tmp_buf, size);
/*
* send reply before posting the receive request again to limit the recursion size to
* number of receive requests.
* send can call opal_progress which calls this function again. If recv req is started
* stack size will be proportional to number of job ranks.
*/
do_recv(status.MPI_SOURCE, msg);
OBJ_RELEASE(msg);
}
rc = PMPI_Start(&r->recv_req);
if (MPI_SUCCESS != rc) {
MEMHEAP_ERROR("Failed to post recv request %d", rc);
ORTE_ERROR_LOG(rc);
return n;
}
opal_list_append(&memheap_oob.req_list, &r->super);
r = (oob_comm_request_t *)opal_list_get_first(&memheap_oob.req_list);
assert(r);
}
return 1;
}
int MPI_Start(MPI_Request * request)
{
return PMPI_Start(request);
}
int MPI_Start(
MPI_Request *request)
{
_MPI_COVERAGE();
return PMPI_Start (request);
}
