int mca_spml_ikrit_get(shmem_ctx_t ctx, void *src_addr, size_t size, void *dst_addr, int src)
{
mxm_send_req_t sreq;
if (0 >= size) {
return OSHMEM_SUCCESS;
}
if (OSHMEM_SUCCESS == mca_spml_ikrit_get_shm(src_addr, size, dst_addr, src))
return OSHMEM_SUCCESS;
if (OSHMEM_SUCCESS
!= mca_spml_ikrit_get_helper(&sreq,
src_addr,
size,
dst_addr,
src)) {
oshmem_shmem_abort(-1);
return OSHMEM_ERROR;
}
sreq.base.completed_cb = NULL;
sreq.flags = 0;
SPML_IKRIT_MXM_POST_SEND(sreq);
mca_spml_irkit_req_wait(&sreq.base);
if (MXM_OK != sreq.base.error) {
SPML_ERROR("get request failed: %s - aborting",
mxm_error_string(sreq.base.error));
oshmem_shmem_abort(-1);
return OSHMEM_ERROR;
}
return OSHMEM_SUCCESS;
}
/* for now only do blocking copy send */
int mca_spml_ikrit_send(void* buf,
size_t size,
int dst,
mca_spml_base_put_mode_t mode)
{
mxm_send_req_t req;
char dummy_buf[1];
SPML_VERBOSE(100,
"sending %p size %d to %d, mode %d",
buf, (int)size, dst, (int)mode);
req.opcode = MXM_REQ_OP_SEND;
req.op.send.tag = oshmem_my_proc_id();
req.base.state = MXM_REQ_NEW;
req.base.mq = mca_spml_ikrit.mxm_mq;
req.base.conn = mca_spml_ikrit.mxm_peers[dst].mxm_conn;
req.flags = MXM_REQ_SEND_FLAG_BLOCKING;
req.base.completed_cb = NULL;
req.base.data_type = MXM_REQ_DATA_BUFFER;
req.base.data.buffer.ptr = buf == NULL ? dummy_buf : buf;
req.base.data.buffer.length = size == 0 ? sizeof(dummy_buf) : size;
req.base.data.buffer.memh = NULL;
SPML_IKRIT_MXM_POST_SEND(req);
mca_spml_irkit_req_wait(&req.base);
if (req.base.error != MXM_OK) {
return OSHMEM_ERROR;
}
return OSHMEM_SUCCESS;
}
/* blocking receive */
int mca_spml_ikrit_recv(void* buf, size_t size, int src)
{
mxm_error_t ret = MXM_OK;
mxm_recv_req_t req;
char dummy_buf[1];
/* tag mask 0 matches any tag */
SPML_VERBOSE(100,
"want to recv from src %d, size %d buf %p",
src, (int)size, buf);
req.tag = src == SHMEM_ANY_SOURCE ? 0 : src;
req.tag_mask = src == SHMEM_ANY_SOURCE ? 0 : 0xFFFFFFFF;
req.base.state = MXM_REQ_NEW;
req.base.mq = mca_spml_ikrit.mxm_mq;
req.base.conn = NULL;
#if MXM_API < MXM_VERSION(2,0)
req.base.flags = MXM_REQ_FLAG_BLOCKING;
#endif
req.base.completed_cb = NULL;
req.base.data_type = MXM_REQ_DATA_BUFFER;
req.base.data.buffer.ptr = buf == NULL ? dummy_buf : buf;
req.base.data.buffer.length = size == 0 ? sizeof(dummy_buf) : size;
req.base.data.buffer.memh = NULL;
ret = mxm_req_recv(&req);
if (MXM_OK != ret) {
return OSHMEM_ERROR;
}
mca_spml_irkit_req_wait(&req.base);
if (MXM_OK != req.base.error) {
return OSHMEM_ERROR;
}
SPML_VERBOSE(100,
"recvd from tag %d len %d",
req.completion.sender_tag, (int)req.completion.actual_len);
return OSHMEM_SUCCESS;
}
int mca_spml_ikrit_get(void *src_addr, size_t size, void *dst_addr, int src)
{
mxm_send_req_t sreq;
if (0 >= size) {
return OSHMEM_SUCCESS;
}
if (OSHMEM_SUCCESS == mca_spml_ikrit_get_shm(src_addr, size, dst_addr, src))
return OSHMEM_SUCCESS;
if (OSHMEM_SUCCESS
!= mca_spml_ikrit_get_helper(&sreq,
src_addr,
size,
dst_addr,
src)) {
oshmem_shmem_abort(-1);
return OSHMEM_ERROR;
}
#if MXM_API < MXM_VERSION(2,0)
sreq.base.flags = MXM_REQ_FLAG_BLOCKING;
#else
sreq.flags = MXM_REQ_SEND_FLAG_BLOCKING;
#endif
sreq.base.completed_cb = NULL;
mxm_req_send(&sreq);
opal_progress();
mca_spml_irkit_req_wait(&sreq.base);
if (MXM_OK != sreq.base.error) {
SPML_ERROR("get request failed: %s - aborting",
mxm_error_string(sreq.base.error));
oshmem_shmem_abort(-1);
return OSHMEM_ERROR;
}
return OSHMEM_SUCCESS;
}