int ompi_mtl_mxm_isend(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm, int dest, int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode, bool blocking,
mca_mtl_request_t * mtl_request)
{
mca_mtl_mxm_request_t *mtl_mxm_request = (mca_mtl_mxm_request_t *)mtl_request;
mxm_send_req_t *mxm_send_req;
mxm_error_t err;
int ret;
assert(mtl == &ompi_mtl_mxm.super);
mtl_mxm_request->convertor = convertor;
ret = ompi_mtl_datatype_pack(mtl_mxm_request->convertor,
&mtl_mxm_request->buf,
&mtl_mxm_request->length,
&mtl_mxm_request->free_after);
if (OMPI_SUCCESS != ret) {
return ret;
}
mxm_send_req = &mtl_mxm_request->mxm.send;
/* prepare a send request embedded in the MTL request */
mxm_send_req->base.state = MXM_REQ_NEW;
mxm_send_req->base.mq = ompi_mtl_mxm_mq_lookup(comm);
mxm_send_req->base.conn = ompi_mtl_mxm_conn_lookup(comm, dest);
mxm_send_req->base.flags = 0;
mxm_send_req->base.data_type = MXM_REQ_DATA_BUFFER;
mxm_send_req->base.data.buffer.ptr = mtl_mxm_request->buf;
mxm_send_req->base.data.buffer.length = mtl_mxm_request->length;
mxm_send_req->base.data.buffer.mkey = MXM_MKEY_NONE;
mxm_send_req->base.context = mtl_mxm_request;
mxm_send_req->base.completed_cb = ompi_mtl_mxm_send_completion_cb;
if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS) {
mxm_send_req->base.flags |= MXM_REQ_FLAG_SEND_SYNC;
}
mxm_send_req->opcode = MXM_REQ_OP_SEND;
mxm_send_req->op.send.tag = tag;
mxm_send_req->op.send.imm_data = ompi_comm_rank(comm);
/* post-send */
err = mxm_req_send(mxm_send_req);
if (MXM_OK != err) {
orte_show_help("help-mtl-mxm.txt", "error posting send", true, 1, mxm_error_string(err));
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
int ompi_mtl_mxm_send(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t* comm, int dest, int tag,
struct opal_convertor_t *convertor,
mca_pml_base_send_mode_t mode)
{
mxm_send_req_t mxm_send_req;
mxm_wait_t wait;
mxm_error_t err;
int ret;
/* prepare local send request */
mxm_send_req.base.state = MXM_REQ_NEW;
mxm_send_req.base.mq = ompi_mtl_mxm_mq_lookup(comm);
mxm_send_req.base.conn = ompi_mtl_mxm_conn_lookup(comm, dest);
mxm_send_req.base.flags = MXM_REQ_FLAG_BLOCKING;
mxm_send_req.base.context = NULL;
ret = ompi_mtl_mxm_choose_send_datatype(&mxm_send_req, convertor);
if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) {
return ret;
}
mxm_send_req.base.data.buffer.mkey = MXM_MKEY_NONE;
mxm_send_req.base.completed_cb = NULL;
if (mode == MCA_PML_BASE_SEND_SYNCHRONOUS) {
mxm_send_req.base.flags |= MXM_REQ_FLAG_SEND_SYNC;
}
mxm_send_req.opcode = MXM_REQ_OP_SEND;
mxm_send_req.op.send.tag = tag;
mxm_send_req.op.send.imm_data = ompi_comm_rank(comm);
/* post-send */
err = mxm_req_send(&mxm_send_req);
if (MXM_OK != err) {
orte_show_help("help-mtl-mxm.txt", "error posting send", true, 0, mxm_error_string(err));
return OMPI_ERROR;
}
/* wait for request completion */
wait.req = &mxm_send_req.base;
wait.state = MXM_REQ_COMPLETED;
wait.progress_cb = ompi_mtl_mxm_send_progress_cb;
wait.progress_arg = NULL;
mxm_wait(&wait);
return OMPI_SUCCESS;
}
int ompi_mtl_mxm_irecv(struct mca_mtl_base_module_t* mtl,
struct ompi_communicator_t *comm, int src, int tag,
struct opal_convertor_t *convertor,
struct mca_mtl_request_t *mtl_request)
{
mca_mtl_mxm_request_t * mtl_mxm_request;
mxm_error_t err;
mxm_recv_req_t *mxm_recv_req;
int ret;
mtl_mxm_request = (mca_mtl_mxm_request_t*) mtl_request;
mtl_mxm_request->convertor = convertor;
ret = ompi_mtl_datatype_recv_buf(mtl_mxm_request->convertor,
&mtl_mxm_request->buf,
&mtl_mxm_request->length,
&mtl_mxm_request->free_after);
if (OMPI_SUCCESS != ret) {
return ret;
}
/* prepare a receive request embedded in the MTL request */
mxm_recv_req = &mtl_mxm_request->mxm.recv;
mxm_recv_req->base.state = MXM_REQ_NEW;
mxm_recv_req->base.mq = ompi_mtl_mxm_mq_lookup(comm);
mxm_recv_req->base.conn = (src == MPI_ANY_SOURCE) ? NULL :
ompi_mtl_mxm_conn_lookup(comm, src);
mxm_recv_req->base.flags = 0;
mxm_recv_req->base.data_type = MXM_REQ_DATA_BUFFER;
mxm_recv_req->base.data.buffer.ptr = mtl_mxm_request->buf;
mxm_recv_req->base.data.buffer.length = mtl_mxm_request->length;
mxm_recv_req->base.data.buffer.mkey = MXM_MKEY_NONE;
mxm_recv_req->base.context = mtl_mxm_request;
mxm_recv_req->base.completed_cb = ompi_mtl_mxm_recv_completion_cb;
mxm_recv_req->tag = tag;
mxm_recv_req->tag_mask = (tag == MPI_ANY_TAG) ? 0 : 0xffffffffU;
/* post-recv */
err = mxm_req_recv(mxm_recv_req);
if (MXM_OK != err) {
orte_show_help("help-mtl-mxm.txt", "error posting receive", true,
mxm_error_string(err), mtl_mxm_request->buf, mtl_mxm_request->length);
return OMPI_ERROR;
}
return OMPI_SUCCESS;
}
