static int mca_btl_mvapi_endpoint_send_eager_rdma( mca_btl_base_endpoint_t* endpoint) { mca_btl_mvapi_module_t* mvapi_btl = endpoint->endpoint_btl; mca_btl_mvapi_eager_rdma_header_t *rdma_hdr; mca_btl_mvapi_frag_t* frag; int rc; MCA_BTL_IB_FRAG_ALLOC_EAGER(mvapi_btl, frag, rc); if(NULL == frag) { BTL_ERROR(("error allocating fragment")); return -1; } frag->base.des_cbfunc = mca_btl_mvapi_endpoint_eager_rdma; frag->base.des_cbdata = NULL; frag->endpoint = endpoint; frag->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY; frag->hdr->tag = MCA_BTL_TAG_BTL; rdma_hdr = (mca_btl_mvapi_eager_rdma_header_t*)frag->segment.seg_addr.pval; rdma_hdr->control.type = MCA_BTL_MVAPI_CONTROL_RDMA; rdma_hdr->rkey = endpoint->eager_rdma_local.reg->r_key; rdma_hdr->rdma_start.lval = ompi_ptr_ptol(endpoint->eager_rdma_local.base.pval); frag->segment.seg_len = sizeof(mca_btl_mvapi_eager_rdma_header_t); if (mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag) != OMPI_SUCCESS) { MCA_BTL_IB_FRAG_RETURN(mvapi_btl, frag); BTL_ERROR(("Error sending RDMA buffer", strerror(errno))); return -1; } return 0; }
/* send the eager rdma connect message to the remote endpoint */ static int mca_btl_wv_endpoint_send_eager_rdma( mca_btl_base_endpoint_t* endpoint) { mca_btl_wv_module_t* wv_btl = endpoint->endpoint_btl; mca_btl_wv_eager_rdma_header_t *rdma_hdr; mca_btl_wv_send_control_frag_t* frag; int rc; frag = alloc_control_frag(wv_btl); if(NULL == frag) { return -1; } to_base_frag(frag)->base.des_cbfunc = mca_btl_wv_endpoint_eager_rdma_connect_cb; to_base_frag(frag)->base.des_cbdata = NULL; to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK; to_base_frag(frag)->base.order = mca_btl_wv_component.credits_qp; to_base_frag(frag)->segment.base.seg_len = sizeof(mca_btl_wv_eager_rdma_header_t); to_com_frag(frag)->endpoint = endpoint; frag->hdr->tag = MCA_BTL_TAG_BTL; rdma_hdr = (mca_btl_wv_eager_rdma_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval; rdma_hdr->control.type = MCA_BTL_WV_CONTROL_RDMA; rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey; rdma_hdr->rdma_start.lval = ompi_ptr_ptol(endpoint->eager_rdma_local.base.pval); BTL_VERBOSE(("sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64 ", pval %p, ival %" PRIu32 " type %d and sizeof(rdma_hdr) %d\n", rdma_hdr->rkey, rdma_hdr->rdma_start.lval, rdma_hdr->rdma_start.pval, rdma_hdr->rdma_start.ival, rdma_hdr->control.type, (int) sizeof(mca_btl_wv_eager_rdma_header_t) )); if(endpoint->nbo) { BTL_WV_EAGER_RDMA_CONTROL_HEADER_HTON((*rdma_hdr)); BTL_VERBOSE(("after HTON: sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64 ", pval %p, ival %" PRIu32 "\n", rdma_hdr->rkey, rdma_hdr->rdma_start.lval, rdma_hdr->rdma_start.pval, rdma_hdr->rdma_start.ival )); } rc = mca_btl_wv_endpoint_send(endpoint, frag); if (OMPI_SUCCESS == rc || OMPI_ERR_RESOURCE_BUSY == rc) return OMPI_SUCCESS; MCA_BTL_IB_FRAG_RETURN(frag); BTL_ERROR(("Error sending RDMA buffer: %s", strerror(errno))); return rc; }