Exemple #1
0
static int mca_btl_mvapi_endpoint_send_eager_rdma(
    mca_btl_base_endpoint_t* endpoint)
{
    mca_btl_mvapi_module_t* mvapi_btl = endpoint->endpoint_btl;
    mca_btl_mvapi_eager_rdma_header_t *rdma_hdr;
    mca_btl_mvapi_frag_t* frag;
    int rc;

    MCA_BTL_IB_FRAG_ALLOC_EAGER(mvapi_btl, frag, rc);
    if(NULL == frag) {
        BTL_ERROR(("error allocating fragment"));
        return -1;
    }

    frag->base.des_cbfunc = mca_btl_mvapi_endpoint_eager_rdma;
    frag->base.des_cbdata = NULL;
    frag->endpoint = endpoint;
    frag->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY;

    frag->hdr->tag = MCA_BTL_TAG_BTL;
    rdma_hdr = (mca_btl_mvapi_eager_rdma_header_t*)frag->segment.seg_addr.pval;
    rdma_hdr->control.type = MCA_BTL_MVAPI_CONTROL_RDMA;
    rdma_hdr->rkey = endpoint->eager_rdma_local.reg->r_key;
    rdma_hdr->rdma_start.lval = ompi_ptr_ptol(endpoint->eager_rdma_local.base.pval);
    frag->segment.seg_len = sizeof(mca_btl_mvapi_eager_rdma_header_t);
    if (mca_btl_mvapi_endpoint_post_send(mvapi_btl, endpoint, frag) !=
            OMPI_SUCCESS) {
        MCA_BTL_IB_FRAG_RETURN(mvapi_btl, frag);
        BTL_ERROR(("Error sending RDMA buffer", strerror(errno)));
        return -1;
    }
    return 0;
}
/* send the eager rdma connect message to the remote endpoint */
static int mca_btl_wv_endpoint_send_eager_rdma(
    mca_btl_base_endpoint_t* endpoint)
{
    mca_btl_wv_module_t* wv_btl = endpoint->endpoint_btl;
    mca_btl_wv_eager_rdma_header_t *rdma_hdr;
    mca_btl_wv_send_control_frag_t* frag;
    int rc;

    frag = alloc_control_frag(wv_btl);
    if(NULL == frag) {
        return -1;
    }

    to_base_frag(frag)->base.des_cbfunc =
        mca_btl_wv_endpoint_eager_rdma_connect_cb;
    to_base_frag(frag)->base.des_cbdata = NULL;
    to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
    to_base_frag(frag)->base.order = mca_btl_wv_component.credits_qp;
    to_base_frag(frag)->segment.base.seg_len =
        sizeof(mca_btl_wv_eager_rdma_header_t);
    to_com_frag(frag)->endpoint = endpoint;

    frag->hdr->tag = MCA_BTL_TAG_BTL;
    rdma_hdr = (mca_btl_wv_eager_rdma_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval;
    rdma_hdr->control.type = MCA_BTL_WV_CONTROL_RDMA;
    rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey;
    rdma_hdr->rdma_start.lval = ompi_ptr_ptol(endpoint->eager_rdma_local.base.pval);
    BTL_VERBOSE(("sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64 
                 ", pval %p, ival %" PRIu32 " type %d and sizeof(rdma_hdr) %d\n",
                 rdma_hdr->rkey,
                 rdma_hdr->rdma_start.lval,
                 rdma_hdr->rdma_start.pval,
                 rdma_hdr->rdma_start.ival,
                 rdma_hdr->control.type,
                 (int) sizeof(mca_btl_wv_eager_rdma_header_t)
                 ));

    if(endpoint->nbo) {
        BTL_WV_EAGER_RDMA_CONTROL_HEADER_HTON((*rdma_hdr));

        BTL_VERBOSE(("after HTON: sending rkey %" PRIu32 ", rdma_start.lval %" PRIx64 ", pval %p, ival %" PRIu32 "\n",
                     rdma_hdr->rkey,
                     rdma_hdr->rdma_start.lval,
                     rdma_hdr->rdma_start.pval,
                     rdma_hdr->rdma_start.ival
                     ));
    }
    rc = mca_btl_wv_endpoint_send(endpoint, frag);
    if (OMPI_SUCCESS == rc || OMPI_ERR_RESOURCE_BUSY == rc)
        return OMPI_SUCCESS;

    MCA_BTL_IB_FRAG_RETURN(frag);
    BTL_ERROR(("Error sending RDMA buffer: %s", strerror(errno)));
    return rc;
}