int mca_btl_openib_get( mca_btl_base_module_t* btl, mca_btl_base_endpoint_t* endpoint, mca_btl_base_descriptor_t* descriptor) { int rc; struct ibv_send_wr* bad_wr; mca_btl_openib_frag_t* frag = (mca_btl_openib_frag_t*) descriptor; mca_btl_openib_module_t* openib_btl = (mca_btl_openib_module_t*) btl; frag->endpoint = endpoint; frag->wr_desc.sr_desc.opcode = IBV_WR_RDMA_READ; /* check for a send wqe */ if (OPAL_THREAD_ADD32(&endpoint->sd_wqe[BTL_OPENIB_LP_QP],-1) < 0) { OPAL_THREAD_ADD32(&endpoint->sd_wqe[BTL_OPENIB_LP_QP],1); OPAL_THREAD_LOCK(&endpoint->endpoint_lock); opal_list_append(&endpoint->pending_get_frags, (opal_list_item_t*)frag); OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock); return OMPI_SUCCESS; /* check for a get token */ } else if(OPAL_THREAD_ADD32(&endpoint->get_tokens,-1) < 0) { OPAL_THREAD_ADD32(&endpoint->sd_wqe[BTL_OPENIB_LP_QP],1); OPAL_THREAD_ADD32(&endpoint->get_tokens,1); OPAL_THREAD_LOCK(&endpoint->endpoint_lock); opal_list_append(&endpoint->pending_get_frags, (opal_list_item_t*)frag); OPAL_THREAD_UNLOCK(&endpoint->endpoint_lock); return OMPI_SUCCESS; } else { frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED; frag->wr_desc.sr_desc.wr.rdma.remote_addr = frag->base.des_src->seg_addr.lval; frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_src->seg_key.key32[0]; frag->sg_entry.addr = (unsigned long) frag->base.des_dst->seg_addr.pval; frag->sg_entry.length = frag->base.des_dst->seg_len; if(ibv_post_send(endpoint->lcl_qp[BTL_OPENIB_LP_QP], &frag->wr_desc.sr_desc, &bad_wr)){ BTL_ERROR(("error posting send request errno (%d) says %s", errno, strerror(errno))); rc = ORTE_ERROR; } else { rc = ORTE_SUCCESS; } if(mca_btl_openib_component.use_srq) { mca_btl_openib_post_srr(openib_btl, 1, BTL_OPENIB_HP_QP); mca_btl_openib_post_srr(openib_btl, 1, BTL_OPENIB_LP_QP); } else { btl_openib_endpoint_post_rr(endpoint, 1, BTL_OPENIB_HP_QP); btl_openib_endpoint_post_rr(endpoint, 1, BTL_OPENIB_LP_QP); } } return rc; }
/* * Called when the connect module has created all the qp's on an * endpoint and needs to have some receive buffers posted. */ int mca_btl_openib_endpoint_post_recvs(mca_btl_openib_endpoint_t *endpoint) { int qp; for (qp = 0; qp < mca_btl_openib_component.num_qps; ++qp) { if (BTL_OPENIB_QP_TYPE_PP(qp)) { mca_btl_openib_endpoint_post_rr_nolock(endpoint, qp); } else { mca_btl_openib_post_srr(endpoint->endpoint_btl, qp); } } return OPAL_SUCCESS; }