int osmt_mtl_mad_post_recv_bufs(osmt_mtl_mad_res_t * res, void *buf_array,
u_int32_t num_o_bufs, u_int32_t size,
VAPI_wr_id_t start_id)
{
uint32_t i;
void *cur_buf;
VAPI_rr_desc_t rr;
VAPI_sg_lst_entry_t sg_entry;
VAPI_ret_t ret;
rr.opcode = VAPI_RECEIVE;
rr.comp_type = VAPI_SIGNALED; /* All with CQE (IB compliant) */
rr.sg_lst_len = 1; /* single buffers */
rr.sg_lst_p = &sg_entry;
sg_entry.lkey = res->l_key;
cur_buf = buf_array;
for (i = 0; i < num_o_bufs; i++) {
rr.id = start_id + i; /* WQE id used is the index to buffers ptr array */
ASSERT_VOIDP2UINTN(cur_buf);
sg_entry.addr = (VAPI_virt_addr_t) cur_buf;
sg_entry.len = size;
memset(cur_buf, 0x00, size); /* fill with 0 */
ret = VAPI_post_rr(res->hca_hndl, res->qp_hndl, &rr);
if (ret != VAPI_OK) {
MTL_ERROR1(__FUNCTION__
": failed posting RQ WQE (%s)\n",
VAPI_strerror_sym(ret));
return i;
}
MTL_DEBUG4(__FUNCTION__ ": posted buf at %p\n", cur_buf);
cur_buf += size;
}
return i; /* num of buffers posted */
}
void PrepareToReceive(ArgStruct *p)
{
VAPI_ret_t ret; /* Return code */
VAPI_rr_desc_t rr; /* Receive request */
VAPI_sg_lst_entry_t sg_entry; /* Scatter/Gather list - holds buff addr */
/* We don't need to post a receive if doing RDMA write with local polling */
if( p->prot.commtype == NP_COMM_RDMAWRITE &&
p->prot.comptype == NP_COMP_LOCALPOLL )
return;
rr.opcode = VAPI_RECEIVE;
/* We only need signaled completions if using VAPI
* completion methods.
*/
if( p->prot.comptype == NP_COMP_LOCALPOLL )
rr.comp_type = VAPI_UNSIGNALED;
else
rr.comp_type = VAPI_SIGNALED;
rr.sg_lst_len = 1;
rr.sg_lst_p = &sg_entry;
sg_entry.lkey = r_mr_out.l_key;
sg_entry.len = p->bufflen;
sg_entry.addr = (VAPI_virt_addr_t)(MT_virt_addr_t)p->r_ptr;
ret = VAPI_post_rr(hca_hndl, qp_hndl, &rr);
if(ret != VAPI_OK) {
fprintf(stderr, "Error posting recv request: %s\n", VAPI_strerror(ret));
CleanUp(p);
exit(-1);
} else {
LOGPRINTF("Posted recv request\n");
}
/* Set receive flag to zero and request event completion
* notification for this receive so the event handler will
* be triggered when the receive completes.
*/
if( p->prot.comptype == NP_COMP_EVENT ) {
receive_complete = 0;
}
}
static
int psib_recvdone(con_info_t *con_info)
{
psib_msg_t *msg;
VAPI_rr_desc_t rr_desc;
VAPI_sg_lst_entry_t sg_lst;
VAPI_ret_t rc;
msg = ((psib_msg_t *)con_info->recv_bufs.ptr) + con_info->recv_pos;
sg_lst.addr = (MT_virt_addr_t)msg;
sg_lst.len = IB_MTU;
sg_lst.lkey = con_info->recv_bufs.rep_mrw.l_key;
rr_desc.id = (VAPI_wr_id_t)(unsigned long)con_info; /* User defined work request ID */
rr_desc.opcode = VAPI_RECEIVE;
rr_desc.comp_type = VAPI_SIGNALED;//VAPI_SIGNALED;
rr_desc.sg_lst_p = &sg_lst;
rr_desc.sg_lst_len = 1;
rc = VAPI_post_rr(con_info->hca_hndl, con_info->qp_hndl, &rr_desc);
if (rc != VAPI_SUCCESS) goto err_VAPI_post_rr;
con_info->n_tosend_toks++;
con_info->n_recv_toks--;
con_info->recv_pos = (con_info->recv_pos + 1) % SIZE_SR_QUEUE;
if (con_info->n_tosend_toks >= MAX_PENDING_TOKS) {
psib_send(con_info, NULL, 0);
}
return 0;
/* --- */
err_VAPI_post_rr:
psib_err_rc("VAPI_post_rr() failed", rc);
con_info->con_broken = 1;
// err_broken:
errno = EPIPE;
return -1;
}
/* Reset is used after a trial to empty the work request queues so we
have enough room for the next trial to run */
void Reset(ArgStruct *p)
{
VAPI_ret_t ret; /* Return code */
VAPI_sr_desc_t sr; /* Send request */
VAPI_rr_desc_t rr; /* Recv request */
/* If comptype is event, then we'll use event handler to detect receive,
* so initialize receive_complete flag
*/
if(p->prot.comptype == NP_COMP_EVENT) receive_complete = 0;
/* Prepost receive */
rr.opcode = VAPI_RECEIVE;
rr.comp_type = VAPI_SIGNALED;
rr.sg_lst_len = 0;
LOGPRINTF("Posting recv request in Reset\n");
ret = VAPI_post_rr(hca_hndl, qp_hndl, &rr);
if(ret != VAPI_OK) {
fprintf(stderr, " Error posting recv request: %s\n", VAPI_strerror(ret));
CleanUp(p);
exit(-1);
}
/* Make sure both nodes have preposted receives */
Sync(p);
/* Post Send */
sr.opcode = VAPI_SEND;
sr.comp_type = VAPI_SIGNALED;
sr.set_se = FALSE; /* This needed due to a bug in Mellanox HW rel a-0 */
sr.sg_lst_len = 0;
LOGPRINTF("Posting send request \n");
ret = VAPI_post_sr(hca_hndl, qp_hndl, &sr);
if(ret != VAPI_OK) {
fprintf(stderr, " Error posting send request in Reset: %s\n",
VAPI_strerror(ret));
exit(-1);
}
if(wc.status != VAPI_SUCCESS) {
fprintf(stderr, " Error in completion status: %s\n",
VAPI_wc_status_sym(wc.status));
exit(-1);
}
LOGPRINTF("Polling for completion of send request\n");
ret = VAPI_CQ_EMPTY;
while(ret == VAPI_CQ_EMPTY)
ret = VAPI_poll_cq(hca_hndl, s_cq_hndl, &wc);
if(ret != VAPI_OK) {
fprintf(stderr, "Error polling CQ for send in Reset: %s\n",
VAPI_strerror(ret));
exit(-1);
}
if(wc.status != VAPI_SUCCESS) {
fprintf(stderr, " Error in completion status: %s\n",
VAPI_wc_status_sym(wc.status));
exit(-1);
}
LOGPRINTF("Status of send completion: %s\n", VAPI_wc_status_sym(wc.status));
if(p->prot.comptype == NP_COMP_EVENT) {
/* If using event completion, the event handler will set receive_complete
* when it gets the completion event.
*/
LOGPRINTF("Waiting for receive_complete flag\n");
while(receive_complete == 0) { /* BUSY WAIT */ }
} else {
LOGPRINTF("Polling for completion of recv request\n");
ret = VAPI_CQ_EMPTY;
while(ret == VAPI_CQ_EMPTY)
ret = VAPI_poll_cq(hca_hndl, r_cq_hndl, &wc);
if(ret != VAPI_OK) {
fprintf(stderr, "Error polling CQ for recv in Reset: %s\n",
VAPI_strerror(ret));
exit(-1);
}
if(wc.status != VAPI_SUCCESS) {
fprintf(stderr, " Error in completion status: %s\n",
VAPI_wc_status_sym(wc.status));
exit(-1);
}
LOGPRINTF("Status of recv completion: %s\n", VAPI_wc_status_sym(wc.status));
}
LOGPRINTF("Done with reset\n");
}