/*
* Initialize the QP etc.
* Given in res: port_num, max_outs_sq, max_outs_rq
*/
VAPI_ret_t osmt_mtl_get_qp_resources(IN OUT osmt_mtl_mad_res_t * res)
{
VAPI_ret_t ret;
VAPI_hca_port_t hca_port_info;
VAPI_qp_init_attr_t qp_init_attr;
VAPI_qp_prop_t qp_prop;
VAPI_cqe_num_t act_num;
/* Get HCA LID */
ret =
VAPI_query_hca_port_prop(res->hca_hndl, res->port_num,
&hca_port_info);
VAPI_CHECK_RET;
res->slid = hca_port_info.lid;
/* Get a PD */
ret = VAPI_alloc_pd(res->hca_hndl, &(res->pd_hndl));
VAPI_CHECK_RET;
/* Create CQ for RQ and SQ *//* TBD - Check we have enough act nums */
ret =
VAPI_create_cq(res->hca_hndl, res->max_outs_sq + 1,
&(res->sq_cq_hndl), &act_num);
VAPI_CHECK_RET;
ret =
VAPI_create_cq(res->hca_hndl, res->max_outs_rq + 1,
&(res->rq_cq_hndl), &act_num);
VAPI_CHECK_RET;
/* register event handlers for polling(block mode) internal use */
/* ret= EVAPI_set_comp_eventh(res->hca_hndl,res->rq_cq_hndl, */
/* EVAPI_POLL_CQ_UNBLOCK_HANDLER,NULL,&(res->rq_cq_eventh)); */
/* VAPI_CHECK_RET; */
/* ret= EVAPI_set_comp_eventh(res->hca_hndl,res->sq_cq_hndl, */
/* EVAPI_POLL_CQ_UNBLOCK_HANDLER,NULL,&(res->sq_cq_eventh)); */
/* VAPI_CHECK_RET; */
/* Create QP */
qp_init_attr.cap.max_oust_wr_sq = res->max_outs_sq + 1;
qp_init_attr.cap.max_oust_wr_rq = res->max_outs_rq + 1;
qp_init_attr.cap.max_sg_size_sq = 4;
qp_init_attr.cap.max_sg_size_rq = 4;
qp_init_attr.pd_hndl = res->pd_hndl;
qp_init_attr.rdd_hndl = 0;
qp_init_attr.rq_cq_hndl = res->rq_cq_hndl;
qp_init_attr.rq_sig_type = VAPI_SIGNAL_ALL_WR; /* That's default for IB */
qp_init_attr.sq_cq_hndl = res->sq_cq_hndl;
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;
qp_init_attr.ts_type = VAPI_TS_UD;
ret =
VAPI_create_qp(res->hca_hndl, &qp_init_attr, &(res->qp_hndl),
&qp_prop);
VAPI_CHECK_RET;
res->qp_id.qp_num = qp_prop.qp_num;
return (VAPI_OK);
}
static
int psib_open_cq(IN VAPI_hca_hndl_t hca_hndl, IN unsigned int cqe_num, OUT VAPI_cq_hndl_t *cq_hndl)
{
unsigned int num_of_entries;
VAPI_ret_t rc;
/* create completion queue - used for both send and receive queues */
rc = VAPI_create_cq(hca_hndl, cqe_num, cq_hndl, &num_of_entries);
if (rc != VAPI_SUCCESS) goto err_VAPI_create_cq;
assert(num_of_entries >= cqe_num);
return 0;
/* --- */
err_VAPI_create_cq:
psib_err_rc("VAPI_create_cq() failed", rc);
return -1;
}
int initIB(ArgStruct *p)
{
VAPI_ret_t ret;
/* Open HCA */
/* open hca just in case it was not opened by system earlier */
ret = VAPI_open_hca("InfiniHost0", &hca_hndl);
ret = EVAPI_get_hca_hndl("InfiniHost0", &hca_hndl);
if(ret != VAPI_OK) {
fprintf(stderr, "Error opening Infiniband HCA: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Opened Infiniband HCA\n");
}
/* Get HCA properties */
port_num=1;
ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t)port_num,
(VAPI_hca_port_t *)&hca_port);
if(ret != VAPI_OK) {
fprintf(stderr, "Error querying Infiniband HCA: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Queried Infiniband HCA\n");
}
lid = hca_port.lid;
LOGPRINTF(" lid = %d\n", lid);
/* Allocate Protection Domain */
ret = VAPI_alloc_pd(hca_hndl, &pd_hndl);
if(ret != VAPI_OK) {
fprintf(stderr, "Error allocating PD: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Allocated Protection Domain\n");
}
/* Create send completion queue */
num_cqe = 30000; /* Requested number of completion q elements */
ret = VAPI_create_cq(hca_hndl, num_cqe, &s_cq_hndl, &act_num_cqe);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating send CQ: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Send Completion Queue with %d elements\n", act_num_cqe);
}
/* Create recv completion queue */
num_cqe = 20000; /* Requested number of completion q elements */
ret = VAPI_create_cq(hca_hndl, num_cqe, &r_cq_hndl, &act_num_cqe);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating recv CQ: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Recv Completion Queue with %d elements\n", act_num_cqe);
}
/* Placeholder for MR */
/* Create Queue Pair */
qp_init_attr.cap.max_oust_wr_rq = max_wq; /* Max outstanding WR on RQ */
qp_init_attr.cap.max_oust_wr_sq = max_wq; /* Max outstanding WR on SQ */
qp_init_attr.cap.max_sg_size_rq = 1; /* Max scatter/gather entries on RQ */
qp_init_attr.cap.max_sg_size_sq = 1; /* Max scatter/gather entries on SQ */
qp_init_attr.pd_hndl = pd_hndl; /* Protection domain handle */
qp_init_attr.rdd_hndl = 0; /* Reliable datagram domain handle */
qp_init_attr.rq_cq_hndl = r_cq_hndl; /* CQ handle for RQ */
qp_init_attr.rq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */
qp_init_attr.sq_cq_hndl = s_cq_hndl; /* CQ handle for RQ */
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */
qp_init_attr.ts_type = IB_TS_RC; /* Transmission type */
ret = VAPI_create_qp(hca_hndl, &qp_init_attr, &qp_hndl, &qp_prop);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating Queue Pair: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Queue Pair, max outstanding WR on RQ: %d, on SQ: %d\n",
qp_prop.cap.max_oust_wr_rq, qp_prop.cap.max_oust_wr_sq);
}
/* Exchange lid and qp_num with other node */
if( write(p->commfd, &lid, sizeof(lid) ) != sizeof(lid) ) {
fprintf(stderr, "Failed to send lid over socket\n");
return -1;
}
if( write(p->commfd, &qp_prop.qp_num, sizeof(qp_prop.qp_num) ) != sizeof(qp_prop.qp_num) ) {
fprintf(stderr, "Failed to send qpnum over socket\n");
return -1;
}
if( read(p->commfd, &d_lid, sizeof(d_lid) ) != sizeof(d_lid) ) {
fprintf(stderr, "Failed to read lid from socket\n");
return -1;
}
if( read(p->commfd, &d_qp_num, sizeof(d_qp_num) ) != sizeof(d_qp_num) ) {
fprintf(stderr, "Failed to read qpnum from socket\n");
return -1;
}
LOGPRINTF("Local: lid=%d qp_num=%d Remote: lid=%d qp_num=%d\n",
lid, qp_prop.qp_num, d_lid, d_qp_num);
/* Bring up Queue Pair */
/******* INIT state ******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = port_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to INIT: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to INIT\n");
/******* RTR (Ready-To-Receive) state *******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.qp_ous_rd_atom = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.dest_qp_num = d_qp_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM);
qp_attr.av.sl = 0;
qp_attr.av.grh_flag = FALSE;
qp_attr.av.dlid = d_lid;
qp_attr.av.static_rate = 0;
qp_attr.av.src_path_bits = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV);
qp_attr.path_mtu = p->prot.ib_mtu;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
qp_attr.rq_psn = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN);
qp_attr.pkey_ix = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.min_rnr_timer = 5;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to RTR: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to RTR\n");
/* Sync before going to RTS state */
Sync(p);
/******* RTS (Ready-to-Send) state *******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
qp_attr.timeout = 31;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT);
qp_attr.retry_count = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY);
qp_attr.ous_dst_rd_atom = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to RTS: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to RTS\n");
/* If using event completion, register event completion handler and request
* the initial notification
*/
if( p->prot.comptype == NP_COMP_EVENT ) {
EVAPI_set_comp_eventh(hca_hndl, r_cq_hndl, event_handler, p, &ceh_hndl);
VAPI_req_comp_notif(hca_hndl, r_cq_hndl, VAPI_NEXT_COMP);
}
return 0;
}
