VAPI_ret_t osmt_mtl_qp_init(osmt_mtl_mad_res_t * res)
{
VAPI_ret_t ret;
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_cap_t qp_cap;
/*
* Change QP to INIT
*
*/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = res->port_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.qkey = res->qkey;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QKEY);
/* If I do not set this mask, I get an error from HH. QPM should catch it */
ret =
VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,
&qp_cap);
VAPI_CHECK_RET;
return (ret);
}
/*
* move_to_rts
*/
static
int move_to_rts(IN VAPI_hca_hndl_t hca_hndl, IN VAPI_qp_hndl_t qp_hndl)
{
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_cap_t qp_cap;
VAPI_ret_t rc;
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);
/* Packet sequence number */
qp_attr.sq_psn = 0; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_SQ_PSN);
qp_attr.timeout = 10;/*0x20*/; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_TIMEOUT);
qp_attr.retry_count = 1; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = 1; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RNR_RETRY);
/* Number of outstanding RDMA rd/atomic ops at destination */
qp_attr.ous_dst_rd_atom = 1; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_OUS_DST_RD_ATOM);
rc = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if (rc != VAPI_SUCCESS) goto err_VAPI_modify_qp;
return 0;
/* --- */
err_VAPI_modify_qp:
psib_err_rc("VAPI_modify_qp() failed", rc);
return -1;
}
VAPI_ret_t osmt_mtl_qp_2_rtr_rts(osmt_mtl_mad_res_t * res)
{
VAPI_ret_t ret;
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_cap_t qp_cap;
/*
* Change QP to RTR
*
*/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
/* qp_attr.rq_psn = 0; */
/* QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RQ_PSN); */
ret =
VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,
&qp_cap);
VAPI_CHECK_RET;
/*
* Change QP to RTS
*
*/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
ret =
VAPI_modify_qp(res->hca_hndl, res->qp_hndl, &qp_attr, &qp_attr_mask,
&qp_cap);
VAPI_CHECK_RET;
return (ret);
}
/*
* move_to_rtr
*/
static
int move_to_rtr(IN VAPI_hca_hndl_t hca_hndl,
IN VAPI_qp_hndl_t qp_hndl,
IN IB_lid_t remote_lid, /* remote peer's LID */
IN IB_wqpn_t remote_qpn) /* remote peer's QPN */
{
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_cap_t qp_cap;
VAPI_ret_t rc;
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.av.sl = 0; /* Service level bits ??? */;
qp_attr.av.grh_flag = FALSE;
qp_attr.av.dlid = remote_lid;
qp_attr.av.static_rate = 0; /* 1x ???? */
qp_attr.av.src_path_bits = 0; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV);
qp_attr.path_mtu = IB_MTU_SPEC; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
/* Packet sequence number */
qp_attr.rq_psn = 0; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_RQ_PSN);
/* Maximum number of oust. RDMA read/atomic as target */
qp_attr.qp_ous_rd_atom = 1; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.dest_qp_num = remote_qpn; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_DEST_QP_NUM);
/* Minimum RNR NAK timer */
qp_attr.min_rnr_timer = 0; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_MIN_RNR_TIMER);
rc = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if (rc != VAPI_SUCCESS) goto err_VAPI_modify_qp;
return 0;
/* --- */
err_VAPI_modify_qp:
psib_err_rc("VAPI_modify_qp() faile", rc);
return -1;
}
int initIB(ArgStruct *p)
{
VAPI_ret_t ret;
/* Open HCA */
/* open hca just in case it was not opened by system earlier */
ret = VAPI_open_hca("InfiniHost0", &hca_hndl);
ret = EVAPI_get_hca_hndl("InfiniHost0", &hca_hndl);
if(ret != VAPI_OK) {
fprintf(stderr, "Error opening Infiniband HCA: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Opened Infiniband HCA\n");
}
/* Get HCA properties */
port_num=1;
ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t)port_num,
(VAPI_hca_port_t *)&hca_port);
if(ret != VAPI_OK) {
fprintf(stderr, "Error querying Infiniband HCA: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Queried Infiniband HCA\n");
}
lid = hca_port.lid;
LOGPRINTF(" lid = %d\n", lid);
/* Allocate Protection Domain */
ret = VAPI_alloc_pd(hca_hndl, &pd_hndl);
if(ret != VAPI_OK) {
fprintf(stderr, "Error allocating PD: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Allocated Protection Domain\n");
}
/* Create send completion queue */
num_cqe = 30000; /* Requested number of completion q elements */
ret = VAPI_create_cq(hca_hndl, num_cqe, &s_cq_hndl, &act_num_cqe);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating send CQ: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Send Completion Queue with %d elements\n", act_num_cqe);
}
/* Create recv completion queue */
num_cqe = 20000; /* Requested number of completion q elements */
ret = VAPI_create_cq(hca_hndl, num_cqe, &r_cq_hndl, &act_num_cqe);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating recv CQ: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Recv Completion Queue with %d elements\n", act_num_cqe);
}
/* Placeholder for MR */
/* Create Queue Pair */
qp_init_attr.cap.max_oust_wr_rq = max_wq; /* Max outstanding WR on RQ */
qp_init_attr.cap.max_oust_wr_sq = max_wq; /* Max outstanding WR on SQ */
qp_init_attr.cap.max_sg_size_rq = 1; /* Max scatter/gather entries on RQ */
qp_init_attr.cap.max_sg_size_sq = 1; /* Max scatter/gather entries on SQ */
qp_init_attr.pd_hndl = pd_hndl; /* Protection domain handle */
qp_init_attr.rdd_hndl = 0; /* Reliable datagram domain handle */
qp_init_attr.rq_cq_hndl = r_cq_hndl; /* CQ handle for RQ */
qp_init_attr.rq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */
qp_init_attr.sq_cq_hndl = s_cq_hndl; /* CQ handle for RQ */
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */
qp_init_attr.ts_type = IB_TS_RC; /* Transmission type */
ret = VAPI_create_qp(hca_hndl, &qp_init_attr, &qp_hndl, &qp_prop);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating Queue Pair: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Queue Pair, max outstanding WR on RQ: %d, on SQ: %d\n",
qp_prop.cap.max_oust_wr_rq, qp_prop.cap.max_oust_wr_sq);
}
/* Exchange lid and qp_num with other node */
if( write(p->commfd, &lid, sizeof(lid) ) != sizeof(lid) ) {
fprintf(stderr, "Failed to send lid over socket\n");
return -1;
}
if( write(p->commfd, &qp_prop.qp_num, sizeof(qp_prop.qp_num) ) != sizeof(qp_prop.qp_num) ) {
fprintf(stderr, "Failed to send qpnum over socket\n");
return -1;
}
if( read(p->commfd, &d_lid, sizeof(d_lid) ) != sizeof(d_lid) ) {
fprintf(stderr, "Failed to read lid from socket\n");
return -1;
}
if( read(p->commfd, &d_qp_num, sizeof(d_qp_num) ) != sizeof(d_qp_num) ) {
fprintf(stderr, "Failed to read qpnum from socket\n");
return -1;
}
LOGPRINTF("Local: lid=%d qp_num=%d Remote: lid=%d qp_num=%d\n",
lid, qp_prop.qp_num, d_lid, d_qp_num);
/* Bring up Queue Pair */
/******* INIT state ******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = port_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to INIT: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to INIT\n");
/******* RTR (Ready-To-Receive) state *******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.qp_ous_rd_atom = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.dest_qp_num = d_qp_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM);
qp_attr.av.sl = 0;
qp_attr.av.grh_flag = FALSE;
qp_attr.av.dlid = d_lid;
qp_attr.av.static_rate = 0;
qp_attr.av.src_path_bits = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV);
qp_attr.path_mtu = p->prot.ib_mtu;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
qp_attr.rq_psn = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN);
qp_attr.pkey_ix = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.min_rnr_timer = 5;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to RTR: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to RTR\n");
/* Sync before going to RTS state */
Sync(p);
/******* RTS (Ready-to-Send) state *******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
qp_attr.timeout = 31;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT);
qp_attr.retry_count = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY);
qp_attr.ous_dst_rd_atom = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to RTS: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to RTS\n");
/* If using event completion, register event completion handler and request
* the initial notification
*/
if( p->prot.comptype == NP_COMP_EVENT ) {
EVAPI_set_comp_eventh(hca_hndl, r_cq_hndl, event_handler, p, &ceh_hndl);
VAPI_req_comp_notif(hca_hndl, r_cq_hndl, VAPI_NEXT_COMP);
}
return 0;
}
/*
* The queue pair has been created and we have received the remote
* queue pair information from the peer so we init this queue pair
* and are ready to roll.
*/
int mca_btl_mvapi_endpoint_qp_init_query(
mca_btl_mvapi_module_t* mvapi_btl,
VAPI_hca_hndl_t nic,
VAPI_qp_hndl_t qp_hndl,
VAPI_qp_num_t remote_qp_num,
IB_lid_t remote_lid,
IB_port_t port_id
)
{
VAPI_ret_t ret;
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_init_attr_t qp_init_attr;
VAPI_qp_cap_t qp_cap;
/* Modifying QP to INIT */
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = mca_btl_mvapi_component.ib_pkey_ix;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = port_id;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
BTL_ERROR(("Error modifying the queue pair: %s", VAPI_strerror(ret)));
return OMPI_ERROR;
}
BTL_VERBOSE(("Modified to init..Qp %d", qp_hndl));
/********************** INIT --> RTR ************************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.qp_ous_rd_atom = mca_btl_mvapi_component.ib_qp_ous_rd_atom;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.path_mtu = mca_btl_mvapi_component.ib_mtu;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
qp_attr.rq_psn = mca_btl_mvapi_component.ib_psn;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN);
qp_attr.pkey_ix = mca_btl_mvapi_component.ib_pkey_ix;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.min_rnr_timer = mca_btl_mvapi_component.ib_min_rnr_timer;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER);
qp_attr.av.sl = mca_btl_mvapi_component.ib_service_level;
qp_attr.av.grh_flag = FALSE;
qp_attr.av.static_rate = mca_btl_mvapi_component.ib_static_rate;
qp_attr.av.src_path_bits = mca_btl_mvapi_component.ib_src_path_bits;
qp_attr.dest_qp_num = remote_qp_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM);
qp_attr.av.dlid = remote_lid;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
BTL_ERROR(("Error modifying the queue pair: %s", VAPI_strerror(ret)));
return OMPI_ERROR;
}
BTL_VERBOSE(("Modified to RTR..Qp %d", qp_hndl));
/************** RTS *******************/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = mca_btl_mvapi_component.ib_psn;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
qp_attr.timeout = mca_btl_mvapi_component.ib_timeout;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT);
qp_attr.retry_count = mca_btl_mvapi_component.ib_retry_count;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = mca_btl_mvapi_component.ib_rnr_retry;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY);
qp_attr.ous_dst_rd_atom = mca_btl_mvapi_component.ib_max_rdma_dst_ops;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM);
ret = VAPI_modify_qp(nic, qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if(VAPI_OK != ret) {
return OMPI_ERROR;
}
BTL_VERBOSE(("Modified to RTS..Qp %d", qp_hndl));
ret = VAPI_query_qp(nic, qp_hndl, &qp_attr, &qp_attr_mask, &qp_init_attr );
if (ret != VAPI_OK) {
BTL_ERROR(("Error modifying the queue pair: %s", VAPI_strerror(ret)));
return OMPI_ERROR;
}
mvapi_btl->ib_inline_max = qp_init_attr.cap.max_inline_data_sq;
return OMPI_SUCCESS;
}
static
int psib_init_con(hca_info_t *hca_info, port_info_t *port_info, con_info_t *con_info)
{
VAPI_qp_init_attr_t qp_init_attr;
VAPI_ret_t rc;
VAPI_qp_attr_t qp_attr;
VAPI_qp_attr_mask_t qp_attr_mask;
VAPI_qp_cap_t qp_cap;
con_info->qp_hndl = 0;
con_info->lid = port_info->hca_port.lid;
con_info->hca_hndl = hca_info->hca_hndl;
con_info->send_bufs.mr_hndl = 0;
con_info->recv_bufs.mr_hndl = 0;
con_info->con_broken = 0;
memset(&qp_init_attr, 0, sizeof(qp_init_attr));
qp_init_attr.sq_cq_hndl = hca_info->cq_hndl;
qp_init_attr.rq_cq_hndl = hca_info->cq_hndl;
qp_init_attr.cap.max_oust_wr_sq = 128; /* Max outstanding WR on the SQ */
qp_init_attr.cap.max_oust_wr_rq = 128; /* Max outstanding WR on the RQ */
qp_init_attr.cap.max_sg_size_sq = 4;/* Max scatter/gather descriptor entries on the SQ */
qp_init_attr.cap.max_sg_size_rq = 4;/* Max scatter/gather descriptor entries on the RQ */
qp_init_attr.cap.max_inline_data_sq = 16; /* Max bytes in inline data on the SQ */
/* max_inline_data_sq is currently valid only for VAPI_query_qp (ignored for VAPI_create_qp) */
/* In order to enlarge the max_inline_data_sq capability, enlarge the max_sg_size_sq parameter */
qp_init_attr.rdd_hndl = 0; /* N/A for RC transport service */
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR;
qp_init_attr.rq_sig_type = VAPI_SIGNAL_ALL_WR;
qp_init_attr.pd_hndl = hca_info->pd_hndl;
qp_init_attr.ts_type = VAPI_TS_RC; /* RC transport service */
/* create the QP */
rc = VAPI_create_qp(hca_info->hca_hndl, &qp_init_attr,
&con_info->qp_hndl, &con_info->qp_prop);
if (rc != VAPI_SUCCESS) goto err_VAPI_create_qp;
if (psib_debug > 1)
print_qp_props(&con_info->qp_prop);
memset(&qp_attr, 0, sizeof(qp_attr));
memset(&qp_attr_mask, 0, sizeof(qp_attr_mask));
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_QP_STATE);
qp_attr.pkey_ix = 0; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PKEY_IX);
qp_attr.port = port_info->hca_port_idx; QP_ATTR_MASK_SET(qp_attr_mask,QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
/* Rem_Atomic ? if (topt_p->atomics) {
qp_attr.remote_atomic_flags |= VAPI_EN_REM_ATOMIC_OP;
} */
rc = VAPI_modify_qp(hca_info->hca_hndl, con_info->qp_hndl,
&qp_attr, &qp_attr_mask, &qp_cap);
if (rc != VAPI_SUCCESS) goto err_VAPI_modify_qp;
/*
* Memory for send and receive bufs
*/
if (psib_vapi_alloc(hca_info, IB_MTU * SIZE_SR_QUEUE, 0, &con_info->send_bufs))
goto err_alloc;
con_info->send_pos = 0;
if (psib_vapi_alloc(hca_info, IB_MTU * SIZE_SR_QUEUE, VAPI_EN_LOCAL_WRITE, &con_info->recv_bufs))
goto err_alloc;
con_info->recv_pos = 0;
return 0;
/* --- */
err_alloc:
psib_cleanup_con(hca_info, con_info);
return -1;
/* --- */
err_VAPI_modify_qp:
psib_err_rc("VAPI_modify_qp failed", rc);
psib_cleanup_con(hca_info, con_info);
return -1;
/* --- */
err_VAPI_create_qp:
psib_err_rc("VAPI_create_qp() failed", rc);
return -1;
}
