/* if hca_name == NULL choose first HCA */
static
int psib_open_hca(IN char *hca_name, OUT VAPI_hca_hndl_t *hca_hndl)
{
VAPI_hca_id_t hca_id;
VAPI_ret_t rc;
assert(hca_hndl != NULL);
if (psib_get_hca_name(&hca_id, hca_name)) goto err_hca_name;
/* try to get a handle to the given hca_id */
rc = EVAPI_get_hca_hndl(hca_id, hca_hndl);
if (rc != VAPI_SUCCESS) goto err_EVAPI_get_hca_hndl;
if (psib_debug > 1) {
VAPI_hca_vendor_t hca_vendor; /* ?? */
VAPI_hca_cap_t hca_cap; /* HCA capabilities */
rc = VAPI_query_hca_cap(*hca_hndl, &hca_vendor, &hca_cap);
if (rc != VAPI_SUCCESS) goto err_VAPI_query_hca_cap;
print_hca_cap(&hca_vendor, &hca_cap);
}
return 0;
/* --- */
err_VAPI_query_hca_cap:
EVAPI_release_hca_hndl(*hca_hndl);
psib_err_rc("VAPI_query_hca_cap() failed", rc);
return -1;
/* --- */
err_EVAPI_get_hca_hndl:
psib_err_rc("EVAPI_get_hca_hndl() failed", rc);
return -1;
/* --- */
err_hca_name:
return -1;
}
int initIB(ArgStruct *p)
{
VAPI_ret_t ret;
/* Open HCA */
/* open hca just in case it was not opened by system earlier */
ret = VAPI_open_hca("InfiniHost0", &hca_hndl);
ret = EVAPI_get_hca_hndl("InfiniHost0", &hca_hndl);
if(ret != VAPI_OK) {
fprintf(stderr, "Error opening Infiniband HCA: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Opened Infiniband HCA\n");
}
/* Get HCA properties */
port_num=1;
ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t)port_num,
(VAPI_hca_port_t *)&hca_port);
if(ret != VAPI_OK) {
fprintf(stderr, "Error querying Infiniband HCA: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Queried Infiniband HCA\n");
}
lid = hca_port.lid;
LOGPRINTF(" lid = %d\n", lid);
/* Allocate Protection Domain */
ret = VAPI_alloc_pd(hca_hndl, &pd_hndl);
if(ret != VAPI_OK) {
fprintf(stderr, "Error allocating PD: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Allocated Protection Domain\n");
}
/* Create send completion queue */
num_cqe = 30000; /* Requested number of completion q elements */
ret = VAPI_create_cq(hca_hndl, num_cqe, &s_cq_hndl, &act_num_cqe);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating send CQ: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Send Completion Queue with %d elements\n", act_num_cqe);
}
/* Create recv completion queue */
num_cqe = 20000; /* Requested number of completion q elements */
ret = VAPI_create_cq(hca_hndl, num_cqe, &r_cq_hndl, &act_num_cqe);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating recv CQ: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Recv Completion Queue with %d elements\n", act_num_cqe);
}
/* Placeholder for MR */
/* Create Queue Pair */
qp_init_attr.cap.max_oust_wr_rq = max_wq; /* Max outstanding WR on RQ */
qp_init_attr.cap.max_oust_wr_sq = max_wq; /* Max outstanding WR on SQ */
qp_init_attr.cap.max_sg_size_rq = 1; /* Max scatter/gather entries on RQ */
qp_init_attr.cap.max_sg_size_sq = 1; /* Max scatter/gather entries on SQ */
qp_init_attr.pd_hndl = pd_hndl; /* Protection domain handle */
qp_init_attr.rdd_hndl = 0; /* Reliable datagram domain handle */
qp_init_attr.rq_cq_hndl = r_cq_hndl; /* CQ handle for RQ */
qp_init_attr.rq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */
qp_init_attr.sq_cq_hndl = s_cq_hndl; /* CQ handle for RQ */
qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */
qp_init_attr.ts_type = IB_TS_RC; /* Transmission type */
ret = VAPI_create_qp(hca_hndl, &qp_init_attr, &qp_hndl, &qp_prop);
if(ret != VAPI_OK) {
fprintf(stderr, "Error creating Queue Pair: %s\n", VAPI_strerror(ret));
return -1;
} else {
LOGPRINTF("Created Queue Pair, max outstanding WR on RQ: %d, on SQ: %d\n",
qp_prop.cap.max_oust_wr_rq, qp_prop.cap.max_oust_wr_sq);
}
/* Exchange lid and qp_num with other node */
if( write(p->commfd, &lid, sizeof(lid) ) != sizeof(lid) ) {
fprintf(stderr, "Failed to send lid over socket\n");
return -1;
}
if( write(p->commfd, &qp_prop.qp_num, sizeof(qp_prop.qp_num) ) != sizeof(qp_prop.qp_num) ) {
fprintf(stderr, "Failed to send qpnum over socket\n");
return -1;
}
if( read(p->commfd, &d_lid, sizeof(d_lid) ) != sizeof(d_lid) ) {
fprintf(stderr, "Failed to read lid from socket\n");
return -1;
}
if( read(p->commfd, &d_qp_num, sizeof(d_qp_num) ) != sizeof(d_qp_num) ) {
fprintf(stderr, "Failed to read qpnum from socket\n");
return -1;
}
LOGPRINTF("Local: lid=%d qp_num=%d Remote: lid=%d qp_num=%d\n",
lid, qp_prop.qp_num, d_lid, d_qp_num);
/* Bring up Queue Pair */
/******* INIT state ******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_INIT;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.pkey_ix = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.port = port_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT);
qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to INIT: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to INIT\n");
/******* RTR (Ready-To-Receive) state *******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTR;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.qp_ous_rd_atom = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM);
qp_attr.dest_qp_num = d_qp_num;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM);
qp_attr.av.sl = 0;
qp_attr.av.grh_flag = FALSE;
qp_attr.av.dlid = d_lid;
qp_attr.av.static_rate = 0;
qp_attr.av.src_path_bits = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV);
qp_attr.path_mtu = p->prot.ib_mtu;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU);
qp_attr.rq_psn = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN);
qp_attr.pkey_ix = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX);
qp_attr.min_rnr_timer = 5;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to RTR: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to RTR\n");
/* Sync before going to RTS state */
Sync(p);
/******* RTS (Ready-to-Send) state *******/
QP_ATTR_MASK_CLR_ALL(qp_attr_mask);
qp_attr.qp_state = VAPI_RTS;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE);
qp_attr.sq_psn = 0;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN);
qp_attr.timeout = 31;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT);
qp_attr.retry_count = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT);
qp_attr.rnr_retry = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY);
qp_attr.ous_dst_rd_atom = 1;
QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM);
ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap);
if(ret != VAPI_OK) {
fprintf(stderr, "Error modifying QP to RTS: %s\n", VAPI_strerror(ret));
return -1;
}
LOGPRINTF("Modified QP to RTS\n");
/* If using event completion, register event completion handler and request
* the initial notification
*/
if( p->prot.comptype == NP_COMP_EVENT ) {
EVAPI_set_comp_eventh(hca_hndl, r_cq_hndl, event_handler, p, &ceh_hndl);
VAPI_req_comp_notif(hca_hndl, r_cq_hndl, VAPI_NEXT_COMP);
}
return 0;
}
ib_api_status_t
osm_vendor_get_guid_ca_and_port(IN osm_vendor_t * const p_vend,
IN ib_net64_t const guid,
OUT VAPI_hca_hndl_t * p_hca_hndl,
OUT VAPI_hca_id_t * p_hca_id,
OUT uint8_t * p_hca_idx,
OUT uint32_t * p_port_num)
{
ib_api_status_t status;
VAPI_hca_id_t *p_ca_ids = NULL;
VAPI_ret_t vapi_res;
VAPI_hca_hndl_t hca_hndl;
VAPI_hca_vendor_t hca_vendor;
VAPI_hca_cap_t hca_cap;
IB_gid_t *p_port_gid = NULL;
uint16_t maxNumGids;
ib_net64_t port_guid;
uint32_t ca, portIdx, ca_count;
OSM_LOG_ENTER(p_vend->p_log);
CL_ASSERT(p_vend);
/*
* 1) Determine the number of CA's
* 2) Allocate an array big enough to hold the ca info objects.
* 3) Call again to retrieve the guids.
*/
status = __osm_vendor_get_ca_ids(p_vend, &p_ca_ids, &ca_count);
if (status != IB_SUCCESS) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_get_guid_ca_and_port: ERR 3D16: "
"Fail to get CA Ids.\n");
goto Exit;
}
/*
* For each CA, retrieve the CA info attributes
*/
for (ca = 0; ca < ca_count; ca++) {
/* get the HCA handle */
vapi_res = EVAPI_get_hca_hndl(p_ca_ids[ca], &hca_hndl);
if (vapi_res != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_get_guid_ca_and_port: ERR 3D17: "
"Fail to get HCA handle (%u).\n", vapi_res);
goto Exit;
}
/* get the CA attributes - to know how many ports it has: */
if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"osm_vendor_get_guid_ca_and_port: "
"Querying CA %s.\n", p_ca_ids[ca]);
}
/* query and get the HCA capability */
vapi_res = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap);
if (vapi_res != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_get_guid_ca_and_port: ERR 3D18: "
"Fail to get HCA Capabilities (%u).\n",
vapi_res);
goto Exit;
}
/* go over all ports - to obtail their guids */
for (portIdx = 0; portIdx < hca_cap.phys_port_num; portIdx++) {
vapi_res =
VAPI_query_hca_gid_tbl(hca_hndl, portIdx + 1, 0,
&maxNumGids, NULL);
p_port_gid =
(IB_gid_t *) malloc(maxNumGids * sizeof(IB_gid_t));
/* get the port guid */
vapi_res =
VAPI_query_hca_gid_tbl(hca_hndl, portIdx + 1,
maxNumGids, &maxNumGids,
p_port_gid);
if (vapi_res != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_get_guid_ca_and_port: ERR 3D19: "
"Fail to get HCA Port GID (%d).\n",
vapi_res);
goto Exit;
}
/* convert to SF style */
__osm_vendor_gid_to_guid(p_port_gid[0],
(VAPI_gid_t *) & port_guid);
/* finally did we find it ? */
if (port_guid == guid) {
*p_hca_hndl = hca_hndl;
memcpy(p_hca_id, p_ca_ids[ca],
sizeof(VAPI_hca_id_t));
*p_hca_idx = ca;
*p_port_num = portIdx + 1;
status = IB_SUCCESS;
goto Exit;
}
free(p_port_gid);
p_port_gid = NULL;
} /* ALL PORTS */
} /* all HCAs */
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"osm_vendor_get_guid_ca_and_port: ERR 3D20: "
"Fail to find HCA and Port for Port Guid 0x%" PRIx64 "\n",
cl_ntoh64(guid));
status = IB_INVALID_GUID;
Exit:
if (p_ca_ids != NULL)
free(p_ca_ids);
if (p_port_gid != NULL)
free(p_port_gid);
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
/**********************************************************************
* Initialize an Info Struct for the Given HCA by its Id
**********************************************************************/
static ib_api_status_t
__osm_ca_info_init(IN osm_vendor_t * const p_vend,
IN VAPI_hca_id_t ca_id, OUT osm_ca_info_t * const p_ca_info)
{
ib_api_status_t status = IB_ERROR;
VAPI_ret_t vapi_res;
VAPI_hca_hndl_t hca_hndl;
VAPI_hca_vendor_t hca_vendor;
VAPI_hca_cap_t hca_cap;
VAPI_hca_port_t hca_port;
uint8_t port_num;
IB_gid_t *p_port_gid;
uint16_t maxNumGids;
OSM_LOG_ENTER(p_vend->p_log);
/* get the HCA handle */
vapi_res = EVAPI_get_hca_hndl(ca_id, &hca_hndl);
if (vapi_res != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_ca_info_init: ERR 3D05: "
"Fail to get HCA handle (%u).\n", vapi_res);
goto Exit;
}
if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) {
osm_log(p_vend->p_log, OSM_LOG_DEBUG,
"__osm_ca_info_init: " "Querying CA %s.\n", ca_id);
}
/* query and get the HCA capability */
vapi_res = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap);
if (vapi_res != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_ca_info_init: ERR 3D06: "
"Fail to get HCA Capabilities (%u).\n", vapi_res);
goto Exit;
}
/* get the guid of the HCA */
memcpy(&(p_ca_info->guid), hca_cap.node_guid, 8 * sizeof(u_int8_t));
p_ca_info->attr_size = 1;
p_ca_info->p_attr = (ib_ca_attr_t *) malloc(sizeof(ib_ca_attr_t));
memcpy(&(p_ca_info->p_attr->ca_guid), hca_cap.node_guid,
8 * sizeof(u_int8_t));
/* now obtain the attributes of the ports */
p_ca_info->p_attr->num_ports = hca_cap.phys_port_num;
p_ca_info->p_attr->p_port_attr =
(ib_port_attr_t *) malloc(hca_cap.phys_port_num *
sizeof(ib_port_attr_t));
for (port_num = 0; port_num < p_ca_info->p_attr->num_ports; port_num++) {
/* query the port attributes */
vapi_res =
VAPI_query_hca_port_prop(hca_hndl, port_num + 1, &hca_port);
if (vapi_res != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_ca_info_init: ERR 3D07: "
"Fail to get HCA Port Attributes (%d).\n",
vapi_res);
goto Exit;
}
/* first call to know the size of the gid table */
vapi_res =
VAPI_query_hca_gid_tbl(hca_hndl, port_num + 1, 0,
&maxNumGids, NULL);
p_port_gid = (IB_gid_t *) malloc(maxNumGids * sizeof(IB_gid_t));
vapi_res =
VAPI_query_hca_gid_tbl(hca_hndl, port_num + 1, maxNumGids,
&maxNumGids, p_port_gid);
if (vapi_res != VAPI_OK) {
osm_log(p_vend->p_log, OSM_LOG_ERROR,
"__osm_ca_info_init: ERR 3D12: "
"Fail to get HCA Port GID (%d).\n", vapi_res);
goto Exit;
}
__osm_vendor_gid_to_guid(p_port_gid[0],
(IB_gid_t *) & p_ca_info->p_attr->
p_port_attr[port_num].port_guid);
p_ca_info->p_attr->p_port_attr[port_num].lid = hca_port.lid;
p_ca_info->p_attr->p_port_attr[port_num].link_state =
hca_port.state;
p_ca_info->p_attr->p_port_attr[port_num].sm_lid =
hca_port.sm_lid;
free(p_port_gid);
}
status = IB_SUCCESS;
Exit:
OSM_LOG_EXIT(p_vend->p_log);
return (status);
}
