/* if hca_name == NULL choose first HCA */ static int psib_open_hca(IN char *hca_name, OUT VAPI_hca_hndl_t *hca_hndl) { VAPI_hca_id_t hca_id; VAPI_ret_t rc; assert(hca_hndl != NULL); if (psib_get_hca_name(&hca_id, hca_name)) goto err_hca_name; /* try to get a handle to the given hca_id */ rc = EVAPI_get_hca_hndl(hca_id, hca_hndl); if (rc != VAPI_SUCCESS) goto err_EVAPI_get_hca_hndl; if (psib_debug > 1) { VAPI_hca_vendor_t hca_vendor; /* ?? */ VAPI_hca_cap_t hca_cap; /* HCA capabilities */ rc = VAPI_query_hca_cap(*hca_hndl, &hca_vendor, &hca_cap); if (rc != VAPI_SUCCESS) goto err_VAPI_query_hca_cap; print_hca_cap(&hca_vendor, &hca_cap); } return 0; /* --- */ err_VAPI_query_hca_cap: EVAPI_release_hca_hndl(*hca_hndl); psib_err_rc("VAPI_query_hca_cap() failed", rc); return -1; /* --- */ err_EVAPI_get_hca_hndl: psib_err_rc("EVAPI_get_hca_hndl() failed", rc); return -1; /* --- */ err_hca_name: return -1; }
int initIB(ArgStruct *p) { VAPI_ret_t ret; /* Open HCA */ /* open hca just in case it was not opened by system earlier */ ret = VAPI_open_hca("InfiniHost0", &hca_hndl); ret = EVAPI_get_hca_hndl("InfiniHost0", &hca_hndl); if(ret != VAPI_OK) { fprintf(stderr, "Error opening Infiniband HCA: %s\n", VAPI_strerror(ret)); return -1; } else { LOGPRINTF("Opened Infiniband HCA\n"); } /* Get HCA properties */ port_num=1; ret = VAPI_query_hca_port_prop(hca_hndl, (IB_port_t)port_num, (VAPI_hca_port_t *)&hca_port); if(ret != VAPI_OK) { fprintf(stderr, "Error querying Infiniband HCA: %s\n", VAPI_strerror(ret)); return -1; } else { LOGPRINTF("Queried Infiniband HCA\n"); } lid = hca_port.lid; LOGPRINTF(" lid = %d\n", lid); /* Allocate Protection Domain */ ret = VAPI_alloc_pd(hca_hndl, &pd_hndl); if(ret != VAPI_OK) { fprintf(stderr, "Error allocating PD: %s\n", VAPI_strerror(ret)); return -1; } else { LOGPRINTF("Allocated Protection Domain\n"); } /* Create send completion queue */ num_cqe = 30000; /* Requested number of completion q elements */ ret = VAPI_create_cq(hca_hndl, num_cqe, &s_cq_hndl, &act_num_cqe); if(ret != VAPI_OK) { fprintf(stderr, "Error creating send CQ: %s\n", VAPI_strerror(ret)); return -1; } else { LOGPRINTF("Created Send Completion Queue with %d elements\n", act_num_cqe); } /* Create recv completion queue */ num_cqe = 20000; /* Requested number of completion q elements */ ret = VAPI_create_cq(hca_hndl, num_cqe, &r_cq_hndl, &act_num_cqe); if(ret != VAPI_OK) { fprintf(stderr, "Error creating recv CQ: %s\n", VAPI_strerror(ret)); return -1; } else { LOGPRINTF("Created Recv Completion Queue with %d elements\n", act_num_cqe); } /* Placeholder for MR */ /* Create Queue Pair */ qp_init_attr.cap.max_oust_wr_rq = max_wq; /* Max outstanding WR on RQ */ qp_init_attr.cap.max_oust_wr_sq = max_wq; /* Max outstanding WR on SQ */ qp_init_attr.cap.max_sg_size_rq = 1; /* Max scatter/gather entries on RQ */ qp_init_attr.cap.max_sg_size_sq = 1; /* Max scatter/gather entries on SQ */ qp_init_attr.pd_hndl = pd_hndl; /* Protection domain handle */ qp_init_attr.rdd_hndl = 0; /* Reliable datagram domain handle */ qp_init_attr.rq_cq_hndl = r_cq_hndl; /* CQ handle for RQ */ qp_init_attr.rq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */ qp_init_attr.sq_cq_hndl = s_cq_hndl; /* CQ handle for RQ */ qp_init_attr.sq_sig_type = VAPI_SIGNAL_REQ_WR; /* Signalling type */ qp_init_attr.ts_type = IB_TS_RC; /* Transmission type */ ret = VAPI_create_qp(hca_hndl, &qp_init_attr, &qp_hndl, &qp_prop); if(ret != VAPI_OK) { fprintf(stderr, "Error creating Queue Pair: %s\n", VAPI_strerror(ret)); return -1; } else { LOGPRINTF("Created Queue Pair, max outstanding WR on RQ: %d, on SQ: %d\n", qp_prop.cap.max_oust_wr_rq, qp_prop.cap.max_oust_wr_sq); } /* Exchange lid and qp_num with other node */ if( write(p->commfd, &lid, sizeof(lid) ) != sizeof(lid) ) { fprintf(stderr, "Failed to send lid over socket\n"); return -1; } if( write(p->commfd, &qp_prop.qp_num, sizeof(qp_prop.qp_num) ) != sizeof(qp_prop.qp_num) ) { fprintf(stderr, "Failed to send qpnum over socket\n"); return -1; } if( read(p->commfd, &d_lid, sizeof(d_lid) ) != sizeof(d_lid) ) { fprintf(stderr, "Failed to read lid from socket\n"); return -1; } if( read(p->commfd, &d_qp_num, sizeof(d_qp_num) ) != sizeof(d_qp_num) ) { fprintf(stderr, "Failed to read qpnum from socket\n"); return -1; } LOGPRINTF("Local: lid=%d qp_num=%d Remote: lid=%d qp_num=%d\n", lid, qp_prop.qp_num, d_lid, d_qp_num); /* Bring up Queue Pair */ /******* INIT state ******/ QP_ATTR_MASK_CLR_ALL(qp_attr_mask); qp_attr.qp_state = VAPI_INIT; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); qp_attr.pkey_ix = 0; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX); qp_attr.port = port_num; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT); qp_attr.remote_atomic_flags = VAPI_EN_REM_WRITE | VAPI_EN_REM_READ; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_REMOTE_ATOMIC_FLAGS); ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap); if(ret != VAPI_OK) { fprintf(stderr, "Error modifying QP to INIT: %s\n", VAPI_strerror(ret)); return -1; } LOGPRINTF("Modified QP to INIT\n"); /******* RTR (Ready-To-Receive) state *******/ QP_ATTR_MASK_CLR_ALL(qp_attr_mask); qp_attr.qp_state = VAPI_RTR; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); qp_attr.qp_ous_rd_atom = 1; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM); qp_attr.dest_qp_num = d_qp_num; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM); qp_attr.av.sl = 0; qp_attr.av.grh_flag = FALSE; qp_attr.av.dlid = d_lid; qp_attr.av.static_rate = 0; qp_attr.av.src_path_bits = 0; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_AV); qp_attr.path_mtu = p->prot.ib_mtu; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU); qp_attr.rq_psn = 0; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN); qp_attr.pkey_ix = 0; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX); qp_attr.min_rnr_timer = 5; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER); ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap); if(ret != VAPI_OK) { fprintf(stderr, "Error modifying QP to RTR: %s\n", VAPI_strerror(ret)); return -1; } LOGPRINTF("Modified QP to RTR\n"); /* Sync before going to RTS state */ Sync(p); /******* RTS (Ready-to-Send) state *******/ QP_ATTR_MASK_CLR_ALL(qp_attr_mask); qp_attr.qp_state = VAPI_RTS; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); qp_attr.sq_psn = 0; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN); qp_attr.timeout = 31; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT); qp_attr.retry_count = 1; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT); qp_attr.rnr_retry = 1; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY); qp_attr.ous_dst_rd_atom = 1; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM); ret = VAPI_modify_qp(hca_hndl, qp_hndl, &qp_attr, &qp_attr_mask, &qp_cap); if(ret != VAPI_OK) { fprintf(stderr, "Error modifying QP to RTS: %s\n", VAPI_strerror(ret)); return -1; } LOGPRINTF("Modified QP to RTS\n"); /* If using event completion, register event completion handler and request * the initial notification */ if( p->prot.comptype == NP_COMP_EVENT ) { EVAPI_set_comp_eventh(hca_hndl, r_cq_hndl, event_handler, p, &ceh_hndl); VAPI_req_comp_notif(hca_hndl, r_cq_hndl, VAPI_NEXT_COMP); } return 0; }
ib_api_status_t osm_vendor_get_guid_ca_and_port(IN osm_vendor_t * const p_vend, IN ib_net64_t const guid, OUT VAPI_hca_hndl_t * p_hca_hndl, OUT VAPI_hca_id_t * p_hca_id, OUT uint8_t * p_hca_idx, OUT uint32_t * p_port_num) { ib_api_status_t status; VAPI_hca_id_t *p_ca_ids = NULL; VAPI_ret_t vapi_res; VAPI_hca_hndl_t hca_hndl; VAPI_hca_vendor_t hca_vendor; VAPI_hca_cap_t hca_cap; IB_gid_t *p_port_gid = NULL; uint16_t maxNumGids; ib_net64_t port_guid; uint32_t ca, portIdx, ca_count; OSM_LOG_ENTER(p_vend->p_log); CL_ASSERT(p_vend); /* * 1) Determine the number of CA's * 2) Allocate an array big enough to hold the ca info objects. * 3) Call again to retrieve the guids. */ status = __osm_vendor_get_ca_ids(p_vend, &p_ca_ids, &ca_count); if (status != IB_SUCCESS) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_get_guid_ca_and_port: ERR 3D16: " "Fail to get CA Ids.\n"); goto Exit; } /* * For each CA, retrieve the CA info attributes */ for (ca = 0; ca < ca_count; ca++) { /* get the HCA handle */ vapi_res = EVAPI_get_hca_hndl(p_ca_ids[ca], &hca_hndl); if (vapi_res != VAPI_OK) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_get_guid_ca_and_port: ERR 3D17: " "Fail to get HCA handle (%u).\n", vapi_res); goto Exit; } /* get the CA attributes - to know how many ports it has: */ if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { osm_log(p_vend->p_log, OSM_LOG_DEBUG, "osm_vendor_get_guid_ca_and_port: " "Querying CA %s.\n", p_ca_ids[ca]); } /* query and get the HCA capability */ vapi_res = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap); if (vapi_res != VAPI_OK) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_get_guid_ca_and_port: ERR 3D18: " "Fail to get HCA Capabilities (%u).\n", vapi_res); goto Exit; } /* go over all ports - to obtail their guids */ for (portIdx = 0; portIdx < hca_cap.phys_port_num; portIdx++) { vapi_res = VAPI_query_hca_gid_tbl(hca_hndl, portIdx + 1, 0, &maxNumGids, NULL); p_port_gid = (IB_gid_t *) malloc(maxNumGids * sizeof(IB_gid_t)); /* get the port guid */ vapi_res = VAPI_query_hca_gid_tbl(hca_hndl, portIdx + 1, maxNumGids, &maxNumGids, p_port_gid); if (vapi_res != VAPI_OK) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_get_guid_ca_and_port: ERR 3D19: " "Fail to get HCA Port GID (%d).\n", vapi_res); goto Exit; } /* convert to SF style */ __osm_vendor_gid_to_guid(p_port_gid[0], (VAPI_gid_t *) & port_guid); /* finally did we find it ? */ if (port_guid == guid) { *p_hca_hndl = hca_hndl; memcpy(p_hca_id, p_ca_ids[ca], sizeof(VAPI_hca_id_t)); *p_hca_idx = ca; *p_port_num = portIdx + 1; status = IB_SUCCESS; goto Exit; } free(p_port_gid); p_port_gid = NULL; } /* ALL PORTS */ } /* all HCAs */ osm_log(p_vend->p_log, OSM_LOG_ERROR, "osm_vendor_get_guid_ca_and_port: ERR 3D20: " "Fail to find HCA and Port for Port Guid 0x%" PRIx64 "\n", cl_ntoh64(guid)); status = IB_INVALID_GUID; Exit: if (p_ca_ids != NULL) free(p_ca_ids); if (p_port_gid != NULL) free(p_port_gid); OSM_LOG_EXIT(p_vend->p_log); return (status); }
/********************************************************************** * Initialize an Info Struct for the Given HCA by its Id **********************************************************************/ static ib_api_status_t __osm_ca_info_init(IN osm_vendor_t * const p_vend, IN VAPI_hca_id_t ca_id, OUT osm_ca_info_t * const p_ca_info) { ib_api_status_t status = IB_ERROR; VAPI_ret_t vapi_res; VAPI_hca_hndl_t hca_hndl; VAPI_hca_vendor_t hca_vendor; VAPI_hca_cap_t hca_cap; VAPI_hca_port_t hca_port; uint8_t port_num; IB_gid_t *p_port_gid; uint16_t maxNumGids; OSM_LOG_ENTER(p_vend->p_log); /* get the HCA handle */ vapi_res = EVAPI_get_hca_hndl(ca_id, &hca_hndl); if (vapi_res != VAPI_OK) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "__osm_ca_info_init: ERR 3D05: " "Fail to get HCA handle (%u).\n", vapi_res); goto Exit; } if (osm_log_is_active(p_vend->p_log, OSM_LOG_DEBUG)) { osm_log(p_vend->p_log, OSM_LOG_DEBUG, "__osm_ca_info_init: " "Querying CA %s.\n", ca_id); } /* query and get the HCA capability */ vapi_res = VAPI_query_hca_cap(hca_hndl, &hca_vendor, &hca_cap); if (vapi_res != VAPI_OK) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "__osm_ca_info_init: ERR 3D06: " "Fail to get HCA Capabilities (%u).\n", vapi_res); goto Exit; } /* get the guid of the HCA */ memcpy(&(p_ca_info->guid), hca_cap.node_guid, 8 * sizeof(u_int8_t)); p_ca_info->attr_size = 1; p_ca_info->p_attr = (ib_ca_attr_t *) malloc(sizeof(ib_ca_attr_t)); memcpy(&(p_ca_info->p_attr->ca_guid), hca_cap.node_guid, 8 * sizeof(u_int8_t)); /* now obtain the attributes of the ports */ p_ca_info->p_attr->num_ports = hca_cap.phys_port_num; p_ca_info->p_attr->p_port_attr = (ib_port_attr_t *) malloc(hca_cap.phys_port_num * sizeof(ib_port_attr_t)); for (port_num = 0; port_num < p_ca_info->p_attr->num_ports; port_num++) { /* query the port attributes */ vapi_res = VAPI_query_hca_port_prop(hca_hndl, port_num + 1, &hca_port); if (vapi_res != VAPI_OK) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "__osm_ca_info_init: ERR 3D07: " "Fail to get HCA Port Attributes (%d).\n", vapi_res); goto Exit; } /* first call to know the size of the gid table */ vapi_res = VAPI_query_hca_gid_tbl(hca_hndl, port_num + 1, 0, &maxNumGids, NULL); p_port_gid = (IB_gid_t *) malloc(maxNumGids * sizeof(IB_gid_t)); vapi_res = VAPI_query_hca_gid_tbl(hca_hndl, port_num + 1, maxNumGids, &maxNumGids, p_port_gid); if (vapi_res != VAPI_OK) { osm_log(p_vend->p_log, OSM_LOG_ERROR, "__osm_ca_info_init: ERR 3D12: " "Fail to get HCA Port GID (%d).\n", vapi_res); goto Exit; } __osm_vendor_gid_to_guid(p_port_gid[0], (IB_gid_t *) & p_ca_info->p_attr-> p_port_attr[port_num].port_guid); p_ca_info->p_attr->p_port_attr[port_num].lid = hca_port.lid; p_ca_info->p_attr->p_port_attr[port_num].link_state = hca_port.state; p_ca_info->p_attr->p_port_attr[port_num].sm_lid = hca_port.sm_lid; free(p_port_gid); } status = IB_SUCCESS; Exit: OSM_LOG_EXIT(p_vend->p_log); return (status); }