/* * Create a queue pair. */ int rd_create_qp(DEVICE *dev, CONNECTION *con, struct ibv_context *context, struct rdma_cm_id *id) { /* Set up and verify rd_atomic parameters */ { struct ibv_device_attr dev_attr; if (ibv_query_device(context, &dev_attr) != SUCCESS0) return error(SYS, "query device failed"); #if 1 if (Req.rd_atomic == 0) con->local.rd_atomic = dev_attr.max_qp_rd_atom; else if (Req.rd_atomic <= dev_attr.max_qp_rd_atom) con->local.rd_atomic = Req.rd_atomic; else return error(0, "device only supports %d (< %d) RDMA reads or atomics", dev_attr.max_qp_rd_atom, Req.rd_atomic); #endif } /* Create queue pair */ { struct ibv_qp_init_attr qp_attr ={ .send_cq = dev->cq, .recv_cq = dev->cq, .cap ={ .max_send_wr = dev->max_send_wr, .max_recv_wr = dev->max_recv_wr, .max_send_sge = 1, .max_recv_sge = 1, }, .qp_type = IBV_QPT_RC, }; #if 0 if (Req.use_cm) { if (rdma_create_qp(id, dev->pd, &qp_attr) != 0) return error(SYS, "failed to create QP"); dev->qp = id->qp; } else { #ifdef HAS_XRC if (dev->trans == IBV_QPT_XRC) { struct ibv_srq_init_attr srq_attr ={ .attr ={ .max_wr = dev->max_recv_wr, .max_sge = 1 } }; dev->xrc = ibv_open_xrc_domain(context, -1, O_CREAT); if (!dev->xrc) return error(SYS, "failed to open XRC domain"); dev->srq = ibv_create_xrc_srq(dev->pd, dev->xrc, dev->cq, &srq_attr); if (!dev->srq) return error(SYS, "failed to create SRQ"); qp_attr.cap.max_recv_wr = 0; qp_attr.cap.max_recv_sge = 0; qp_attr.xrc_domain = dev->xrc; } #endif /* HAS_XRC */ #endif con->qp = ibv_create_qp(dev->pd, &qp_attr); if (!con->qp) return error(SYS, "failed to create QP"); #if 0 } #endif }
/* This func. opens XRC domain */ int mca_btl_openib_open_xrc_domain(struct mca_btl_openib_device_t *device) { int len; char *xrc_file_name; const char *dev_name; #if OPAL_HAVE_CONNECTX_XRC_DOMAINS struct ibv_xrcd_init_attr xrcd_attr; #endif dev_name = ibv_get_device_name(device->ib_dev); len = asprintf(&xrc_file_name, "%s"OPAL_PATH_SEP"openib_xrc_domain_%s", opal_process_info.job_session_dir, dev_name); if (0 > len) { BTL_ERROR(("Failed to allocate memomry for XRC file name: %s\n", strerror(errno))); return OPAL_ERROR; } device->xrc_fd = open(xrc_file_name, O_CREAT, S_IWUSR|S_IRUSR); if (0 > device->xrc_fd) { BTL_ERROR(("Failed to open XRC domain file %s, errno says %s\n", xrc_file_name,strerror(errno))); free(xrc_file_name); return OPAL_ERROR; } #if OPAL_HAVE_CONNECTX_XRC_DOMAINS memset(&xrcd_attr, 0, sizeof xrcd_attr); xrcd_attr.comp_mask = IBV_XRCD_INIT_ATTR_FD | IBV_XRCD_INIT_ATTR_OFLAGS; xrcd_attr.fd = device->xrc_fd; xrcd_attr.oflags = O_CREAT; device->xrcd = ibv_open_xrcd(device->ib_dev_context, &xrcd_attr); if (NULL == device->xrcd) { #else device->xrc_domain = ibv_open_xrc_domain(device->ib_dev_context, device->xrc_fd, O_CREAT); if (NULL == device->xrc_domain) { #endif BTL_ERROR(("Failed to open XRC domain\n")); close(device->xrc_fd); free(xrc_file_name); return OPAL_ERROR; } return OPAL_SUCCESS; } /* This func. closes XRC domain */ int mca_btl_openib_close_xrc_domain(struct mca_btl_openib_device_t *device) { #if OPAL_HAVE_CONNECTX_XRC_DOMAINS if (NULL == device->xrcd) { #else if (NULL == device->xrc_domain) { #endif /* No XRC domain, just exit */ return OPAL_SUCCESS; } #if OPAL_HAVE_CONNECTX_XRC_DOMAINS if (ibv_close_xrcd(device->xrcd)) { #else if (ibv_close_xrc_domain(device->xrc_domain)) { #endif BTL_ERROR(("Failed to close XRC domain, errno %d says %s\n", device->xrc_fd, strerror(errno))); return OPAL_ERROR; } /* do we need to check exit status */ if (close(device->xrc_fd)) { BTL_ERROR(("Failed to close XRC file descriptor, errno %d says %s\n", device->xrc_fd, strerror(errno))); return OPAL_ERROR; } return OPAL_SUCCESS; } static void ib_address_constructor(ib_address_t *ib_addr) { ib_addr->key = NULL; ib_addr->subnet_id = 0; ib_addr->lid = 0; ib_addr->status = MCA_BTL_IB_ADDR_CLOSED; ib_addr->qp = NULL; OBJ_CONSTRUCT(&ib_addr->addr_lock, opal_mutex_t); OBJ_CONSTRUCT(&ib_addr->pending_ep, opal_list_t); } static void ib_address_destructor(ib_address_t *ib_addr) { if (NULL != ib_addr->key) { free(ib_addr->key); } OBJ_DESTRUCT(&ib_addr->addr_lock); OBJ_DESTRUCT(&ib_addr->pending_ep); } static int ib_address_init(ib_address_t *ib_addr, uint16_t lid, uint64_t s_id, opal_jobid_t ep_jobid) { ib_addr->key = malloc(SIZE_OF3(s_id, lid, ep_jobid)); if (NULL == ib_addr->key) { BTL_ERROR(("Failed to allocate memory for key\n")); return OPAL_ERROR; } memset(ib_addr->key, 0, SIZE_OF3(s_id, lid, ep_jobid)); /* creating the key = lid + s_id + ep_jobid */ memcpy(ib_addr->key, &lid, sizeof(lid)); memcpy((void*)((char*)ib_addr->key + sizeof(lid)), &s_id, sizeof(s_id)); memcpy((void*)((char*)ib_addr->key + sizeof(lid) + sizeof(s_id)), &ep_jobid, sizeof(ep_jobid)); /* caching lid and subnet id */ ib_addr->subnet_id = s_id; ib_addr->lid = lid; return OPAL_SUCCESS; } /* Create new entry in hash table for subnet_id and lid, * update the endpoint pointer. * Before call to this function you need to protect with */ int mca_btl_openib_ib_address_add_new (uint16_t lid, uint64_t s_id, opal_jobid_t ep_jobid, mca_btl_openib_endpoint_t *ep) { void *tmp; int ret = OPAL_SUCCESS; struct ib_address_t *ib_addr = OBJ_NEW(ib_address_t); ret = ib_address_init(ib_addr, lid, s_id, ep_jobid); if (OPAL_SUCCESS != ret ) { BTL_ERROR(("XRC Internal error. Failed to init ib_addr\n")); OBJ_DESTRUCT(ib_addr); return ret; } /* is it already in the table ?*/ OPAL_THREAD_LOCK(&mca_btl_openib_component.ib_lock); if (OPAL_SUCCESS != opal_hash_table_get_value_ptr(&mca_btl_openib_component.ib_addr_table, ib_addr->key, SIZE_OF3(s_id, lid, ep_jobid), &tmp)) { /* It is new one, lets put it on the table */ ret = opal_hash_table_set_value_ptr(&mca_btl_openib_component.ib_addr_table, ib_addr->key, SIZE_OF3(s_id, lid, ep_jobid), (void*)ib_addr); if (OPAL_SUCCESS != ret) { BTL_ERROR(("XRC Internal error." " Failed to add element to mca_btl_openib_component.ib_addr_table\n")); OPAL_THREAD_UNLOCK(&mca_btl_openib_component.ib_lock); OBJ_DESTRUCT(ib_addr); return ret; } /* update the endpoint with pointer to ib address */ ep->ib_addr = ib_addr; } else { /* so we have this one in the table, just add the pointer to the endpoint */ ep->ib_addr = (ib_address_t *)tmp; assert(lid == ep->ib_addr->lid && s_id == ep->ib_addr->subnet_id); OBJ_DESTRUCT(ib_addr); } OPAL_THREAD_UNLOCK(&mca_btl_openib_component.ib_lock); return ret; }