static void destroy_node(struct cmatest_node *node) { if (!node->cma_id) return; if (node->cma_id->qp) rdma_destroy_qp(node->cma_id); if (node->cq[SEND_CQ_INDEX]) ibv_destroy_cq(node->cq[SEND_CQ_INDEX]); if (node->cq[RECV_CQ_INDEX]) ibv_destroy_cq(node->cq[RECV_CQ_INDEX]); if (node->mem) { ibv_dereg_mr(node->mr); free(node->mem); } if (node->pd) ibv_dealloc_pd(node->pd); /* Destroy the RDMA ID after all device resources */ rdma_destroy_id(node->cma_id); }
static void rdma_trans_destroy(void *a) { Rdmatrans *rdma; struct ibv_qp_attr attr; rdma = a; if (rdma->connected) rdma_disconnect(rdma->cm_id); if (rdma->qp) { attr.qp_state = IBV_QPS_ERR; ibv_modify_qp(rdma->qp, &attr, IBV_QP_STATE); ibv_destroy_qp(rdma->qp); } if (rdma->cq) ibv_destroy_cq(rdma->cq); if (rdma->ch) ibv_destroy_comp_channel(rdma->ch); if (rdma->snd_mr) ibv_dereg_mr(rdma->snd_mr); if (rdma->snd_buf) free(rdma->snd_buf); if (rdma->rcv_mr) ibv_dereg_mr(rdma->rcv_mr); if (rdma->rcv_buf) free(rdma->rcv_buf); if (rdma->pd) ibv_dealloc_pd(rdma->pd); if (rdma->cm_id) rdma_destroy_id(rdma->cm_id); }
static UCS_CLASS_CLEANUP_FUNC(uct_ib_iface_t) { int ret; ret = ibv_destroy_cq(self->recv_cq); if (ret != 0) { ucs_warn("ibv_destroy_cq(recv_cq) returned %d: %m", ret); } ret = ibv_destroy_cq(self->send_cq); if (ret != 0) { ucs_warn("ibv_destroy_cq(send_cq) returned %d: %m", ret); } ucs_free(self->path_bits); }
/// The IBConnectionGroup default destructor. virtual ~IBConnectionGroup() { for (auto& c : conn_) c = nullptr; if (listen_id_) { int err = rdma_destroy_id(listen_id_); if (err) { L_(error) << "rdma_destroy_id() failed"; } listen_id_ = nullptr; } if (cq_) { int err = ibv_destroy_cq(cq_); if (err) { L_(error) << "ibv_destroy_cq() failed"; } cq_ = nullptr; } if (pd_) { int err = ibv_dealloc_pd(pd_); if (err) { L_(error) << "ibv_dealloc_pd() failed"; } pd_ = nullptr; } rdma_destroy_event_channel(ec_); }
static void rping_free_qp(struct rping_cb *cb) { ibv_destroy_qp(cb->qp); ibv_destroy_cq(cb->cq); ibv_destroy_comp_channel(cb->channel); ibv_dealloc_pd(cb->pd); }
static void free_sa_qp_cache(void) { struct mca_btl_openib_sa_qp_cache *cache, *tmp; cache = sa_qp_cache; while (NULL != cache) { /* free cache data */ if (cache->device_name) free(cache->device_name); if (NULL != cache->qp) ibv_destroy_qp(cache->qp); if (NULL != cache->ah) ibv_destroy_ah(cache->ah); if (NULL != cache->cq) ibv_destroy_cq(cache->cq); if (NULL != cache->mr) ibv_dereg_mr(cache->mr); if (NULL != cache->pd) ibv_dealloc_pd(cache->pd); tmp = cache->next; free(cache); cache = tmp; } sa_qp_cache = NULL; }
RDMAAdapter::~RDMAAdapter() { StopInternalThread(); CHECK(!ibv_destroy_cq(cq_)) << "Failed to destroy CQ"; CHECK(!ibv_destroy_comp_channel(channel_)) << "Failed to destroy channel"; CHECK(!ibv_dealloc_pd(pd_)) << "Failed to deallocate PD"; CHECK(!ibv_close_device(context_)) << "Failed to release context"; }
static int fi_ibv_rdm_tagged_find_max_inline_size(struct ibv_pd *pd, struct ibv_context *context) { struct ibv_qp_init_attr qp_attr; struct ibv_qp *qp = NULL; struct ibv_cq *cq = ibv_create_cq(context, 1, NULL, NULL, 0); assert(cq); int max_inline = 2; int rst = 0; memset(&qp_attr, 0, sizeof(qp_attr)); qp_attr.send_cq = cq; qp_attr.recv_cq = cq; qp_attr.qp_type = IBV_QPT_RC; qp_attr.cap.max_send_wr = 1; qp_attr.cap.max_recv_wr = 1; qp_attr.cap.max_send_sge = 1; qp_attr.cap.max_recv_sge = 1; do { if (qp) ibv_destroy_qp(qp); qp_attr.cap.max_inline_data = max_inline; qp = ibv_create_qp(pd, &qp_attr); if (qp) rst = max_inline; } while (qp && (max_inline *= 2)); if (rst != 0) { int pos = rst, neg = max_inline; do { max_inline = pos + (neg - pos) / 2; if (qp) ibv_destroy_qp(qp); qp_attr.cap.max_inline_data = max_inline; qp = ibv_create_qp(pd, &qp_attr); if (qp) pos = max_inline; else neg = max_inline; } while (neg - pos > 2); rst = pos; } if (qp) { ibv_destroy_qp(qp); } if (cq) { ibv_destroy_cq(cq); } return rst; }
int resource_destroy(resource_t *res) { int rc = 0; // Delete QP if (res->qp && ibv_destroy_qp(res->qp)){ fprintf(stderr, "failed to destroy QP\n"); rc = 1; } // Deregister MR if(res->mr_list && res->mr_size > 0){ int i; for(i=0; i<res->mr_size; i++){ ibv_dereg_mr(res->mr_list[i]); } free(res->mr_list); } if(res->buf){ free(res->buf); } // Delete CQ if (res->scq && ibv_destroy_cq(res->scq)){ fprintf(stderr, "failed to destroy SCQ\n"); rc = 1; } if (res->rcq && ibv_destroy_cq(res->rcq)){ fprintf(stderr, "failed to destroy RCQ\n"); rc = 1; } if(res->comp_ch && ibv_destroy_comp_channel(res->comp_ch)){ fprintf(stderr, "failed to destroy Complete CH\n"); rc = 1; } // Deallocate PD if (res->pd && ibv_dealloc_pd(res->pd)){ fprintf(stderr, "failed to deallocate PD\n"); rc = 1; } if (res->ib_ctx && ibv_close_device(res->ib_ctx)){ fprintf(stderr, "failed to close device context\n"); rc = 1; } return rc; }
void network_release() { ibv_dereg_mr(mr_data); rdma_destroy_qp(cm_id); ibv_destroy_cq(cq); ibv_destroy_comp_channel(comp_chan); rdma_destroy_id(cm_id); rdma_destroy_event_channel(cm_channel); }
void xfer_rdma_destroy_ctx(struct xfer_context *ctx) { rdma_destroy_qp(ctx->cm_id); ibv_destroy_cq(ctx->cq); ibv_destroy_comp_channel(ctx->ch); ibv_dereg_mr(ctx->send_mr); ibv_dereg_mr(ctx->recv_mr); ibv_dealloc_pd(ctx->pd); free(ctx); }
CompletionQueue& CompletionQueue::operator=(CompletionQueue&& other) { if (mQueue != nullptr && ibv_destroy_cq(mQueue)) { throw std::system_error(errno, std::generic_category()); } mQueue = other.mQueue; other.mQueue = nullptr; return *this; }
int __ibv_destroy_cq_1_0(struct ibv_cq_1_0 *cq) { int ret; ret = ibv_destroy_cq(cq->real_cq); if (ret) return ret; free(cq); return 0; }
int __ibv_destroy_cq_1_0(struct ibv_cq_1_0 *cq) { fprintf(stderr, "%s:%s:%d \n", __func__, __FILE__, __LINE__); int ret; ret = ibv_destroy_cq(cq->real_cq); if (ret) return ret; free(cq); return 0; }
static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f) { struct rdmaio_data *rd = td->io_ops->data; struct ibv_send_wr *bad_wr; /* unregister rdma buffer */ /* * Client sends notification to the server side */ /* refer to: http://linux.die.net/man/7/rdma_cm */ if ((rd->is_client == 1) && ((rd->rdma_protocol == FIO_RDMA_MEM_WRITE) || (rd->rdma_protocol == FIO_RDMA_MEM_READ))) { if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) { log_err("fio: ibv_post_send fail"); return 1; } dprint(FD_IO, "fio: close information sent success\n"); rdma_poll_wait(td, IBV_WC_SEND); } if (rd->is_client == 1) rdma_disconnect(rd->cm_id); else { rdma_disconnect(rd->child_cm_id); #if 0 rdma_disconnect(rd->cm_id); #endif } #if 0 if (get_next_channel_event(td, rd->cm_channel, RDMA_CM_EVENT_DISCONNECTED) != 0) { log_err("fio: wait for RDMA_CM_EVENT_DISCONNECTED\n"); return 1; } #endif ibv_destroy_cq(rd->cq); ibv_destroy_qp(rd->qp); if (rd->is_client == 1) rdma_destroy_id(rd->cm_id); else { rdma_destroy_id(rd->child_cm_id); rdma_destroy_id(rd->cm_id); } ibv_destroy_comp_channel(rd->channel); ibv_dealloc_pd(rd->pd); return 0; }
static void uct_ib_md_umr_qp_destroy(uct_ib_md_t *md) { #if HAVE_EXP_UMR if (md->umr_qp != NULL) { ibv_destroy_qp(md->umr_qp); } if (md->umr_cq != NULL) { ibv_destroy_cq(md->umr_cq); } #endif }
int BClientContext::destroy_context () { if (qp) TEST_NZ(ibv_destroy_qp (qp)); if (recv_memory_mr) TEST_NZ (ibv_dereg_mr (recv_memory_mr)); if (lock_result_mr) TEST_NZ (ibv_dereg_mr (lock_result_mr)); if (cq) TEST_NZ (ibv_destroy_cq (cq)); if (pd) TEST_NZ (ibv_dealloc_pd (pd)); if (ib_ctx) TEST_NZ (ibv_close_device (ib_ctx)); if (sockfd >= 0) TEST_NZ (close (sockfd)); return 0; }
int opal_common_verbs_qp_test(struct ibv_context *device_context, int flags) { int rc = OPAL_SUCCESS; struct ibv_pd *pd = NULL; struct ibv_cq *cq = NULL; /* Bozo check */ if (NULL == device_context || (0 == (flags & (OPAL_COMMON_VERBS_FLAGS_RC | OPAL_COMMON_VERBS_FLAGS_UD)))) { return OPAL_ERR_BAD_PARAM; } /* Try to make both the PD and CQ */ pd = ibv_alloc_pd(device_context); if (NULL == pd) { return OPAL_ERR_OUT_OF_RESOURCE; } cq = ibv_create_cq(device_context, 2, NULL, NULL, 0); if (NULL == cq) { rc = OPAL_ERR_OUT_OF_RESOURCE; goto out; } /* Now try to make the QP(s) of the desired type(s) */ if (flags & OPAL_COMMON_VERBS_FLAGS_RC && !make_qp(pd, cq, IBV_QPT_RC)) { rc = OPAL_ERR_NOT_SUPPORTED; goto out; } if (flags & OPAL_COMMON_VERBS_FLAGS_NOT_RC && make_qp(pd, cq, IBV_QPT_RC)) { rc = OPAL_ERR_TYPE_MISMATCH; goto out; } if (flags & OPAL_COMMON_VERBS_FLAGS_UD && !make_qp(pd, cq, IBV_QPT_UD)) { rc = OPAL_ERR_NOT_SUPPORTED; goto out; } out: /* Free the PD and/or CQ */ if (NULL != pd) { ibv_dealloc_pd(pd); } if (NULL != cq) { ibv_destroy_cq(cq); } return rc; }
static int rping_setup_qp(struct rping_cb *cb, struct rdma_cm_id *cm_id) { int ret; cb->pd = ibv_alloc_pd(cm_id->verbs); if (!cb->pd) { fprintf(stderr, "ibv_alloc_pd failed\n"); return errno; } DEBUG_LOG("created pd %p\n", cb->pd); cb->channel = ibv_create_comp_channel(cm_id->verbs); if (!cb->channel) { fprintf(stderr, "ibv_create_comp_channel failed\n"); ret = errno; goto err1; } DEBUG_LOG("created channel %p\n", cb->channel); cb->cq = ibv_create_cq(cm_id->verbs, RPING_SQ_DEPTH * 2, cb, cb->channel, 0); if (!cb->cq) { fprintf(stderr, "ibv_create_cq failed\n"); ret = errno; goto err2; } DEBUG_LOG("created cq %p\n", cb->cq); ret = ibv_req_notify_cq(cb->cq, 0); if (ret) { fprintf(stderr, "ibv_create_cq failed\n"); ret = errno; goto err3; } ret = rping_create_qp(cb); if (ret) { perror("rdma_create_qp"); goto err3; } DEBUG_LOG("created qp %p\n", cb->qp); return 0; err3: ibv_destroy_cq(cb->cq); err2: ibv_destroy_comp_channel(cb->channel); err1: ibv_dealloc_pd(cb->pd); return ret; }
static int resources_destroy(struct resources *res) { int rc = 0; if (res->qp) { if (ibv_destroy_qp(res->qp)) { fprintf(stderr, "failed to destroy QP\n"); rc = 1; } } if (res->mr) { if (ibv_dereg_mr(res->mr)) { fprintf(stderr, "failed to deregister MR\n"); rc = 1; } } if (res->buf) free(res->buf); if (res->cq) { if (ibv_destroy_cq(res->cq)) { fprintf(stderr, "failed to destroy CQ\n"); rc = 1; } } if (res->pd) { if (ibv_dealloc_pd(res->pd)) { fprintf(stderr, "failed to deallocate PD\n"); rc = 1; } } if (res->ib_ctx) { if (ibv_close_device(res->ib_ctx)) { fprintf(stderr, "failed to close device context\n"); rc = 1; } } if (res->sock >= 0) { if (close(res->sock)) { fprintf(stderr, "failed to close socket\n"); rc = 1; } } return rc; }
static inline int fi_ibv_get_qp_cap(struct ibv_context *ctx, struct fi_info *info) { struct ibv_pd *pd; struct ibv_cq *cq; struct ibv_qp *qp; struct ibv_qp_init_attr init_attr; int ret = 0; pd = ibv_alloc_pd(ctx); if (!pd) { VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_alloc_pd", errno); return -errno; } cq = ibv_create_cq(ctx, 1, NULL, NULL, 0); if (!cq) { VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_cq", errno); ret = -errno; goto err1; } memset(&init_attr, 0, sizeof init_attr); init_attr.send_cq = cq; init_attr.recv_cq = cq; init_attr.cap.max_send_wr = verbs_default_tx_size; init_attr.cap.max_recv_wr = verbs_default_rx_size; init_attr.cap.max_send_sge = verbs_default_tx_iov_limit; init_attr.cap.max_recv_sge = verbs_default_rx_iov_limit; init_attr.cap.max_inline_data = verbs_default_inline_size; init_attr.qp_type = IBV_QPT_RC; qp = ibv_create_qp(pd, &init_attr); if (!qp) { VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_create_qp", errno); ret = -errno; goto err2; } info->tx_attr->inject_size = init_attr.cap.max_inline_data; ibv_destroy_qp(qp); err2: ibv_destroy_cq(cq); err1: ibv_dealloc_pd(pd); return ret; }
/** ========================================================================= */ static void destroy_sa_qp(struct oib_port *port) { int i; // if the user just unregistered trap messages those messages may still // be on this list, wait 5 seconds for the thread to handle the response. for (i = 0; i < 5000; i++) { if (!LIST_EMPTY(&port->pending_reg_msg_head)) { usleep(1000); } else { DBGPRINT("destroy_sa_qp: wait %d ms for LIST_EMPTY\n", i); break; } } stop_ud_cq_monitor(port); join_port_thread(port); /* Free any remaining unregistration messages */ if (!LIST_EMPTY(&port->pending_reg_msg_head)) { OUTPUT_ERROR("Ignoring Pending Notice un-registation requests\n"); oib_sa_remove_all_pending_reg_msgs(port); } if (port->sa_ah) ibv_destroy_ah(port->sa_ah); if (port->sa_qp) ibv_destroy_qp(port->sa_qp); for (i = 0; i<port->num_userspace_recv_buf; i++) if (port->recv_bufs) ibv_dereg_mr(port->recv_bufs[i].mr); if (port->sa_qp_pd) ibv_dealloc_pd(port->sa_qp_pd); if (port->sa_qp_cq) ibv_destroy_cq(port->sa_qp_cq); if (port->recv_bufs) { free(port->recv_bufs); port->recv_bufs = NULL; } if (port->sa_qp_comp_channel) ibv_destroy_comp_channel(port->sa_qp_comp_channel); }
static void rdmasniff_cleanup(pcap_t *handle) { struct pcap_rdmasniff *priv = handle->priv; ibv_dereg_mr(priv->mr); ibv_destroy_flow(priv->flow); ibv_destroy_qp(priv->qp); ibv_destroy_cq(priv->cq); ibv_dealloc_pd(priv->pd); ibv_destroy_comp_channel(priv->channel); ibv_close_device(priv->context); free(priv->oneshot_buffer); pcap_cleanup_live_common(handle); }
static int ibw_conn_priv_destruct(struct ibw_conn_priv *pconn) { DEBUG(DEBUG_DEBUG, ("ibw_conn_priv_destruct(%p, cmid: %p)\n", pconn, pconn->cm_id)); /* pconn->wr_index is freed by talloc */ /* pconn->wr_index[i] are freed by talloc */ /* * tevent_fd must be removed before the fd is closed */ TALLOC_FREE(pconn->verbs_channel_event); /* destroy verbs */ if (pconn->cm_id!=NULL && pconn->cm_id->qp!=NULL) { rdma_destroy_qp(pconn->cm_id); pconn->cm_id->qp = NULL; } if (pconn->cq!=NULL) { ibv_destroy_cq(pconn->cq); pconn->cq = NULL; } if (pconn->verbs_channel!=NULL) { ibv_destroy_comp_channel(pconn->verbs_channel); pconn->verbs_channel = NULL; } /* free memory regions */ ibw_free_mr(&pconn->buf_send, &pconn->mr_send); ibw_free_mr(&pconn->buf_recv, &pconn->mr_recv); if (pconn->pd) { ibv_dealloc_pd(pconn->pd); pconn->pd = NULL; DEBUG(DEBUG_DEBUG, ("pconn=%p pd deallocated\n", pconn)); } if (pconn->cm_id) { rdma_destroy_id(pconn->cm_id); pconn->cm_id = NULL; DEBUG(DEBUG_DEBUG, ("pconn=%p cm_id destroyed\n", pconn)); } return 0; }
static ucs_status_t uct_ib_mlx5_check_dc(uct_ib_device_t *dev) { ucs_status_t status = UCS_OK; struct ibv_context *ctx = dev->ibv_context; struct ibv_qp_init_attr_ex qp_attr = {}; struct mlx5dv_qp_init_attr dv_attr = {}; struct ibv_pd *pd; struct ibv_cq *cq; struct ibv_qp *qp; pd = ibv_alloc_pd(ctx); if (pd == NULL) { ucs_error("ibv_alloc_pd() failed: %m"); return UCS_ERR_IO_ERROR; } cq = ibv_create_cq(ctx, 1, NULL, NULL, 0); if (cq == NULL) { ucs_error("ibv_create_cq() failed: %m"); status = UCS_ERR_IO_ERROR; goto err_cq; } qp_attr.send_cq = cq; qp_attr.recv_cq = cq; qp_attr.cap.max_send_wr = 1; qp_attr.cap.max_send_sge = 1; qp_attr.qp_type = IBV_QPT_DRIVER; qp_attr.comp_mask = IBV_QP_INIT_ATTR_PD; qp_attr.pd = pd; dv_attr.comp_mask = MLX5DV_QP_INIT_ATTR_MASK_DC; dv_attr.dc_init_attr.dc_type = MLX5DV_DCTYPE_DCI; /* create DCI qp successful means DC is supported */ qp = mlx5dv_create_qp(ctx, &qp_attr, &dv_attr); if (qp) { ibv_destroy_qp(qp); dev->flags |= UCT_IB_DEVICE_FLAG_DC; } ibv_destroy_cq(cq); err_cq: ibv_dealloc_pd(pd); return status; }
int rd_close(DEVICE *dev) { #if 0 if (Req.use_cm) cm_close(dev); else #endif ib_close1(dev); if (dev->ah) ibv_destroy_ah(dev->ah); if (dev->cq) ibv_destroy_cq(dev->cq); return 0; }
static int destroy_ctx_resources(struct pingpong_context *ctx) { int test_result = 0; if (ibv_destroy_qp(ctx->qp)) { fprintf(stderr, "failed to destroy QP\n"); test_result = 1; } if (ibv_destroy_cq(ctx->cq)) { fprintf(stderr, "failed to destroy CQ\n"); test_result = 1; } if (ibv_dereg_mr(ctx->mr)) { fprintf(stderr, "failed to deregister MR\n"); test_result = 1; } if (ctx->channel) { if (ibv_destroy_comp_channel(ctx->channel)) { fprintf(stderr, "failed to destroy channel \n"); test_result = 1; } } if (ibv_dealloc_pd(ctx->pd)) { fprintf(stderr, "failed to deallocate PD\n"); test_result = 1; } if (ibv_close_device(ctx->context)) { fprintf(stderr, "failed to close device context\n"); test_result = 1; } free(ctx->buf); free(ctx); free(tstamp); return test_result; }
static int destroy_ctx_resources(struct pingpong_context *ctx,int num_qps) { int i; int test_result = 0; for (i = 0; i < num_qps; i++) { if (ibv_destroy_qp(ctx->qp[i])) { fprintf(stderr, "failed to destroy QP\n"); test_result = 1; } } if (ibv_destroy_cq(ctx->cq)) { fprintf(stderr, "failed to destroy CQ\n"); test_result = 1; } if (ibv_dereg_mr(ctx->mr)) { fprintf(stderr, "failed to deregister MR\n"); test_result = 1; } if (ibv_dealloc_pd(ctx->pd)) { fprintf(stderr, "failed to deallocate PD\n"); test_result = 1; } if (ibv_close_device(ctx->context)) { fprintf(stderr, "failed to close device context\n"); test_result = 1; } free(ctx->buf); free(ctx->qp); free(ctx->scnt); free(ctx->ccnt); free(ctx); return test_result; }
void release_resources(void) { int i; // Destroy the registration cache reg_cache_destroy(nprocs); for(i = 0; i < nprocs ; i++) { if (me != i) { if(ibv_destroy_qp(conn.qp[i])) { printf("Exiting\n"); exit(1); } } } #if 0 if(ibv_destroy_cq(hca.cq)) { fprintf(stderr,"Couldn't destroy cq %s\n", ibv_get_device_name(hca.ib_dev)); } if(ibv_dealloc_pd(hca.pd)) { fprintf(stderr,"Couldn't free pd %s\n", ibv_get_device_name(hca.ib_dev)); } #endif free(conn.qp); free(conn.lid); free(conn.qp_num); free(rbuf.qp_num); free(rbuf.lid); #if 0 free(rbuf.rkey); free(rbuf.buf); #endif }
CompletionQueue::~CompletionQueue() { if (mQueue != nullptr && ibv_destroy_cq(mQueue)) { std::error_code ec(errno, std::generic_category()); LOG_ERROR("Failed to destroy completion queue [error = %1% %2%]", ec, ec.message()); } }