RDMAServerSocket::~RDMAServerSocket() { running = false; for (auto &&id : ids) { rdma_disconnect(id.get()); } rdma_disconnect(id.get()); // should cause an event to be fired, but doesn't. // use epoll w/ timeouts instead id.reset(); }
static int fio_rdmaio_close_file(struct thread_data *td, struct fio_file *f) { struct rdmaio_data *rd = td->io_ops->data; struct ibv_send_wr *bad_wr; /* unregister rdma buffer */ /* * Client sends notification to the server side */ /* refer to: http://linux.die.net/man/7/rdma_cm */ if ((rd->is_client == 1) && ((rd->rdma_protocol == FIO_RDMA_MEM_WRITE) || (rd->rdma_protocol == FIO_RDMA_MEM_READ))) { if (ibv_post_send(rd->qp, &rd->sq_wr, &bad_wr) != 0) { log_err("fio: ibv_post_send fail"); return 1; } dprint(FD_IO, "fio: close information sent success\n"); rdma_poll_wait(td, IBV_WC_SEND); } if (rd->is_client == 1) rdma_disconnect(rd->cm_id); else { rdma_disconnect(rd->child_cm_id); #if 0 rdma_disconnect(rd->cm_id); #endif } #if 0 if (get_next_channel_event(td, rd->cm_channel, RDMA_CM_EVENT_DISCONNECTED) != 0) { log_err("fio: wait for RDMA_CM_EVENT_DISCONNECTED\n"); return 1; } #endif ibv_destroy_cq(rd->cq); ibv_destroy_qp(rd->qp); if (rd->is_client == 1) rdma_destroy_id(rd->cm_id); else { rdma_destroy_id(rd->child_cm_id); rdma_destroy_id(rd->cm_id); } ibv_destroy_comp_channel(rd->channel); ibv_dealloc_pd(rd->pd); return 0; }
void client_disconnect(void){ struct rdma_event_channel *ec = s_ctx->ec; struct rdma_cm_id *id = s_ctx->id; struct rdma_cm_event *event = NULL; struct timeval start, end, dt; gettimeofday(&start, NULL); printf("ready to disconnect\n"); rdma_disconnect(id); printf("send disconnect\n"); gettimeofday(&end, NULL); timersub(&end, &start, &dt); long usec = dt.tv_usec + 1000000 * dt.tv_sec; printf("[rdma_disconnect] takes %ld micro_secs.\n", usec); while(1){ if(rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){ on_disconnect(event_copy.id); rdma_destroy_event_channel(ec); break; } } } return; }
void isert_conn_disconnect(struct isert_connection *isert_conn) { int err = rdma_disconnect(isert_conn->cm_id); if (unlikely(err)) pr_err("Failed to rdma disconnect, err:%d\n", err); }
static int rping_run_server(struct rping_cb *cb) { struct ibv_recv_wr *bad_wr; int ret; ret = rping_bind_server(cb); if (ret) return ret; sem_wait(&cb->sem); if (cb->state != CONNECT_REQUEST) { fprintf(stderr, "wait for CONNECT_REQUEST state %d\n", cb->state); return -1; } ret = rping_setup_qp(cb, cb->child_cm_id); if (ret) { fprintf(stderr, "setup_qp failed: %d\n", ret); return ret; } ret = rping_setup_buffers(cb); if (ret) { fprintf(stderr, "rping_setup_buffers failed: %d\n", ret); goto err1; } ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { fprintf(stderr, "ibv_post_recv failed: %d\n", ret); goto err2; } pthread_create(&cb->cqthread, NULL, cq_thread, cb); ret = rping_accept(cb); if (ret) { fprintf(stderr, "connect error %d\n", ret); goto err2; } ret = rping_test_server(cb); if (ret) { fprintf(stderr, "rping server failed: %d\n", ret); goto err3; } ret = 0; err3: rdma_disconnect(cb->child_cm_id); pthread_join(cb->cqthread, NULL); rdma_destroy_id(cb->child_cm_id); err2: rping_free_buffers(cb); err1: rping_free_qp(cb); return ret; }
void disconnect_conn_locked(conn_t *conn) { if (conn->transport.type == CONN_TYPE_RDMA) { switch (conn->state) { case CONN_STATE_CONNECTING: case CONN_STATE_CONNECTED: case CONN_STATE_RESOLVING_ROUTE: conn->state = CONN_STATE_DISCONNECTING; if (conn->rdma.cm_id) rdma_disconnect(conn->rdma.cm_id); break; case CONN_STATE_RESOLVING_ADDR: conn->state = CONN_STATE_DISCONNECTING; break; case CONN_STATE_DISCONNECTING: /* That case should not be possible because it would mean * this function got called twice. */ abort(); break; case CONN_STATE_DISCONNECTED: break; } } }
int ibw_disconnect(struct ibw_conn *conn) { int rc; struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); DEBUG(DEBUG_DEBUG, ("ibw_disconnect: cmid=%p\n", pconn->cm_id)); assert(pconn!=NULL); switch(conn->state) { case IBWC_ERROR: ibw_conn_priv_destruct(pconn); /* do this here right now */ break; case IBWC_CONNECTED: rc = rdma_disconnect(pconn->cm_id); if (rc) { sprintf(ibw_lasterr, "ibw_disconnect failed with %d\n", rc); DEBUG(DEBUG_ERR, (ibw_lasterr)); return rc; } break; default: DEBUG(DEBUG_DEBUG, ("invalid state for disconnect: %d\n", conn->state)); break; } return 0; }
void close() { util::spinlock::scoped_lock lk(mtx_); if(buffer_mr_) { ibv_dereg_mr(buffer_mr_); buffer_mr_ = 0; } if(server_msg_mr_) { ibv_dereg_mr(server_msg_mr_); server_msg_mr_ = 0; } if(client_msg_mr_) { ibv_dereg_mr(client_msg_mr_); client_msg_mr_ = 0; } if(id_) { rdma_disconnect(id_); id_ = 0; } size_ = 0; }
// closes all connections inline int closeConnections() { for (int i=0; i<threadCounter; i++) { while (!cqPollerThreadEnded[i]) usleep(1000); } waitSendThread(); for (unsigned int i=0; i<connections.size(); i++) rdma_disconnect(connections[i]->id); for (unsigned int i=0; i<connections.size(); i++) waitForEvent(connections[i]->id, RDMA_CM_EVENT_DISCONNECTED, connections[i]->getContext()); for(unsigned i=0;i<connections.size();++i) { connections[i]->destroyConnection(); delete connections[i]; connections[i]=NULL; } for (int i=0; i<threadCounter; i++) free(contexts[i]); return 0; }
static void rdma_trans_destroy(void *a) { Rdmatrans *rdma; struct ibv_qp_attr attr; rdma = a; if (rdma->connected) rdma_disconnect(rdma->cm_id); if (rdma->qp) { attr.qp_state = IBV_QPS_ERR; ibv_modify_qp(rdma->qp, &attr, IBV_QP_STATE); ibv_destroy_qp(rdma->qp); } if (rdma->cq) ibv_destroy_cq(rdma->cq); if (rdma->ch) ibv_destroy_comp_channel(rdma->ch); if (rdma->snd_mr) ibv_dereg_mr(rdma->snd_mr); if (rdma->snd_buf) free(rdma->snd_buf); if (rdma->rcv_mr) ibv_dereg_mr(rdma->rcv_mr); if (rdma->rcv_buf) free(rdma->rcv_buf); if (rdma->pd) ibv_dealloc_pd(rdma->pd); if (rdma->cm_id) rdma_destroy_id(rdma->cm_id); }
RDMACMSocket::~RDMACMSocket() { rdma_disconnect(this->client_id); rdma_dereg_mr(this->verbs_mr); rdma_destroy_ep(this->client_id); this->verbs_buf.free(); }
int xfer_rdma_finalize(struct xfer_data *data) { struct rdma_cm_event *event; int rc; if (data->servername) { rc = rdma_disconnect(data->cm_id); if (rc) { perror("rdma_disconnect"); fprintf(stderr, "%d:%s: rdma disconnect error\n", pid, __func__); return -1; } } rdma_get_cm_event(data->cm_channel, &event); if (event->event != RDMA_CM_EVENT_DISCONNECTED) fprintf(stderr, "%d:%s: unexpected event during disconnect %d\n", pid, __func__, event->event); rdma_ack_cm_event(event); rdma_destroy_id(data->cm_id); rdma_destroy_event_channel(data->cm_channel); return 0; }
void client_disconnect(void){ struct rdma_event_channel *ec = s_ctx->ec; struct rdma_cm_id *id = s_ctx->id; struct rdma_cm_event *event = NULL; printf("ready to disconnect\n"); rdma_disconnect(id); printf("send disconnect\n"); while(1){ if(rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){ on_disconnect(event_copy.id); rdma_destroy_event_channel(ec); break; } } } return; }
void RDMAServerSocket::cm_events() const { eventThread.send([=]() { hydra::util::epoll poll; epoll_event poll_event; poll_event.events = EPOLLIN; poll_event.data.fd = ec->fd; poll.add(ec->fd, &poll_event); while (running) { rdma_cm_event *cm_event = nullptr; int ret = poll.wait(&poll_event, 1, 2000); if (ret) { if (poll_event.data.fd == ec->fd) { check_zero(rdma_get_cm_event(ec.get(), &cm_event)); check_zero(cm_event->status); if (cm_event->event == RDMA_CM_EVENT_CONNECT_REQUEST) { accept(client_t(cm_event->id)); } else if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED) { rdma_disconnect(cm_event->id); } check_zero(rdma_ack_cm_event(cm_event)); } else { log_err() << "Unkown fd " << poll_event.data.fd << " set. Expected " << id->channel->fd; std::terminate(); } } } }); }
void on_completion(struct ibv_wc *wc) { struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id; if (wc->status != IBV_WC_SUCCESS) die("on_completion: status is not IBV_WC_SUCCESS."); if (wc->opcode & IBV_WC_RECV) { conn->recv_state++; printf("RECV: Recieved: TYPE=%d\n", conn->recv_msg->type); if (conn->recv_msg->type == MSG_MR) { memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr)); post_receives(conn); /* only rearm for MSG_MR */ if (conn->send_state == SS_INIT) /* received peer's MR before sending ours, so send ours back */ send_mr(conn); } } else { conn->send_state++; printf("SEND: Sent out: TYPE=%d\n", conn->send_msg->type); } if (conn->send_state == SS_MR_SENT && conn->recv_state == RS_MR_RECV) { /* struct ibv_send_wr wr, *bad_wr = NULL; struct ibv_sge sge; if (s_mode == M_WRITE) printf(" -> received MSG_MR. writing message to remote memory...\n"); else printf(" -> received MSG_MR. reading message from remote memory...\n"); memset(&wr, 0, sizeof(wr)); wr.wr_id = (uintptr_t)conn; wr.opcode = (s_mode == M_WRITE) ? IBV_WR_RDMA_WRITE : IBV_WR_RDMA_READ; wr.sg_list = &sge; wr.num_sge = 1; wr.send_flags = IBV_SEND_SIGNALED; wr.wr.rdma.remote_addr = (uintptr_t)conn->peer_mr.addr; wr.wr.rdma.rkey = conn->peer_mr.rkey; sge.addr = (uintptr_t)conn->rdma_local_region; sge.length = RDMA_BUFFER_SIZE; sge.lkey = conn->rdma_local_mr->lkey; TEST_NZ(ibv_post_send(conn->qp, &wr, &bad_wr)); printf("PSEND: Posted send request: MSG=%s\n", conn->rdma_local_region); */ conn->send_msg->type = MSG_DONE; send_message(conn); } else if (conn->send_state == SS_DONE_SENT && conn->recv_state == RS_DONE_RECV) { printf(" -> remote buffer: %s\n", get_peer_message_region(conn)); rdma_disconnect(conn->id); } printf("== STATE: send=%d / recv=%d ==\n", conn->send_state, conn->recv_state); }
void IBConnection::disconnect() { L_(debug) << "[" << index_ << "] " << "disconnect"; int err = rdma_disconnect(cm_id_); if (err) throw InfinibandException("rdma_disconnect failed"); }
/** * xprt_rdma_inject_disconnect - inject a connection fault * @xprt: transport context * * If @xprt is connected, disconnect it to simulate spurious connection * loss. */ static void xprt_rdma_inject_disconnect(struct rpc_xprt *xprt) { struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); trace_xprtrdma_inject_dsc(r_xprt); rdma_disconnect(r_xprt->rx_ia.ri_id); }
static void xprt_rdma_inject_disconnect(struct rpc_xprt *xprt) { struct rpcrdma_xprt *r_xprt = container_of(xprt, struct rpcrdma_xprt, rx_xprt); pr_info("rpcrdma: injecting transport disconnect on xprt=%p\n", xprt); rdma_disconnect(r_xprt->rx_ia.ri_id); }
static int rping_run_client(struct rping_cb *cb) { struct ibv_recv_wr *bad_wr; int ret; ret = rping_bind_client(cb); if (ret) return ret; ret = rping_setup_qp(cb, cb->cm_id); if (ret) { fprintf(stderr, "setup_qp failed: %d\n", ret); return ret; } ret = rping_setup_buffers(cb); if (ret) { fprintf(stderr, "rping_setup_buffers failed: %d\n", ret); goto err1; } ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { fprintf(stderr, "ibv_post_recv failed: %d\n", ret); goto err2; } ret = pthread_create(&cb->cqthread, NULL, cq_thread, cb); if (ret) { perror("pthread_create"); goto err2; } ret = rping_connect_client(cb); if (ret) { fprintf(stderr, "connect error %d\n", ret); goto err3; } ret = rping_test_client(cb); if (ret) { fprintf(stderr, "rping client failed: %d\n", ret); goto err4; } ret = 0; err4: rdma_disconnect(cb->cm_id); err3: pthread_join(cb->cqthread, NULL); err2: rping_free_buffers(cb); err1: rping_free_qp(cb); return ret; }
static void *rping_persistent_server_thread(void *arg) { struct rping_cb *cb = arg; struct ibv_recv_wr *bad_wr; int ret; ret = rping_setup_qp(cb, cb->child_cm_id); if (ret) { fprintf(stderr, "setup_qp failed: %d\n", ret); goto err0; } ret = rping_setup_buffers(cb); if (ret) { fprintf(stderr, "rping_setup_buffers failed: %d\n", ret); goto err1; } ret = ibv_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { fprintf(stderr, "ibv_post_recv failed: %d\n", ret); goto err2; } ret = pthread_create(&cb->cqthread, NULL, cq_thread, cb); if (ret) { perror("pthread_create"); goto err2; } ret = rping_accept(cb); if (ret) { fprintf(stderr, "connect error %d\n", ret); goto err3; } rping_test_server(cb); rdma_disconnect(cb->child_cm_id); pthread_join(cb->cqthread, NULL); rping_free_buffers(cb); rping_free_qp(cb); rdma_destroy_id(cb->child_cm_id); free_cb(cb); return NULL; err3: pthread_cancel(cb->cqthread); pthread_join(cb->cqthread, NULL); err2: rping_free_buffers(cb); err1: rping_free_qp(cb); err0: free_cb(cb); return NULL; }
/* Like tcp_fin - called when SDP_MID_DISCONNECT is received */ static void sdp_handle_disconn(struct sdp_sock *ssk) { sdp_dbg(ssk->socket, "%s\n", __func__); SDP_WLOCK_ASSERT(ssk); if (TCPS_HAVERCVDFIN(ssk->state) == 0) socantrcvmore(ssk->socket); switch (ssk->state) { case TCPS_SYN_RECEIVED: case TCPS_ESTABLISHED: ssk->state = TCPS_CLOSE_WAIT; break; case TCPS_FIN_WAIT_1: /* Received a reply FIN - start Infiniband tear down */ sdp_dbg(ssk->socket, "%s: Starting Infiniband tear down sending DREQ\n", __func__); sdp_cancel_dreq_wait_timeout(ssk); ssk->qp_active = 0; if (ssk->id) { struct rdma_cm_id *id; id = ssk->id; SDP_WUNLOCK(ssk); rdma_disconnect(id); SDP_WLOCK(ssk); } else { sdp_warn(ssk->socket, "%s: ssk->id is NULL\n", __func__); return; } break; case TCPS_TIME_WAIT: /* This is a mutual close situation and we've got the DREQ from the peer before the SDP_MID_DISCONNECT */ break; case TCPS_CLOSED: /* FIN arrived after IB teardown started - do nothing */ sdp_dbg(ssk->socket, "%s: fin in state %s\n", __func__, sdp_state_str(ssk->state)); return; default: sdp_warn(ssk->socket, "%s: FIN in unexpected state. state=%d\n", __func__, ssk->state); break; } }
void cfio_rdma_client_final() { cfio_rdma_client_wait(NULL); // rdma_debug("final ..."); rdma_disconnect(rdma_conn->id); wait_rdma_event(); rdma_destroy_event_channel(ec); // rdma_debug("final successfully"); }
static int p9_cm_event_handler(struct rdma_cm_id *id, struct rdma_cm_event *event) { struct p9_client *c = id->context; struct p9_trans_rdma *rdma = c->trans; switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: BUG_ON(rdma->state != P9_RDMA_INIT); rdma->state = P9_RDMA_ADDR_RESOLVED; break; case RDMA_CM_EVENT_ROUTE_RESOLVED: BUG_ON(rdma->state != P9_RDMA_ADDR_RESOLVED); rdma->state = P9_RDMA_ROUTE_RESOLVED; break; case RDMA_CM_EVENT_ESTABLISHED: BUG_ON(rdma->state != P9_RDMA_ROUTE_RESOLVED); rdma->state = P9_RDMA_CONNECTED; break; case RDMA_CM_EVENT_DISCONNECTED: if (rdma) rdma->state = P9_RDMA_CLOSED; if (c) c->status = Disconnected; break; case RDMA_CM_EVENT_TIMEWAIT_EXIT: break; case RDMA_CM_EVENT_ADDR_CHANGE: case RDMA_CM_EVENT_ROUTE_ERROR: case RDMA_CM_EVENT_DEVICE_REMOVAL: case RDMA_CM_EVENT_MULTICAST_JOIN: case RDMA_CM_EVENT_MULTICAST_ERROR: case RDMA_CM_EVENT_REJECTED: case RDMA_CM_EVENT_CONNECT_REQUEST: case RDMA_CM_EVENT_CONNECT_RESPONSE: case RDMA_CM_EVENT_CONNECT_ERROR: case RDMA_CM_EVENT_ADDR_ERROR: case RDMA_CM_EVENT_UNREACHABLE: c->status = Disconnected; rdma_disconnect(rdma->cm_id); break; default: BUG(); } complete(&rdma->cm_done); return 0; }
static void krping_run_server(struct krping_cb *cb) { struct ib_recv_wr *bad_wr; int ret; ret = krping_bind_server(cb); if (ret) return; ret = krping_setup_qp(cb, cb->child_cm_id); if (ret) { log(LOG_ERR, "setup_qp failed: %d\n", ret); return; } ret = krping_setup_buffers(cb); if (ret) { log(LOG_ERR, "krping_setup_buffers failed: %d\n", ret); goto err1; } ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); if (ret) { log(LOG_ERR, "ib_post_recv failed: %d\n", ret); goto err2; } ret = krping_accept(cb); if (ret) { log(LOG_ERR, "connect error %d\n", ret); goto err2; } if (cb->wlat) krping_wlat_test_server(cb); else if (cb->rlat) krping_rlat_test_server(cb); else if (cb->bw) krping_bw_test_server(cb); else krping_test_server(cb); rdma_disconnect(cb->child_cm_id); rdma_destroy_id(cb->child_cm_id); err2: krping_free_buffers(cb); err1: krping_free_qp(cb); }
static void rdma_close(struct p9_client *client) { struct p9_trans_rdma *rdma; if (!client) return; rdma = client->trans; if (!rdma) return; client->status = Disconnected; rdma_disconnect(rdma->cm_id); rdma_destroy_trans(rdma); }
void iser_conn_terminate(struct iser_conn *ib_conn) { int err = 0; iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, ISER_CONN_TERMINATING); err = rdma_disconnect(ib_conn->cma_id); if (err) iser_err("Failed to disconnect, conn: 0x%p err %d\n", ib_conn,err); wait_event_interruptible(ib_conn->wait, ib_conn->state == ISER_CONN_DOWN); iser_conn_put(ib_conn, 1); }
void on_completion(struct ibv_wc *wc) { struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id; if (wc->status != IBV_WC_SUCCESS) die("on_completion: status is not IBV_WC_SUCCESS."); if (wc->opcode & IBV_WC_RECV) printf("received message: %s\n", conn->recv_region); else if (wc->opcode == IBV_WC_SEND) printf("send completed successfully.\n"); else die("on_completion: completion isn't a send or a receive."); if (++conn->num_completions == 2) rdma_disconnect(conn->id); }
void on_completion(struct ibv_wc *wc) { struct connection *conn = (struct connection *)(uintptr_t)wc->wr_id; if (wc->status != IBV_WC_SUCCESS){ if (wc->opcode == IBV_WC_RDMA_WRITE) printf("IBV_WC_RDMA_WRITE failed!\n"); die("on_completion: status is not IBV_WC_SUCCESS."); } if (wc->opcode & IBV_WC_RECV) { if ((conn->recv_state == RS_INIT) && (conn->recv_msg->type == MSG_MR)) { //RECV Server MR memcpy(&conn->peer_mr, &conn->recv_msg->data.mr, sizeof(conn->peer_mr)); post_receives(conn); /* only rearm for MSG_MR */ conn->recv_state = RS_MR_RECV; if (conn->send_state == SS_INIT && conn->recv_state == RS_MR_RECV) { sem_post(&init_wait); //end of init } }else if (conn->recv_state == RS_MR_RECV){ //RECV Server Done msg conn->recv_state = RS_DONE_RECV; } }else { //Send completion if ((conn->send_state == SS_INIT) && (conn->recv_state == RS_MR_RECV)) { if (wc->opcode == IBV_WC_RDMA_READ){ time_stamp(7); sem_post(&read_ops); } if (wc->opcode == IBV_WC_SEND){ //Send Done msg conn->send_state = SS_DONE_SENT; sem_post(&done_ops); } if (wc->opcode == IBV_WC_RDMA_WRITE){ sem_post(&write_ops); time_stamp(3); } } } if ((conn->send_state == SS_DONE_SENT) && (conn->recv_state == RS_DONE_RECV)) { //printf("remote buffer: %s\n", get_local_message_region(conn)); rdma_disconnect(conn->id); } }
/* * rpcrdma_ep_disconnect * * This is separate from destroy to facilitate the ability * to reconnect without recreating the endpoint. * * This call is not reentrant, and must not be made in parallel * on the same endpoint. */ int rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia) { int rc; rpcrdma_clean_cq(ep->rep_cq); rc = rdma_disconnect(ia->ri_id); if (!rc) { /* returns without wait if not connected */ wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 1); dprintk("RPC: %s: after wait, %sconnected\n", __func__, (ep->rep_connected == 1) ? "still " : "dis"); } else { dprintk("RPC: %s: rdma_disconnect %i\n", __func__, rc); ep->rep_connected = rc; } return rc; }
/** * triggers start of the disconnect procedures and wait for them to be done */ void iser_conn_terminate(struct iser_conn *ib_conn) { int err = 0; /* change the ib conn state only if the conn is UP, however always call * rdma_disconnect since this is the only way to cause the CMA to change * the QP state to ERROR */ iser_conn_state_comp_exch(ib_conn, ISER_CONN_UP, ISER_CONN_TERMINATING); err = rdma_disconnect(ib_conn->cma_id); if (err) iser_err("Failed to disconnect, conn: 0x%p err %d\n", ib_conn,err); wait_event_interruptible(ib_conn->wait, ib_conn->state == ISER_CONN_DOWN); iser_conn_put(ib_conn, 1); /* deref ib conn deallocate */ }