int main(int argc, char **argv) { struct sockaddr_in addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *listener = NULL; struct rdma_event_channel *ec = NULL; uint16_t port = 0; memset(&addr, 0, sizeof(addr)); addr.sin_family = AF_INET; TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_bind_addr(listener, (struct sockaddr *)&addr)); TEST_NZ(rdma_listen(listener, 10)); /* backlog=10 is arbitrary */ port = ntohs(rdma_get_src_port(listener)); printf("listening on port %d.\n", port); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) break; } rdma_destroy_id(listener); rdma_destroy_event_channel(ec); return 0; }
int main(int argc, char **argv) { struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *conn= NULL; struct rdma_event_channel *ec = NULL; if (argc != 3) die("usage: client <server-address> <server-port>"); TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) break; } rdma_destroy_event_channel(ec); return 0; }
void client_disconnect(void){ struct rdma_event_channel *ec = s_ctx->ec; struct rdma_cm_id *id = s_ctx->id; struct rdma_cm_event *event = NULL; printf("ready to disconnect\n"); rdma_disconnect(id); printf("send disconnect\n"); while(1){ if(rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){ on_disconnect(event_copy.id); rdma_destroy_event_channel(ec); break; } } } return; }
static void *cma_thread(void *arg) { struct rdma_cm_event *event; int ret; while (1) { ret = rdma_get_cm_event(test.channel, &event); if (ret) { perror("rdma_get_cm_event"); break; } switch (event->event) { case RDMA_CM_EVENT_MULTICAST_ERROR: case RDMA_CM_EVENT_ADDR_CHANGE: printf("mckey: event: %s, status: %d\n", rdma_event_str(event->event), event->status); break; default: break; } rdma_ack_cm_event(event); } return NULL; }
static int get_next_channel_event(struct thread_data *td, struct rdma_event_channel *channel, enum rdma_cm_event_type wait_event) { struct rdmaio_data *rd = td->io_ops->data; struct rdma_cm_event *event; int ret; ret = rdma_get_cm_event(channel, &event); if (ret) { log_err("fio: rdma_get_cm_event: %d\n", ret); return 1; } if (event->event != wait_event) { log_err("fio: event is %s instead of %s\n", rdma_event_str(event->event), rdma_event_str(wait_event)); return 1; } switch (event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: rd->child_cm_id = event->id; break; default: break; } rdma_ack_cm_event(event); return 0; }
void wait_event(struct rdma_event_channel *ec, int et, int (*handler)(struct rdma_cm_id *id)) { struct rdma_cm_event *ep = NULL; while (rdma_get_cm_event(ec, &ep) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, ep, sizeof(*ep)); rdma_ack_cm_event(ep); if (et == event_copy.event) { if (handler) { handler(event_copy.id); } break; } else { fprintf(stderr, "got an event (=%d) not what expected (=%d). " "event.status:%d\n", event_copy.event, et, event_copy.status); // // infiniband/cm.h ib_cm_rej_reason // IB_CM_REJ_INVALID_SERVICE_ID = 8, exit(EXIT_FAILURE); } } }
void rdmaWaitEvent(struct rdma_event_channel *ec, int et, int (*handler)(struct rdma_cm_id *id)) { struct rdma_cm_event *ep = NULL; while (rdma_get_cm_event(ec, &ep) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, ep, sizeof(*ep)); rdma_ack_cm_event(ep); if (et == event_copy.event) { if (handler) { handler(event_copy.id); } break; } else { fprintf(stderr, "got an event (=%d) not what expected (=%d). " "event.status:%d\n", event_copy.event, et, event_copy.status); switch (event_copy.event) { case RDMA_CM_EVENT_REJECTED: fprintf(stderr, "RDMA connection rejected. The peer may be down.\n"); break; default: break; } // // infiniband/cm.h ib_cm_rej_reason // IB_CM_REJ_INVALID_SERVICE_ID = 8, exit(EXIT_FAILURE); } } }
int xfer_rdma_finalize(struct xfer_data *data) { struct rdma_cm_event *event; int rc; if (data->servername) { rc = rdma_disconnect(data->cm_id); if (rc) { perror("rdma_disconnect"); fprintf(stderr, "%d:%s: rdma disconnect error\n", pid, __func__); return -1; } } rdma_get_cm_event(data->cm_channel, &event); if (event->event != RDMA_CM_EVENT_DISCONNECTED) fprintf(stderr, "%d:%s: unexpected event during disconnect %d\n", pid, __func__, event->event); rdma_ack_cm_event(event); rdma_destroy_id(data->cm_id); rdma_destroy_event_channel(data->cm_channel); return 0; }
void client_disconnect(void){ struct rdma_event_channel *ec = s_ctx->ec; struct rdma_cm_id *id = s_ctx->id; struct rdma_cm_event *event = NULL; struct timeval start, end, dt; gettimeofday(&start, NULL); printf("ready to disconnect\n"); rdma_disconnect(id); printf("send disconnect\n"); gettimeofday(&end, NULL); timersub(&end, &start, &dt); long usec = dt.tv_usec + 1000000 * dt.tv_sec; printf("[rdma_disconnect] takes %ld micro_secs.\n", usec); while(1){ if(rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){ on_disconnect(event_copy.id); rdma_destroy_event_channel(ec); break; } } } return; }
void RDMAServerSocket::cm_events() const { eventThread.send([=]() { hydra::util::epoll poll; epoll_event poll_event; poll_event.events = EPOLLIN; poll_event.data.fd = ec->fd; poll.add(ec->fd, &poll_event); while (running) { rdma_cm_event *cm_event = nullptr; int ret = poll.wait(&poll_event, 1, 2000); if (ret) { if (poll_event.data.fd == ec->fd) { check_zero(rdma_get_cm_event(ec.get(), &cm_event)); check_zero(cm_event->status); if (cm_event->event == RDMA_CM_EVENT_CONNECT_REQUEST) { accept(client_t(cm_event->id)); } else if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED) { rdma_disconnect(cm_event->id); } check_zero(rdma_ack_cm_event(cm_event)); } else { log_err() << "Unkown fd " << poll_event.data.fd << " set. Expected " << id->channel->fd; std::terminate(); } } } }); }
void wait_rdma_event() { struct rdma_cm_event *event = NULL; while (!rdma_get_cm_event(ec, &event)) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (client_on_event(&event_copy)) break; } }
static int connect_events(void) { struct rdma_cm_event *event; int ret = 0; while (test.connects_left && !ret) { ret = rdma_get_cm_event(test.channel, &event); if (!ret) { ret = cma_handler(event->id, event); rdma_ack_cm_event(event); } } return ret; }
void diod_rdma_accept_one (Npsrv *srv, diod_rdma_t rdma) { Npconn *conn; Nptrans *trans; struct rdma_cm_event *event; struct rdma_cm_id *cmid; enum rdma_cm_event_type etype; int n; n = rdma_get_cm_event(rdma->event_channel, &event); if (n) errn_exit (n, "rdma_get_cm_event"); cmid = (struct rdma_cm_id *)event->id; etype = event->event; rdma_ack_cm_event(event); switch (etype) { case RDMA_CM_EVENT_CONNECT_REQUEST: msg ("rdma: connection request"); trans = np_rdmatrans_create(cmid, rdma_qdepth, rdma_maxmsize); if (trans) { conn = np_conn_create(srv, trans, "rdma", 0); cmid->context = conn; np_srv_add_conn(srv, conn); } else errn (np_rerror (), "np_rdmatrns_create failed"); break; case RDMA_CM_EVENT_ESTABLISHED: msg ("rdma: connection established"); break; case RDMA_CM_EVENT_DISCONNECTED: msg ("rdma: connection shutting down"); conn = cmid->context; //np_conn_shutdown(conn); /* FIXME: clean up properly */ break; default: msg ("rdma: event %d received waiting for a connect request\n", etype); } }
inline bool get_next_event( rdma_event_channel *event_channel, rdma_cm_event & event_copy, Connection * c , boost::system::error_code &ec ) { if(!event_channel) { HPX_IBVERBS_THROWS_IF(ec, boost::asio::error::not_connected); return false; } rdma_cm_event * event = NULL; if(rdma_get_cm_event(event_channel, &event) == 0) { std::memcpy(&event_copy, event, sizeof(rdma_cm_event)); rdma_ack_cm_event(event); if(event_copy.event == RDMA_CM_EVENT_DISCONNECTED) { c->on_disconnect(event_copy.id); return get_next_event(event_channel, event_copy, c, ec); } return true; } else { int verrno = errno; if(verrno == EBADF) return false; if(verrno == EAGAIN) return false; if(verrno == EWOULDBLOCK) return false; boost::system::error_code err(verrno, boost::system::system_category()); HPX_IBVERBS_THROWS_IF( ec , err ); return false; } HPX_ASSERT(false); return false; }
static void *cm_thread(void *arg) { struct rping_cb *cb = arg; struct rdma_cm_event *event; int ret; while (1) { ret = rdma_get_cm_event(cb->cm_channel, &event); if (ret) { perror("rdma_get_cm_event"); exit(ret); } ret = rping_cma_event_handler(event->id, event); rdma_ack_cm_event(event); if (ret) exit(ret); } }
static int disconnect_events(void) { struct rdma_cm_event *event; int ret = 0; while (test.disconnects_left && !ret) { ret = rdma_get_cm_event(test.channel, &event); if (!ret) { ret = cma_handler(event->id, event); rdma_ack_cm_event(event); } else { perror("cmatose: failure in rdma_get_cm_event in disconnect events"); ret = errno; } } return ret; }
void Connector::event_loop(struct rdma_event_channel* ec_, bool exit_on_disconn) { struct rdma_cm_event* event_ = NULL; struct rdma_conn_param cm_param; build_param(&cm_param); while (rdma_get_cm_event(ec_, &event_) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event_, sizeof(*event_) ); rdma_ack_cm_event(event_); if (event_copy.event == RDMA_CM_EVENT_ADDR_RESOLVED) { build_conn(event_copy.id); ib_end_->on_pre_conn(event_copy.id); TEST_NZ(rdma_resolve_route(event_copy.id, TIMEOUT_IN_MS) ) } else if (event_copy.event == RDMA_CM_EVENT_ROUTE_RESOLVED) {
/************************************************************************* * Wait for the rdma_cm event specified. * If another event comes in return an error. */ static int wait_for_event(struct rdma_event_channel *channel, enum rdma_cm_event_type requested_event) { struct rdma_cm_event *event; int rc = 0; int rv = -1; if ((rc = rdma_get_cm_event(channel, &event))) { debug(printf("get event failed : %d\n", rc), 1); return (rc); } debug(printf("got \"%s\" event\n", event_type_str(event->event)), 1); if (event->event == requested_event) rv = 0; rdma_ack_cm_event(event); return (rv); }
static ssize_t __fi_eq_cm_read_data(struct fid_eq *eq, void *buf, size_t len) { struct __fid_eq_cm *_eq; struct fi_eq_cm_entry *entry; struct rdma_cm_event *event; size_t left; ssize_t ret = -FI_EINVAL; _eq = container_of(eq, struct __fid_eq_cm, eq_fid); entry = (struct fi_eq_cm_entry *) buf; if (_eq->err.err) return -FI_EAVAIL; for (left = len; left >= sizeof(*entry); ) { ret = rdma_get_cm_event(_eq->channel, &event); if (!ret) { ret = __fi_eq_cm_process_event(_eq, event, entry, left); rdma_ack_cm_event(event); if (ret < 0) break; else if (!ret) continue; left -= ret; entry = ((void *) entry) + ret; } else if (errno == EAGAIN) { if (left < len) return len - left; if (!(_eq->flags & FI_BLOCK)) return 0; fi_poll_fd(_eq->channel->fd); } else { ret = -errno; break; } } return (left < len) ? len - left : ret; }
int main(int argc, char **argv) { struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *conn= NULL; struct rdma_event_channel *ec = NULL; if (argc != 4) usage(argv[0]); if (strcmp(argv[1], "write") == 0) set_mode(M_WRITE); else if (strcmp(argv[1], "read") == 0) set_mode(M_READ); else usage(argv[0]); TEST_NZ(getaddrinfo(argv[2], argv[3], NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) break; } rdma_destroy_event_channel(ec); return 0; }
/// The connection manager event handler. void poll_cm_events() { int err; struct rdma_cm_event* event; struct rdma_cm_event event_copy; void* private_data_copy = nullptr; while ((err = rdma_get_cm_event(ec_, &event)) == 0) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" VALGRIND_MAKE_MEM_DEFINED(event, sizeof(struct rdma_cm_event)); memcpy(&event_copy, event, sizeof(struct rdma_cm_event)); if (event_copy.param.conn.private_data) { VALGRIND_MAKE_MEM_DEFINED( event_copy.param.conn.private_data, event_copy.param.conn.private_data_len); private_data_copy = malloc(event_copy.param.conn.private_data_len); if (!private_data_copy) throw InfinibandException("malloc failed"); memcpy(private_data_copy, event_copy.param.conn.private_data, event_copy.param.conn.private_data_len); event_copy.param.conn.private_data = private_data_copy; } #pragma GCC diagnostic pop rdma_ack_cm_event(event); on_cm_event(&event_copy); if (private_data_copy) { free(private_data_copy); private_data_copy = nullptr; } } if (err == -1 && errno == EAGAIN) return; if (err) throw InfinibandException("rdma_get_cm_event failed"); }
void client_test(char *ip, char *port) { struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *conn= NULL; struct rdma_event_channel *ec = NULL; TEST_NZ(getaddrinfo(ip, port, NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)); TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) { s_ctx->ec = ec; s_ctx->id = conn; on_connection(event_copy.id);//send our memory information to server using post_send //printf("wait for msg_send_completion\n"); poll_cq(NULL);//wait for send_completion //printf("wait for msg_recv_completion\n"); poll_cq(NULL);//wait for recv_completion break; } } return; };
static ssize_t fi_ibv_eq_read(struct fid_eq *eq_fid, uint32_t *event, void *buf, size_t len, uint64_t flags) { struct fi_ibv_eq *eq; struct rdma_cm_event *cma_event; ssize_t ret = 0; eq = container_of(eq_fid, struct fi_ibv_eq, eq_fid.fid); if (eq->err.err) return -FI_EAVAIL; if ((ret = fi_ibv_eq_read_event(eq, event, buf, len, flags))) return ret; if (eq->channel) { ret = rdma_get_cm_event(eq->channel, &cma_event); if (ret) return -errno; if (len < sizeof(struct fi_eq_cm_entry)) return -FI_ETOOSMALL; ret = fi_ibv_eq_cm_process_event(eq, cma_event, event, (struct fi_eq_cm_entry *)buf, len); if (flags & FI_PEEK) fi_ibv_eq_write_event(eq, *event, buf, len); rdma_ack_cm_event(cma_event); return ret; } return -FI_EAGAIN; }
/** * Process CM event. * * there is a listening rdmacm id per iface * this is called as a handler from libev * * @param[in] w * @param[in] revents */ void process_cm_event(EV_P_ ev_io *w, int revents) { struct iface *iface = w->data; ni_t *ni; struct rdma_cm_event *event; conn_t *conn; struct rdma_conn_param conn_param; struct cm_priv_request priv; struct ibv_qp_init_attr init; uintptr_t ctx; if (rdma_get_cm_event(iface->cm_channel, &event)) { WARN(); return; } /* In case of connection requests conn will be NULL. */ ctx = (uintptr_t) event->id->context; if (ctx & 1) { /* Loopback. The context is not a conn but the NI. */ ctx &= ~1; conn = NULL; ni = (void *)ctx; } else { conn = (void *)ctx; ni = conn ? conn->obj.obj_ni : NULL; } ptl_info("Rank got CM event %d for id %p\n", event->event, event->id); switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: if (!conn) break; pthread_mutex_lock(&conn->mutex); if (conn->state != CONN_STATE_RESOLVING_ADDR) { /* Our connect attempt got overriden by the remote * side. */ conn_put(conn); pthread_mutex_unlock(&conn->mutex); break; } assert(conn->rdma.cm_id == event->id); conn->state = CONN_STATE_RESOLVING_ROUTE; if (rdma_resolve_route(event->id, get_param(PTL_RDMA_TIMEOUT))) { conn->state = CONN_STATE_DISCONNECTED; pthread_cond_broadcast(&conn->move_wait); conn->rdma.cm_id = NULL; conn_put(conn); } pthread_mutex_unlock(&conn->mutex); break; case RDMA_CM_EVENT_ROUTE_RESOLVED: if (!conn) break; memset(&conn_param, 0, sizeof conn_param); conn_param.responder_resources = 1; conn_param.initiator_depth = 1; conn_param.retry_count = 7; conn_param.rnr_retry_count = 7; conn_param.private_data = &priv; conn_param.private_data_len = sizeof(priv); pthread_mutex_lock(&conn->mutex); if (conn->state != CONN_STATE_RESOLVING_ROUTE) { /* Our connect attempt got overriden by the remote * side. */ conn_put(conn); pthread_mutex_unlock(&conn->mutex); break; } assert(conn->rdma.cm_id == event->id); /* Create the QP. */ memset(&init, 0, sizeof(init)); init.qp_context = ni; init.send_cq = ni->rdma.cq; init.recv_cq = ni->rdma.cq; init.cap.max_send_wr = ni->iface->cap.max_send_wr; init.cap.max_send_sge = ni->iface->cap.max_send_sge; init.qp_type = IBV_QPT_RC; init.srq = ni->rdma.srq; priv.src_id = ni->id; priv.options = ni->options; assert(conn->rdma.cm_id == event->id); if (rdma_create_qp(event->id, ni->iface->pd, &init)) { WARN(); conn->state = CONN_STATE_DISCONNECTED; pthread_cond_broadcast(&conn->move_wait); conn->rdma.cm_id = NULL; conn_put(conn); } else if (rdma_connect(event->id, &conn_param)) { WARN(); conn->state = CONN_STATE_DISCONNECTED; pthread_cond_broadcast(&conn->move_wait); rdma_destroy_qp(conn->rdma.cm_id); conn->rdma.cm_id = NULL; conn_put(conn); } else { conn->state = CONN_STATE_CONNECTING; } pthread_mutex_unlock(&conn->mutex); break; case RDMA_CM_EVENT_ESTABLISHED: if (!conn) { /* Self connection. Let the initiator side finish the * connection. */ break; } pthread_mutex_lock(&conn->mutex); atomic_inc(&ni->rdma.num_conn); if (conn->state != CONN_STATE_CONNECTING) { pthread_mutex_unlock(&conn->mutex); break; } assert(conn->rdma.cm_id == event->id); get_qp_param(conn); conn->state = CONN_STATE_CONNECTED; pthread_cond_broadcast(&conn->move_wait); pthread_mutex_unlock(&conn->mutex); break; case RDMA_CM_EVENT_CONNECT_REQUEST: process_connect_request(iface, event); break; case RDMA_CM_EVENT_REJECTED: if (!conn) break; process_connect_reject(event, conn); break; case RDMA_CM_EVENT_DISCONNECTED: if (!conn) { /* That should be the loopback connection only. */ assert(ni->rdma.self_cm_id == event->id); rdma_disconnect(ni->rdma.self_cm_id); rdma_destroy_qp(ni->rdma.self_cm_id); break; } pthread_mutex_lock(&conn->mutex); assert(conn->state != CONN_STATE_DISCONNECTED); if (conn->state != CONN_STATE_DISCONNECTING) { /* Not disconnecting yet, so we have to disconnect too. */ rdma_disconnect(conn->rdma.cm_id); rdma_destroy_qp(conn->rdma.cm_id); } conn->state = CONN_STATE_DISCONNECTED; pthread_cond_broadcast(&conn->move_wait); atomic_dec(&ni->rdma.num_conn); pthread_mutex_unlock(&conn->mutex); break; case RDMA_CM_EVENT_CONNECT_ERROR: if (!conn) break; pthread_mutex_lock(&conn->mutex); if (conn->state != CONN_STATE_DISCONNECTED) { conn->state = CONN_STATE_DISCONNECTED; pthread_cond_broadcast(&conn->move_wait); conn->rdma.cm_id->context = NULL; rdma_destroy_qp(conn->rdma.cm_id); pthread_mutex_unlock(&conn->mutex); conn_put(conn); } else { pthread_mutex_unlock(&conn->mutex); } break; case RDMA_CM_EVENT_TIMEWAIT_EXIT: break; default: ptl_warn("Got unexpected CM event: %d\n", event->event); break; } rdma_ack_cm_event(event); }
static int ibw_refill_cq_recv(struct ibw_conn *conn) { struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv); struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); int rc; struct ibv_sge list = { .addr = (uintptr_t) NULL, /* filled below */ .length = pctx->opts.recv_bufsize, .lkey = pconn->mr_recv->lkey /* always the same */ }; struct ibv_recv_wr wr = { .wr_id = 0, /* filled below */ .sg_list = &list, .num_sge = 1, }; struct ibv_recv_wr *bad_wr; DEBUG(DEBUG_DEBUG, ("ibw_refill_cq_recv(cmid: %p)\n", pconn->cm_id)); list.addr = (uintptr_t) pconn->buf_recv + pctx->opts.recv_bufsize * pconn->recv_index; wr.wr_id = pconn->recv_index; pconn->recv_index = (pconn->recv_index + 1) % pctx->opts.max_recv_wr; rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr); if (rc) { sprintf(ibw_lasterr, "refill/ibv_post_recv failed with %d\n", rc); DEBUG(DEBUG_ERR, (ibw_lasterr)); return -2; } return 0; } static int ibw_fill_cq(struct ibw_conn *conn) { struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv); struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); int i, rc; struct ibv_sge list = { .addr = (uintptr_t) NULL, /* filled below */ .length = pctx->opts.recv_bufsize, .lkey = pconn->mr_recv->lkey /* always the same */ }; struct ibv_recv_wr wr = { .wr_id = 0, /* filled below */ .sg_list = &list, .num_sge = 1, }; struct ibv_recv_wr *bad_wr; DEBUG(DEBUG_DEBUG, ("ibw_fill_cq(cmid: %p)\n", pconn->cm_id)); for(i = pctx->opts.max_recv_wr; i!=0; i--) { list.addr = (uintptr_t) pconn->buf_recv + pctx->opts.recv_bufsize * pconn->recv_index; wr.wr_id = pconn->recv_index; pconn->recv_index = (pconn->recv_index + 1) % pctx->opts.max_recv_wr; rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr); if (rc) { sprintf(ibw_lasterr, "fill/ibv_post_recv failed with %d\n", rc); DEBUG(DEBUG_ERR, (ibw_lasterr)); return -2; } } return 0; } static int ibw_manage_connect(struct ibw_conn *conn) { struct rdma_conn_param conn_param; struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); int rc; DEBUG(DEBUG_DEBUG, ("ibw_manage_connect(cmid: %p)\n", pconn->cm_id)); if (ibw_setup_cq_qp(conn)) return -1; /* cm connect */ memset(&conn_param, 0, sizeof conn_param); conn_param.responder_resources = 1; conn_param.initiator_depth = 1; conn_param.retry_count = 10; rc = rdma_connect(pconn->cm_id, &conn_param); if (rc) sprintf(ibw_lasterr, "rdma_connect error %d\n", rc); return rc; } static void ibw_event_handler_cm(struct tevent_context *ev, struct tevent_fd *fde, uint16_t flags, void *private_data) { int rc; struct ibw_ctx *ctx = talloc_get_type(private_data, struct ibw_ctx); struct ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, struct ibw_ctx_priv); struct ibw_conn *conn = NULL; struct ibw_conn_priv *pconn = NULL; struct rdma_cm_id *cma_id = NULL; struct rdma_cm_event *event = NULL; assert(ctx!=NULL); rc = rdma_get_cm_event(pctx->cm_channel, &event); if (rc) { ctx->state = IBWS_ERROR; event = NULL; sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc); goto error; } cma_id = event->id; DEBUG(DEBUG_DEBUG, ("cma_event type %d cma_id %p (%s)\n", event->event, cma_id, (cma_id == pctx->cm_id) ? "parent" : "child")); switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_ADDR_RESOLVED\n")); /* continuing from ibw_connect ... */ rc = rdma_resolve_route(cma_id, 2000); if (rc) { sprintf(ibw_lasterr, "rdma_resolve_route error %d\n", rc); goto error; } /* continued at RDMA_CM_EVENT_ROUTE_RESOLVED */ break; case RDMA_CM_EVENT_ROUTE_RESOLVED: DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_ROUTE_RESOLVED\n")); /* after RDMA_CM_EVENT_ADDR_RESOLVED: */ assert(cma_id->context!=NULL); conn = talloc_get_type(cma_id->context, struct ibw_conn); rc = ibw_manage_connect(conn); if (rc) goto error; break; case RDMA_CM_EVENT_CONNECT_REQUEST: DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_CONNECT_REQUEST\n")); ctx->state = IBWS_CONNECT_REQUEST; conn = ibw_conn_new(ctx, ctx); pconn = talloc_get_type(conn->internal, struct ibw_conn_priv); pconn->cm_id = cma_id; /* !!! event will be freed but id not */ cma_id->context = (void *)conn; DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p\n", pconn->cm_id)); if (ibw_setup_cq_qp(conn)) goto error; conn->state = IBWC_INIT; pctx->connstate_func(ctx, conn); /* continued at ibw_accept when invoked by the func above */ if (!pconn->is_accepted) { rc = rdma_reject(cma_id, NULL, 0); if (rc) DEBUG(DEBUG_ERR, ("rdma_reject failed with rc=%d\n", rc)); talloc_free(conn); DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p wasn't accepted\n", pconn->cm_id)); } /* TODO: clarify whether if it's needed by upper layer: */ ctx->state = IBWS_READY; pctx->connstate_func(ctx, NULL); /* NOTE: more requests can arrive until RDMA_CM_EVENT_ESTABLISHED ! */ break; case RDMA_CM_EVENT_ESTABLISHED: /* expected after ibw_accept and ibw_connect[not directly] */ DEBUG(DEBUG_INFO, ("ESTABLISHED (conn: %p)\n", cma_id->context)); conn = talloc_get_type(cma_id->context, struct ibw_conn); assert(conn!=NULL); /* important assumption */ DEBUG(DEBUG_DEBUG, ("ibw_setup_cq_qp succeeded (cmid=%p)\n", cma_id)); /* client conn is up */ conn->state = IBWC_CONNECTED; /* both ctx and conn have changed */ pctx->connstate_func(ctx, conn); break; case RDMA_CM_EVENT_ADDR_ERROR: sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status); case RDMA_CM_EVENT_ROUTE_ERROR: sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status); case RDMA_CM_EVENT_CONNECT_ERROR: sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status); case RDMA_CM_EVENT_UNREACHABLE: sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status); goto error; case RDMA_CM_EVENT_REJECTED: sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status); DEBUG(DEBUG_INFO, ("cm event handler: %s", ibw_lasterr)); conn = talloc_get_type(cma_id->context, struct ibw_conn); if (conn) { /* must be done BEFORE connstate */ if ((rc=rdma_ack_cm_event(event))) DEBUG(DEBUG_ERR, ("reject/rdma_ack_cm_event failed with %d\n", rc)); event = NULL; /* not to touch cma_id or conn */ conn->state = IBWC_ERROR; /* it should free the conn */ pctx->connstate_func(NULL, conn); } break; /* this is not strictly an error */ case RDMA_CM_EVENT_DISCONNECTED: DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_DISCONNECTED\n")); if ((rc=rdma_ack_cm_event(event))) DEBUG(DEBUG_ERR, ("disc/rdma_ack_cm_event failed with %d\n", rc)); event = NULL; /* don't ack more */ if (cma_id!=pctx->cm_id) { DEBUG(DEBUG_ERR, ("client DISCONNECT event cm_id=%p\n", cma_id)); conn = talloc_get_type(cma_id->context, struct ibw_conn); conn->state = IBWC_DISCONNECTED; pctx->connstate_func(NULL, conn); } break; case RDMA_CM_EVENT_DEVICE_REMOVAL: sprintf(ibw_lasterr, "cma detected device removal!\n"); goto error; default: sprintf(ibw_lasterr, "unknown event %d\n", event->event); goto error; } if (event!=NULL && (rc=rdma_ack_cm_event(event))) { sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc); goto error; } return; error: DEBUG(DEBUG_ERR, ("cm event handler: %s", ibw_lasterr)); if (event!=NULL) { if (cma_id!=NULL && cma_id!=pctx->cm_id) { conn = talloc_get_type(cma_id->context, struct ibw_conn); if (conn) { conn->state = IBWC_ERROR; pctx->connstate_func(NULL, conn); } } else { ctx->state = IBWS_ERROR; pctx->connstate_func(ctx, NULL); } if ((rc=rdma_ack_cm_event(event))!=0) { DEBUG(DEBUG_ERR, ("rdma_ack_cm_event failed with %d\n", rc)); } } return; }
int main(int argc, char *argv[]) { struct pdata rep_pdata; struct rdma_event_channel *cm_channel; struct rdma_cm_id *listen_id; struct rdma_cm_id *cm_id; struct rdma_cm_event *event; struct rdma_conn_param conn_param = { }; struct ibv_pd *pd; struct ibv_comp_channel *comp_chan; struct ibv_cq *cq; struct ibv_cq *evt_cq; struct ibv_mr *mr; struct ibv_qp_init_attr qp_attr = { }; struct ibv_sge sge; struct ibv_send_wr send_wr = { }; struct ibv_send_wr *bad_send_wr; struct ibv_recv_wr recv_wr = { }; struct ibv_recv_wr *bad_recv_wr; struct ibv_wc wc; void *cq_context; struct sockaddr_in sin; uint32_t *buf; int err; /* Set up RDMA CM structures */ cm_channel = rdma_create_event_channel(); if (!cm_channel) return 1; err = rdma_create_id(cm_channel, &listen_id, NULL, RDMA_PS_TCP); if (err) return err; sin.sin_family = AF_INET; sin.sin_port = htons(20079); sin.sin_addr.s_addr = INADDR_ANY; /* Bind to local port and listen for connection request */ err = rdma_bind_addr(listen_id, (struct sockaddr *) &sin); if (err) return 1; err = rdma_listen(listen_id, 1); if (err) return 1; err = rdma_get_cm_event(cm_channel, &event); if (err) return err; printf("after get_cm_event\n"); if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) return 1; cm_id = event->id; rdma_ack_cm_event(event); /* Create verbs objects now that we know which device to use */ pd = ibv_alloc_pd(cm_id->verbs); if (!pd) return 1; comp_chan = ibv_create_comp_channel(cm_id->verbs); if (!comp_chan) return 1; cq = ibv_create_cq(cm_id->verbs, 2, NULL, comp_chan, 0); if (!cq) return 1; if (ibv_req_notify_cq(cq, 0)) return 1; buf = calloc(2, sizeof(uint32_t)); if (!buf) return 1; mr = ibv_reg_mr(pd, buf, 2 * sizeof(uint32_t), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ | IBV_ACCESS_REMOTE_WRITE); if (!mr) return 1; qp_attr.cap.max_send_wr = 1; qp_attr.cap.max_send_sge = 1; qp_attr.cap.max_recv_wr = 1; qp_attr.cap.max_recv_sge = 1; qp_attr.send_cq = cq; qp_attr.recv_cq = cq; qp_attr.qp_type = IBV_QPT_RC; err = rdma_create_qp(cm_id, pd, &qp_attr); if (err) return err; /* Post receive before accepting connection */ sge.addr = (uintptr_t) buf + sizeof(uint32_t); sge.length = sizeof(uint32_t); sge.lkey = mr->lkey; recv_wr.sg_list = &sge; recv_wr.num_sge = 1; if (ibv_post_recv(cm_id->qp, &recv_wr, &bad_recv_wr)) return 1; rep_pdata.buf_va = htonll((uintptr_t) buf); rep_pdata.buf_rkey = htonl(mr->rkey); conn_param.responder_resources = 1; conn_param.private_data = &rep_pdata; conn_param.private_data_len = sizeof rep_pdata; /* Accept connection */ printf("before accept\n"); err = rdma_accept(cm_id, &conn_param); if (err) return 1; printf("after accept\n"); err = rdma_get_cm_event(cm_channel, &event); if (err) return err; if (event->event != RDMA_CM_EVENT_ESTABLISHED) return 1; rdma_ack_cm_event(event); /* Wait for receive completion */ if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context)) return 1; if (ibv_req_notify_cq(cq, 0)) return 1; if (ibv_poll_cq(cq, 1, &wc) < 1) return 1; if (wc.status != IBV_WC_SUCCESS) return 1; /* Add two integers and send reply back */ buf[0] = htonl(ntohl(buf[0]) + ntohl(buf[1])); sge.addr = (uintptr_t) buf; sge.length = sizeof(uint32_t); sge.lkey = mr->lkey; send_wr.opcode = IBV_WR_SEND; send_wr.send_flags = IBV_SEND_SIGNALED; send_wr.sg_list = &sge; send_wr.num_sge = 1; if (ibv_post_send(cm_id->qp, &send_wr, &bad_send_wr)) return 1; /* Wait for send completion */ if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context)) return 1; if (ibv_poll_cq(cq, 1, &wc) < 1) return 1; if (wc.status != IBV_WC_SUCCESS) return 1; printf("before ack cq 2\n"); ibv_ack_cq_events(cq, 2); return 0; }
JNIEXPORT jint JNICALL Java_com_madsys_rdma_RDMASelector_isSet(JNIEnv *env, jobject obj, jlong readfd, jlong ChannelStruct) { fd_set *fd = (fd_set *)readfd; struct rdma_core *core = (struct rdma_core *)ChannelStruct; int err; struct ibv_wc wc[2]; struct ibv_cq *evt_cq; void *cq_context; if(core->cm_channel != NULL) { if(FD_ISSET(core->cm_channel->fd,fd)) { if((err = rdma_get_cm_event(core->cm_channel,&core->event)) < 0) { printf("rdma_get_cm_event fails.\n"); return RDMA_INVALID; } switch(core->event->event) { case RDMA_CM_EVENT_CONNECT_REQUEST: return RDMA_CONNECT; case RDMA_CM_EVENT_ESTABLISHED: rdma_ack_cm_event(core->event); return RDMA_ESTABLISHED; case RDMA_CM_EVENT_DISCONNECTED: rdma_ack_cm_event(core->event); return RDMA_DISCONNECT; default: rdma_ack_cm_event(core->event); return RDMA_INVALID; } } } if(core->comp_chan != NULL) { int i,num; int type = 0; if(FD_ISSET(core->comp_chan->fd,fd)) { while(ibv_get_cq_event(core->comp_chan,&evt_cq,&cq_context)); ibv_req_notify_cq(core->cq,0); ibv_ack_cq_events(core->cq,1); if( (num = ibv_poll_cq(core->cq,2,wc)) > 0 ) { for(i=0;i<num;i++) { if(wc[0].status == IBV_WC_SUCCESS) { if(wc[0].opcode == IBV_WC_SEND) type |= RDMA_WRITE; else if(wc[0].opcode & IBV_WC_RECV) type |= RDMA_READ; } } if(type == 0) type = RDMA_INVALID; } else return RDMA_INVALID; return type; } } return RDMA_NOTHING; }
int rdma_client_connect(struct pingpong_context *ctx,struct perftest_parameters *user_param) { char *service; int temp,num_of_retry= NUM_OF_RETRIES; struct sockaddr_in sin; struct addrinfo *res; struct rdma_cm_event *event; struct rdma_conn_param conn_param; struct addrinfo hints; memset(&hints, 0, sizeof hints); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; if (check_add_port(&service,user_param->port,user_param->servername,&hints,&res)) { fprintf(stderr, "Problem in resolving basic adress and port\n"); return FAILURE; } sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr; sin.sin_family = PF_INET; sin.sin_port = htons((unsigned short)user_param->port); while (1) { if (num_of_retry == 0) { fprintf(stderr, "Received %d times ADDR_ERROR\n",NUM_OF_RETRIES); return FAILURE; } if (rdma_resolve_addr(ctx->cm_id, NULL,(struct sockaddr *)&sin,2000)) { fprintf(stderr, "rdma_resolve_addr failed\n"); return FAILURE; } if (rdma_get_cm_event(ctx->cm_channel,&event)) { fprintf(stderr, "rdma_get_cm_events failed\n"); return FAILURE; } if (event->event == RDMA_CM_EVENT_ADDR_ERROR) { num_of_retry--; rdma_ack_cm_event(event); continue; } if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) { fprintf(stderr, "unexpected CM event %d\n",event->event); rdma_ack_cm_event(event); return FAILURE; } rdma_ack_cm_event(event); break; } if (user_param->tos != DEF_TOS) { if (rdma_set_option(ctx->cm_id,RDMA_OPTION_ID,RDMA_OPTION_ID_TOS,&user_param->tos,sizeof(uint8_t))) { fprintf(stderr, " Set TOS option failed: %d\n",event->event); return FAILURE; } } while (1) { if (num_of_retry <= 0) { fprintf(stderr, "Received %d times ADDR_ERROR - aborting\n",NUM_OF_RETRIES); return FAILURE; } if (rdma_resolve_route(ctx->cm_id,2000)) { fprintf(stderr, "rdma_resolve_route failed\n"); return FAILURE; } if (rdma_get_cm_event(ctx->cm_channel,&event)) { fprintf(stderr, "rdma_get_cm_events failed\n"); return FAILURE; } if (event->event == RDMA_CM_EVENT_ROUTE_ERROR) { num_of_retry--; rdma_ack_cm_event(event); continue; } if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) { fprintf(stderr, "unexpected CM event %d\n",event->event); rdma_ack_cm_event(event); return FAILURE; } rdma_ack_cm_event(event); break; } ctx->context = ctx->cm_id->verbs; temp = user_param->work_rdma_cm; user_param->work_rdma_cm = ON; if (ctx_init(ctx,user_param)) { fprintf(stderr," Unable to create the resources needed by comm struct\n"); return FAILURE; } memset(&conn_param, 0, sizeof conn_param); if (user_param->verb == READ || user_param->verb == ATOMIC) { conn_param.responder_resources = user_param->out_reads; conn_param.initiator_depth = user_param->out_reads; } user_param->work_rdma_cm = temp; conn_param.retry_count = user_param->retry_count; conn_param.rnr_retry_count = 7; if (user_param->work_rdma_cm == OFF) { if (post_one_recv_wqe(ctx)) { fprintf(stderr, "Couldn't post send \n"); return 1; } } if (rdma_connect(ctx->cm_id,&conn_param)) { fprintf(stderr, "Function rdma_connect failed\n"); return FAILURE; } if (rdma_get_cm_event(ctx->cm_channel,&event)) { fprintf(stderr, "rdma_get_cm_events failed\n"); return FAILURE; } if (event->event != RDMA_CM_EVENT_ESTABLISHED) { rdma_ack_cm_event(event); fprintf(stderr, "Unexpected CM event bl blka %d\n", event->event); return FAILURE; } if (user_param->connection_type == UD) { user_param->rem_ud_qpn = event->param.ud.qp_num; user_param->rem_ud_qkey = event->param.ud.qkey; ctx->ah[0] = ibv_create_ah(ctx->pd,&event->param.ud.ah_attr); if (!ctx->ah) { printf(" Unable to create address handler for UD QP\n"); return FAILURE; } if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) { if (send_qp_num_for_ah(ctx,user_param)) { printf(" Unable to send my QP number\n"); return FAILURE; } } } rdma_ack_cm_event(event); return SUCCESS; }
/****************************************************************************** + * + ******************************************************************************/ int rdma_server_connect(struct pingpong_context *ctx, struct perftest_parameters *user_param) { int temp; struct addrinfo *res; struct rdma_cm_event *event; struct rdma_conn_param conn_param; struct addrinfo hints; char *service; struct sockaddr_in sin; memset(&hints, 0, sizeof hints); hints.ai_flags = AI_PASSIVE; hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; if (check_add_port(&service,user_param->port,user_param->servername,&hints,&res)) { fprintf(stderr, "Problem in resolving basic adress and port\n"); return FAILURE; } sin.sin_addr.s_addr = 0; sin.sin_family = PF_INET; sin.sin_port = htons((unsigned short)user_param->port); if (rdma_bind_addr(ctx->cm_id_control,(struct sockaddr *)&sin)) { fprintf(stderr," rdma_bind_addr failed\n"); return 1; } if (rdma_listen(ctx->cm_id_control,0)) { fprintf(stderr, "rdma_listen failed\n"); return 1; } if (rdma_get_cm_event(ctx->cm_channel,&event)) { fprintf(stderr, "rdma_get_cm_events failed\n"); return 1; } if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) { fprintf(stderr, "bad event waiting for connect request %d\n",event->event); return 1; } ctx->cm_id = (struct rdma_cm_id*)event->id; ctx->context = ctx->cm_id->verbs; if (user_param->work_rdma_cm == ON) alloc_ctx(ctx,user_param); temp = user_param->work_rdma_cm; user_param->work_rdma_cm = ON; if (ctx_init(ctx,user_param)) { fprintf(stderr," Unable to create the resources needed by comm struct\n"); return FAILURE; } memset(&conn_param, 0, sizeof conn_param); if (user_param->verb == READ || user_param->verb == ATOMIC) { conn_param.responder_resources = user_param->out_reads; conn_param.initiator_depth = user_param->out_reads; } if (user_param->connection_type == UD) conn_param.qp_num = ctx->qp[0]->qp_num; conn_param.retry_count = user_param->retry_count; conn_param.rnr_retry_count = 7; user_param->work_rdma_cm = temp; if (user_param->work_rdma_cm == OFF) { if (post_one_recv_wqe(ctx)) { fprintf(stderr, "Couldn't post send \n"); return 1; } } else if (user_param->connection_type == UD) { if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) { if (post_recv_to_get_ah(ctx)) { fprintf(stderr, "Couldn't post send \n"); return 1; } } } if (rdma_accept(ctx->cm_id, &conn_param)) { fprintf(stderr, "Function rdma_accept failed\n"); return 1; } if (user_param->work_rdma_cm && user_param->connection_type == UD) { if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) { if (create_ah_from_wc_recv(ctx,user_param)) { fprintf(stderr, "Unable to create AH from WC\n"); return 1; } } } rdma_ack_cm_event(event); rdma_destroy_id(ctx->cm_id_control); freeaddrinfo(res); return 0; }
void client_test(char *ip, char *port) { struct addrinfo *addr; struct rdma_cm_event *event = NULL; struct rdma_cm_id *conn= NULL; struct rdma_event_channel *ec = NULL; struct timeval t1, t2, t3, t11, t12; struct timeval dt, dt1, dt2, dt11, dt12, dt13; gettimeofday(&t1, NULL); TEST_NZ(getaddrinfo(ip, port, NULL, &addr)); TEST_Z(ec = rdma_create_event_channel()); gettimeofday(&t11, NULL); TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP)); gettimeofday(&t12, NULL); TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS)); freeaddrinfo(addr); gettimeofday(&t2, NULL); while (rdma_get_cm_event(ec, &event) == 0) { struct rdma_cm_event event_copy; memcpy(&event_copy, event, sizeof(*event)); rdma_ack_cm_event(event); if (on_event(&event_copy)) { s_ctx->ec = ec; s_ctx->id = conn; on_connection(event_copy.id);//send our memory information to server using post_send poll_cq(NULL);//wait for send_completion poll_cq(NULL);//wait for recv_completion break; } } gettimeofday(&t3, NULL); timersub(&t3, &t1, &dt); timersub(&t3, &t2, &dt2); timersub(&t2, &t1, &dt1); timersub(&t2, &t12, &dt13); timersub(&t12, &t11, &dt12); timersub(&t11, &t1, &dt11); long usec = dt.tv_usec + 10000 * dt.tv_sec; printf("[dt]:\t%ld us.\n", usec); printf("[dt1]:\t%ld us.\n", dt1.tv_usec+1000000 *dt1.tv_sec); printf("Including the following steps: \n"); printf("[dt11]:\t%ld us.\n", dt11.tv_usec+1000000 *dt11.tv_sec); printf("[dt12]:\t%ld us.\n", dt12.tv_usec+1000000 *dt12.tv_sec); printf("[dt13]:\t%ld us.\n", dt13.tv_usec+1000000 *dt13.tv_sec); printf("[dt2] takes %ld micro_secs.\n", dt2.tv_usec+1000000*dt2.tv_sec); printf("[dt]:total time\t[dt1]:pre_setup\t[dt2]:send/recv\t.\n"); printf("[dt11]:create_event_channel\t[dt12]:create_id\t[dt13]:resolve_address.\n"); return; };