int main(int argc, char **argv)
{
struct sockaddr_in addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *listener = NULL;
struct rdma_event_channel *ec = NULL;
uint16_t port = 0;
memset(&addr, 0, sizeof(addr));
addr.sin_family = AF_INET;
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_bind_addr(listener, (struct sockaddr *)&addr));
TEST_NZ(rdma_listen(listener, 10)); /* backlog=10 is arbitrary */
port = ntohs(rdma_get_src_port(listener));
printf("listening on port %d.\n", port);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_id(listener);
rdma_destroy_event_channel(ec);
return 0;
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
if (argc != 3)
die("usage: client <server-address> <server-port>");
TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_event_channel(ec);
return 0;
}
void client_disconnect(void){
struct rdma_event_channel *ec = s_ctx->ec;
struct rdma_cm_id *id = s_ctx->id;
struct rdma_cm_event *event = NULL;
printf("ready to disconnect\n");
rdma_disconnect(id);
printf("send disconnect\n");
while(1){
if(rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){
on_disconnect(event_copy.id);
rdma_destroy_event_channel(ec);
break;
}
}
}
return;
}
static void *cma_thread(void *arg)
{
struct rdma_cm_event *event;
int ret;
while (1) {
ret = rdma_get_cm_event(test.channel, &event);
if (ret) {
perror("rdma_get_cm_event");
break;
}
switch (event->event) {
case RDMA_CM_EVENT_MULTICAST_ERROR:
case RDMA_CM_EVENT_ADDR_CHANGE:
printf("mckey: event: %s, status: %d\n",
rdma_event_str(event->event), event->status);
break;
default:
break;
}
rdma_ack_cm_event(event);
}
return NULL;
}
static int get_next_channel_event(struct thread_data *td,
struct rdma_event_channel *channel,
enum rdma_cm_event_type wait_event)
{
struct rdmaio_data *rd = td->io_ops->data;
struct rdma_cm_event *event;
int ret;
ret = rdma_get_cm_event(channel, &event);
if (ret) {
log_err("fio: rdma_get_cm_event: %d\n", ret);
return 1;
}
if (event->event != wait_event) {
log_err("fio: event is %s instead of %s\n",
rdma_event_str(event->event),
rdma_event_str(wait_event));
return 1;
}
switch (event->event) {
case RDMA_CM_EVENT_CONNECT_REQUEST:
rd->child_cm_id = event->id;
break;
default:
break;
}
rdma_ack_cm_event(event);
return 0;
}
void
wait_event(struct rdma_event_channel *ec, int et, int (*handler)(struct rdma_cm_id *id))
{
struct rdma_cm_event *ep = NULL;
while (rdma_get_cm_event(ec, &ep) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, ep, sizeof(*ep));
rdma_ack_cm_event(ep);
if (et == event_copy.event) {
if (handler) {
handler(event_copy.id);
}
break;
}
else {
fprintf(stderr, "got an event (=%d) not what expected (=%d). "
"event.status:%d\n",
event_copy.event, et, event_copy.status);
//
// infiniband/cm.h ib_cm_rej_reason
// IB_CM_REJ_INVALID_SERVICE_ID = 8,
exit(EXIT_FAILURE);
}
}
}
void
rdmaWaitEvent(struct rdma_event_channel *ec, int et, int (*handler)(struct rdma_cm_id *id))
{
struct rdma_cm_event *ep = NULL;
while (rdma_get_cm_event(ec, &ep) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, ep, sizeof(*ep));
rdma_ack_cm_event(ep);
if (et == event_copy.event) {
if (handler) {
handler(event_copy.id);
}
break;
}
else {
fprintf(stderr, "got an event (=%d) not what expected (=%d). "
"event.status:%d\n",
event_copy.event, et, event_copy.status);
switch (event_copy.event) {
case RDMA_CM_EVENT_REJECTED:
fprintf(stderr, "RDMA connection rejected. The peer may be down.\n");
break;
default:
break;
}
//
// infiniband/cm.h ib_cm_rej_reason
// IB_CM_REJ_INVALID_SERVICE_ID = 8,
exit(EXIT_FAILURE);
}
}
}
int xfer_rdma_finalize(struct xfer_data *data)
{
struct rdma_cm_event *event;
int rc;
if (data->servername) {
rc = rdma_disconnect(data->cm_id);
if (rc) {
perror("rdma_disconnect");
fprintf(stderr, "%d:%s: rdma disconnect error\n", pid,
__func__);
return -1;
}
}
rdma_get_cm_event(data->cm_channel, &event);
if (event->event != RDMA_CM_EVENT_DISCONNECTED)
fprintf(stderr, "%d:%s: unexpected event during disconnect %d\n",
pid, __func__, event->event);
rdma_ack_cm_event(event);
rdma_destroy_id(data->cm_id);
rdma_destroy_event_channel(data->cm_channel);
return 0;
}
void client_disconnect(void){
struct rdma_event_channel *ec = s_ctx->ec;
struct rdma_cm_id *id = s_ctx->id;
struct rdma_cm_event *event = NULL;
struct timeval start, end, dt;
gettimeofday(&start, NULL);
printf("ready to disconnect\n");
rdma_disconnect(id);
printf("send disconnect\n");
gettimeofday(&end, NULL);
timersub(&end, &start, &dt);
long usec = dt.tv_usec + 1000000 * dt.tv_sec;
printf("[rdma_disconnect] takes %ld micro_secs.\n", usec);
while(1){
if(rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (event_copy.event == RDMA_CM_EVENT_DISCONNECTED){
on_disconnect(event_copy.id);
rdma_destroy_event_channel(ec);
break;
}
}
}
return;
}
void RDMAServerSocket::cm_events() const {
eventThread.send([=]() {
hydra::util::epoll poll;
epoll_event poll_event;
poll_event.events = EPOLLIN;
poll_event.data.fd = ec->fd;
poll.add(ec->fd, &poll_event);
while (running) {
rdma_cm_event *cm_event = nullptr;
int ret = poll.wait(&poll_event, 1, 2000);
if (ret) {
if (poll_event.data.fd == ec->fd) {
check_zero(rdma_get_cm_event(ec.get(), &cm_event));
check_zero(cm_event->status);
if (cm_event->event == RDMA_CM_EVENT_CONNECT_REQUEST) {
accept(client_t(cm_event->id));
} else if (cm_event->event == RDMA_CM_EVENT_DISCONNECTED) {
rdma_disconnect(cm_event->id);
}
check_zero(rdma_ack_cm_event(cm_event));
} else {
log_err() << "Unkown fd " << poll_event.data.fd << " set. Expected "
<< id->channel->fd;
std::terminate();
}
}
}
});
}
void wait_rdma_event()
{
struct rdma_cm_event *event = NULL;
while (!rdma_get_cm_event(ec, &event)) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (client_on_event(&event_copy))
break;
}
}
static int connect_events(void)
{
struct rdma_cm_event *event;
int ret = 0;
while (test.connects_left && !ret) {
ret = rdma_get_cm_event(test.channel, &event);
if (!ret) {
ret = cma_handler(event->id, event);
rdma_ack_cm_event(event);
}
}
return ret;
}
void
diod_rdma_accept_one (Npsrv *srv, diod_rdma_t rdma)
{
Npconn *conn;
Nptrans *trans;
struct rdma_cm_event *event;
struct rdma_cm_id *cmid;
enum rdma_cm_event_type etype;
int n;
n = rdma_get_cm_event(rdma->event_channel, &event);
if (n)
errn_exit (n, "rdma_get_cm_event");
cmid = (struct rdma_cm_id *)event->id;
etype = event->event;
rdma_ack_cm_event(event);
switch (etype) {
case RDMA_CM_EVENT_CONNECT_REQUEST:
msg ("rdma: connection request");
trans = np_rdmatrans_create(cmid, rdma_qdepth, rdma_maxmsize);
if (trans) {
conn = np_conn_create(srv, trans, "rdma", 0);
cmid->context = conn;
np_srv_add_conn(srv, conn);
} else
errn (np_rerror (), "np_rdmatrns_create failed");
break;
case RDMA_CM_EVENT_ESTABLISHED:
msg ("rdma: connection established");
break;
case RDMA_CM_EVENT_DISCONNECTED:
msg ("rdma: connection shutting down");
conn = cmid->context;
//np_conn_shutdown(conn);
/* FIXME: clean up properly */
break;
default:
msg ("rdma: event %d received waiting for a connect request\n",
etype);
}
}
inline bool get_next_event(
rdma_event_channel *event_channel, rdma_cm_event & event_copy, Connection * c
, boost::system::error_code &ec
)
{
if(!event_channel)
{
HPX_IBVERBS_THROWS_IF(ec, boost::asio::error::not_connected);
return false;
}
rdma_cm_event * event = NULL;
if(rdma_get_cm_event(event_channel, &event) == 0)
{
std::memcpy(&event_copy, event, sizeof(rdma_cm_event));
rdma_ack_cm_event(event);
if(event_copy.event == RDMA_CM_EVENT_DISCONNECTED)
{
c->on_disconnect(event_copy.id);
return get_next_event(event_channel, event_copy, c, ec);
}
return true;
}
else
{
int verrno = errno;
if(verrno == EBADF) return false;
if(verrno == EAGAIN) return false;
if(verrno == EWOULDBLOCK) return false;
boost::system::error_code err(verrno, boost::system::system_category());
HPX_IBVERBS_THROWS_IF(
ec
, err
);
return false;
}
HPX_ASSERT(false);
return false;
}
static void *cm_thread(void *arg)
{
struct rping_cb *cb = arg;
struct rdma_cm_event *event;
int ret;
while (1) {
ret = rdma_get_cm_event(cb->cm_channel, &event);
if (ret) {
perror("rdma_get_cm_event");
exit(ret);
}
ret = rping_cma_event_handler(event->id, event);
rdma_ack_cm_event(event);
if (ret)
exit(ret);
}
}
static int disconnect_events(void)
{
struct rdma_cm_event *event;
int ret = 0;
while (test.disconnects_left && !ret) {
ret = rdma_get_cm_event(test.channel, &event);
if (!ret) {
ret = cma_handler(event->id, event);
rdma_ack_cm_event(event);
} else {
perror("cmatose: failure in rdma_get_cm_event in disconnect events");
ret = errno;
}
}
return ret;
}
void Connector::event_loop(struct rdma_event_channel* ec_, bool exit_on_disconn)
{
struct rdma_cm_event* event_ = NULL;
struct rdma_conn_param cm_param;
build_param(&cm_param);
while (rdma_get_cm_event(ec_, &event_) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event_, sizeof(*event_) );
rdma_ack_cm_event(event_);
if (event_copy.event == RDMA_CM_EVENT_ADDR_RESOLVED) {
build_conn(event_copy.id);
ib_end_->on_pre_conn(event_copy.id);
TEST_NZ(rdma_resolve_route(event_copy.id, TIMEOUT_IN_MS) )
}
else if (event_copy.event == RDMA_CM_EVENT_ROUTE_RESOLVED) {
/*************************************************************************
* Wait for the rdma_cm event specified.
* If another event comes in return an error.
*/
static int
wait_for_event(struct rdma_event_channel *channel, enum rdma_cm_event_type requested_event)
{
struct rdma_cm_event *event;
int rc = 0;
int rv = -1;
if ((rc = rdma_get_cm_event(channel, &event)))
{
debug(printf("get event failed : %d\n", rc), 1);
return (rc);
}
debug(printf("got \"%s\" event\n", event_type_str(event->event)), 1);
if (event->event == requested_event)
rv = 0;
rdma_ack_cm_event(event);
return (rv);
}
static ssize_t __fi_eq_cm_read_data(struct fid_eq *eq, void *buf, size_t len)
{
struct __fid_eq_cm *_eq;
struct fi_eq_cm_entry *entry;
struct rdma_cm_event *event;
size_t left;
ssize_t ret = -FI_EINVAL;
_eq = container_of(eq, struct __fid_eq_cm, eq_fid);
entry = (struct fi_eq_cm_entry *) buf;
if (_eq->err.err)
return -FI_EAVAIL;
for (left = len; left >= sizeof(*entry); ) {
ret = rdma_get_cm_event(_eq->channel, &event);
if (!ret) {
ret = __fi_eq_cm_process_event(_eq, event, entry, left);
rdma_ack_cm_event(event);
if (ret < 0)
break;
else if (!ret)
continue;
left -= ret;
entry = ((void *) entry) + ret;
} else if (errno == EAGAIN) {
if (left < len)
return len - left;
if (!(_eq->flags & FI_BLOCK))
return 0;
fi_poll_fd(_eq->channel->fd);
} else {
ret = -errno;
break;
}
}
return (left < len) ? len - left : ret;
}
int main(int argc, char **argv)
{
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
if (argc != 4)
usage(argv[0]);
if (strcmp(argv[1], "write") == 0)
set_mode(M_WRITE);
else if (strcmp(argv[1], "read") == 0)
set_mode(M_READ);
else
usage(argv[0]);
TEST_NZ(getaddrinfo(argv[2], argv[3], NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy))
break;
}
rdma_destroy_event_channel(ec);
return 0;
}
/// The connection manager event handler.
void poll_cm_events()
{
int err;
struct rdma_cm_event* event;
struct rdma_cm_event event_copy;
void* private_data_copy = nullptr;
while ((err = rdma_get_cm_event(ec_, &event)) == 0) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
VALGRIND_MAKE_MEM_DEFINED(event, sizeof(struct rdma_cm_event));
memcpy(&event_copy, event, sizeof(struct rdma_cm_event));
if (event_copy.param.conn.private_data) {
VALGRIND_MAKE_MEM_DEFINED(
event_copy.param.conn.private_data,
event_copy.param.conn.private_data_len);
private_data_copy =
malloc(event_copy.param.conn.private_data_len);
if (!private_data_copy)
throw InfinibandException("malloc failed");
memcpy(private_data_copy, event_copy.param.conn.private_data,
event_copy.param.conn.private_data_len);
event_copy.param.conn.private_data = private_data_copy;
}
#pragma GCC diagnostic pop
rdma_ack_cm_event(event);
on_cm_event(&event_copy);
if (private_data_copy) {
free(private_data_copy);
private_data_copy = nullptr;
}
}
if (err == -1 && errno == EAGAIN)
return;
if (err)
throw InfinibandException("rdma_get_cm_event failed");
}
void client_test(char *ip, char *port) {
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
TEST_NZ(getaddrinfo(ip, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy)) {
s_ctx->ec = ec;
s_ctx->id = conn;
on_connection(event_copy.id);//send our memory information to server using post_send
//printf("wait for msg_send_completion\n");
poll_cq(NULL);//wait for send_completion
//printf("wait for msg_recv_completion\n");
poll_cq(NULL);//wait for recv_completion
break;
}
}
return;
};
static ssize_t
fi_ibv_eq_read(struct fid_eq *eq_fid, uint32_t *event,
void *buf, size_t len, uint64_t flags)
{
struct fi_ibv_eq *eq;
struct rdma_cm_event *cma_event;
ssize_t ret = 0;
eq = container_of(eq_fid, struct fi_ibv_eq, eq_fid.fid);
if (eq->err.err)
return -FI_EAVAIL;
if ((ret = fi_ibv_eq_read_event(eq, event, buf, len, flags)))
return ret;
if (eq->channel) {
ret = rdma_get_cm_event(eq->channel, &cma_event);
if (ret)
return -errno;
if (len < sizeof(struct fi_eq_cm_entry))
return -FI_ETOOSMALL;
ret = fi_ibv_eq_cm_process_event(eq, cma_event, event,
(struct fi_eq_cm_entry *)buf, len);
if (flags & FI_PEEK)
fi_ibv_eq_write_event(eq, *event, buf, len);
rdma_ack_cm_event(cma_event);
return ret;
}
return -FI_EAGAIN;
}
/**
* Process CM event.
*
* there is a listening rdmacm id per iface
* this is called as a handler from libev
*
* @param[in] w
* @param[in] revents
*/
void process_cm_event(EV_P_ ev_io *w, int revents)
{
struct iface *iface = w->data;
ni_t *ni;
struct rdma_cm_event *event;
conn_t *conn;
struct rdma_conn_param conn_param;
struct cm_priv_request priv;
struct ibv_qp_init_attr init;
uintptr_t ctx;
if (rdma_get_cm_event(iface->cm_channel, &event)) {
WARN();
return;
}
/* In case of connection requests conn will be NULL. */
ctx = (uintptr_t) event->id->context;
if (ctx & 1) {
/* Loopback. The context is not a conn but the NI. */
ctx &= ~1;
conn = NULL;
ni = (void *)ctx;
} else {
conn = (void *)ctx;
ni = conn ? conn->obj.obj_ni : NULL;
}
ptl_info("Rank got CM event %d for id %p\n", event->event, event->id);
switch (event->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
if (!conn)
break;
pthread_mutex_lock(&conn->mutex);
if (conn->state != CONN_STATE_RESOLVING_ADDR) {
/* Our connect attempt got overriden by the remote
* side. */
conn_put(conn);
pthread_mutex_unlock(&conn->mutex);
break;
}
assert(conn->rdma.cm_id == event->id);
conn->state = CONN_STATE_RESOLVING_ROUTE;
if (rdma_resolve_route(event->id, get_param(PTL_RDMA_TIMEOUT))) {
conn->state = CONN_STATE_DISCONNECTED;
pthread_cond_broadcast(&conn->move_wait);
conn->rdma.cm_id = NULL;
conn_put(conn);
}
pthread_mutex_unlock(&conn->mutex);
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
if (!conn)
break;
memset(&conn_param, 0, sizeof conn_param);
conn_param.responder_resources = 1;
conn_param.initiator_depth = 1;
conn_param.retry_count = 7;
conn_param.rnr_retry_count = 7;
conn_param.private_data = &priv;
conn_param.private_data_len = sizeof(priv);
pthread_mutex_lock(&conn->mutex);
if (conn->state != CONN_STATE_RESOLVING_ROUTE) {
/* Our connect attempt got overriden by the remote
* side. */
conn_put(conn);
pthread_mutex_unlock(&conn->mutex);
break;
}
assert(conn->rdma.cm_id == event->id);
/* Create the QP. */
memset(&init, 0, sizeof(init));
init.qp_context = ni;
init.send_cq = ni->rdma.cq;
init.recv_cq = ni->rdma.cq;
init.cap.max_send_wr = ni->iface->cap.max_send_wr;
init.cap.max_send_sge = ni->iface->cap.max_send_sge;
init.qp_type = IBV_QPT_RC;
init.srq = ni->rdma.srq;
priv.src_id = ni->id;
priv.options = ni->options;
assert(conn->rdma.cm_id == event->id);
if (rdma_create_qp(event->id, ni->iface->pd, &init)) {
WARN();
conn->state = CONN_STATE_DISCONNECTED;
pthread_cond_broadcast(&conn->move_wait);
conn->rdma.cm_id = NULL;
conn_put(conn);
} else if (rdma_connect(event->id, &conn_param)) {
WARN();
conn->state = CONN_STATE_DISCONNECTED;
pthread_cond_broadcast(&conn->move_wait);
rdma_destroy_qp(conn->rdma.cm_id);
conn->rdma.cm_id = NULL;
conn_put(conn);
} else {
conn->state = CONN_STATE_CONNECTING;
}
pthread_mutex_unlock(&conn->mutex);
break;
case RDMA_CM_EVENT_ESTABLISHED:
if (!conn) {
/* Self connection. Let the initiator side finish the
* connection. */
break;
}
pthread_mutex_lock(&conn->mutex);
atomic_inc(&ni->rdma.num_conn);
if (conn->state != CONN_STATE_CONNECTING) {
pthread_mutex_unlock(&conn->mutex);
break;
}
assert(conn->rdma.cm_id == event->id);
get_qp_param(conn);
conn->state = CONN_STATE_CONNECTED;
pthread_cond_broadcast(&conn->move_wait);
pthread_mutex_unlock(&conn->mutex);
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
process_connect_request(iface, event);
break;
case RDMA_CM_EVENT_REJECTED:
if (!conn)
break;
process_connect_reject(event, conn);
break;
case RDMA_CM_EVENT_DISCONNECTED:
if (!conn) {
/* That should be the loopback connection only. */
assert(ni->rdma.self_cm_id == event->id);
rdma_disconnect(ni->rdma.self_cm_id);
rdma_destroy_qp(ni->rdma.self_cm_id);
break;
}
pthread_mutex_lock(&conn->mutex);
assert(conn->state != CONN_STATE_DISCONNECTED);
if (conn->state != CONN_STATE_DISCONNECTING) {
/* Not disconnecting yet, so we have to disconnect too. */
rdma_disconnect(conn->rdma.cm_id);
rdma_destroy_qp(conn->rdma.cm_id);
}
conn->state = CONN_STATE_DISCONNECTED;
pthread_cond_broadcast(&conn->move_wait);
atomic_dec(&ni->rdma.num_conn);
pthread_mutex_unlock(&conn->mutex);
break;
case RDMA_CM_EVENT_CONNECT_ERROR:
if (!conn)
break;
pthread_mutex_lock(&conn->mutex);
if (conn->state != CONN_STATE_DISCONNECTED) {
conn->state = CONN_STATE_DISCONNECTED;
pthread_cond_broadcast(&conn->move_wait);
conn->rdma.cm_id->context = NULL;
rdma_destroy_qp(conn->rdma.cm_id);
pthread_mutex_unlock(&conn->mutex);
conn_put(conn);
} else {
pthread_mutex_unlock(&conn->mutex);
}
break;
case RDMA_CM_EVENT_TIMEWAIT_EXIT:
break;
default:
ptl_warn("Got unexpected CM event: %d\n", event->event);
break;
}
rdma_ack_cm_event(event);
}
static int ibw_refill_cq_recv(struct ibw_conn *conn)
{
struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
int rc;
struct ibv_sge list = {
.addr = (uintptr_t) NULL, /* filled below */
.length = pctx->opts.recv_bufsize,
.lkey = pconn->mr_recv->lkey /* always the same */
};
struct ibv_recv_wr wr = {
.wr_id = 0, /* filled below */
.sg_list = &list,
.num_sge = 1,
};
struct ibv_recv_wr *bad_wr;
DEBUG(DEBUG_DEBUG, ("ibw_refill_cq_recv(cmid: %p)\n", pconn->cm_id));
list.addr = (uintptr_t) pconn->buf_recv + pctx->opts.recv_bufsize * pconn->recv_index;
wr.wr_id = pconn->recv_index;
pconn->recv_index = (pconn->recv_index + 1) % pctx->opts.max_recv_wr;
rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr);
if (rc) {
sprintf(ibw_lasterr, "refill/ibv_post_recv failed with %d\n", rc);
DEBUG(DEBUG_ERR, (ibw_lasterr));
return -2;
}
return 0;
}
static int ibw_fill_cq(struct ibw_conn *conn)
{
struct ibw_ctx_priv *pctx = talloc_get_type(conn->ctx->internal, struct ibw_ctx_priv);
struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
int i, rc;
struct ibv_sge list = {
.addr = (uintptr_t) NULL, /* filled below */
.length = pctx->opts.recv_bufsize,
.lkey = pconn->mr_recv->lkey /* always the same */
};
struct ibv_recv_wr wr = {
.wr_id = 0, /* filled below */
.sg_list = &list,
.num_sge = 1,
};
struct ibv_recv_wr *bad_wr;
DEBUG(DEBUG_DEBUG, ("ibw_fill_cq(cmid: %p)\n", pconn->cm_id));
for(i = pctx->opts.max_recv_wr; i!=0; i--) {
list.addr = (uintptr_t) pconn->buf_recv + pctx->opts.recv_bufsize * pconn->recv_index;
wr.wr_id = pconn->recv_index;
pconn->recv_index = (pconn->recv_index + 1) % pctx->opts.max_recv_wr;
rc = ibv_post_recv(pconn->cm_id->qp, &wr, &bad_wr);
if (rc) {
sprintf(ibw_lasterr, "fill/ibv_post_recv failed with %d\n", rc);
DEBUG(DEBUG_ERR, (ibw_lasterr));
return -2;
}
}
return 0;
}
static int ibw_manage_connect(struct ibw_conn *conn)
{
struct rdma_conn_param conn_param;
struct ibw_conn_priv *pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
int rc;
DEBUG(DEBUG_DEBUG, ("ibw_manage_connect(cmid: %p)\n", pconn->cm_id));
if (ibw_setup_cq_qp(conn))
return -1;
/* cm connect */
memset(&conn_param, 0, sizeof conn_param);
conn_param.responder_resources = 1;
conn_param.initiator_depth = 1;
conn_param.retry_count = 10;
rc = rdma_connect(pconn->cm_id, &conn_param);
if (rc)
sprintf(ibw_lasterr, "rdma_connect error %d\n", rc);
return rc;
}
static void ibw_event_handler_cm(struct tevent_context *ev,
struct tevent_fd *fde, uint16_t flags, void *private_data)
{
int rc;
struct ibw_ctx *ctx = talloc_get_type(private_data, struct ibw_ctx);
struct ibw_ctx_priv *pctx = talloc_get_type(ctx->internal, struct ibw_ctx_priv);
struct ibw_conn *conn = NULL;
struct ibw_conn_priv *pconn = NULL;
struct rdma_cm_id *cma_id = NULL;
struct rdma_cm_event *event = NULL;
assert(ctx!=NULL);
rc = rdma_get_cm_event(pctx->cm_channel, &event);
if (rc) {
ctx->state = IBWS_ERROR;
event = NULL;
sprintf(ibw_lasterr, "rdma_get_cm_event error %d\n", rc);
goto error;
}
cma_id = event->id;
DEBUG(DEBUG_DEBUG, ("cma_event type %d cma_id %p (%s)\n", event->event, cma_id,
(cma_id == pctx->cm_id) ? "parent" : "child"));
switch (event->event) {
case RDMA_CM_EVENT_ADDR_RESOLVED:
DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_ADDR_RESOLVED\n"));
/* continuing from ibw_connect ... */
rc = rdma_resolve_route(cma_id, 2000);
if (rc) {
sprintf(ibw_lasterr, "rdma_resolve_route error %d\n", rc);
goto error;
}
/* continued at RDMA_CM_EVENT_ROUTE_RESOLVED */
break;
case RDMA_CM_EVENT_ROUTE_RESOLVED:
DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_ROUTE_RESOLVED\n"));
/* after RDMA_CM_EVENT_ADDR_RESOLVED: */
assert(cma_id->context!=NULL);
conn = talloc_get_type(cma_id->context, struct ibw_conn);
rc = ibw_manage_connect(conn);
if (rc)
goto error;
break;
case RDMA_CM_EVENT_CONNECT_REQUEST:
DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_CONNECT_REQUEST\n"));
ctx->state = IBWS_CONNECT_REQUEST;
conn = ibw_conn_new(ctx, ctx);
pconn = talloc_get_type(conn->internal, struct ibw_conn_priv);
pconn->cm_id = cma_id; /* !!! event will be freed but id not */
cma_id->context = (void *)conn;
DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p\n", pconn->cm_id));
if (ibw_setup_cq_qp(conn))
goto error;
conn->state = IBWC_INIT;
pctx->connstate_func(ctx, conn);
/* continued at ibw_accept when invoked by the func above */
if (!pconn->is_accepted) {
rc = rdma_reject(cma_id, NULL, 0);
if (rc)
DEBUG(DEBUG_ERR, ("rdma_reject failed with rc=%d\n", rc));
talloc_free(conn);
DEBUG(DEBUG_DEBUG, ("pconn->cm_id %p wasn't accepted\n", pconn->cm_id));
}
/* TODO: clarify whether if it's needed by upper layer: */
ctx->state = IBWS_READY;
pctx->connstate_func(ctx, NULL);
/* NOTE: more requests can arrive until RDMA_CM_EVENT_ESTABLISHED ! */
break;
case RDMA_CM_EVENT_ESTABLISHED:
/* expected after ibw_accept and ibw_connect[not directly] */
DEBUG(DEBUG_INFO, ("ESTABLISHED (conn: %p)\n", cma_id->context));
conn = talloc_get_type(cma_id->context, struct ibw_conn);
assert(conn!=NULL); /* important assumption */
DEBUG(DEBUG_DEBUG, ("ibw_setup_cq_qp succeeded (cmid=%p)\n", cma_id));
/* client conn is up */
conn->state = IBWC_CONNECTED;
/* both ctx and conn have changed */
pctx->connstate_func(ctx, conn);
break;
case RDMA_CM_EVENT_ADDR_ERROR:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_ADDR_ERROR, error %d\n", event->status);
case RDMA_CM_EVENT_ROUTE_ERROR:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_ROUTE_ERROR, error %d\n", event->status);
case RDMA_CM_EVENT_CONNECT_ERROR:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_CONNECT_ERROR, error %d\n", event->status);
case RDMA_CM_EVENT_UNREACHABLE:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_UNREACHABLE, error %d\n", event->status);
goto error;
case RDMA_CM_EVENT_REJECTED:
sprintf(ibw_lasterr, "RDMA_CM_EVENT_REJECTED, error %d\n", event->status);
DEBUG(DEBUG_INFO, ("cm event handler: %s", ibw_lasterr));
conn = talloc_get_type(cma_id->context, struct ibw_conn);
if (conn) {
/* must be done BEFORE connstate */
if ((rc=rdma_ack_cm_event(event)))
DEBUG(DEBUG_ERR, ("reject/rdma_ack_cm_event failed with %d\n", rc));
event = NULL; /* not to touch cma_id or conn */
conn->state = IBWC_ERROR;
/* it should free the conn */
pctx->connstate_func(NULL, conn);
}
break; /* this is not strictly an error */
case RDMA_CM_EVENT_DISCONNECTED:
DEBUG(DEBUG_DEBUG, ("RDMA_CM_EVENT_DISCONNECTED\n"));
if ((rc=rdma_ack_cm_event(event)))
DEBUG(DEBUG_ERR, ("disc/rdma_ack_cm_event failed with %d\n", rc));
event = NULL; /* don't ack more */
if (cma_id!=pctx->cm_id) {
DEBUG(DEBUG_ERR, ("client DISCONNECT event cm_id=%p\n", cma_id));
conn = talloc_get_type(cma_id->context, struct ibw_conn);
conn->state = IBWC_DISCONNECTED;
pctx->connstate_func(NULL, conn);
}
break;
case RDMA_CM_EVENT_DEVICE_REMOVAL:
sprintf(ibw_lasterr, "cma detected device removal!\n");
goto error;
default:
sprintf(ibw_lasterr, "unknown event %d\n", event->event);
goto error;
}
if (event!=NULL && (rc=rdma_ack_cm_event(event))) {
sprintf(ibw_lasterr, "rdma_ack_cm_event failed with %d\n", rc);
goto error;
}
return;
error:
DEBUG(DEBUG_ERR, ("cm event handler: %s", ibw_lasterr));
if (event!=NULL) {
if (cma_id!=NULL && cma_id!=pctx->cm_id) {
conn = talloc_get_type(cma_id->context, struct ibw_conn);
if (conn) {
conn->state = IBWC_ERROR;
pctx->connstate_func(NULL, conn);
}
} else {
ctx->state = IBWS_ERROR;
pctx->connstate_func(ctx, NULL);
}
if ((rc=rdma_ack_cm_event(event))!=0) {
DEBUG(DEBUG_ERR, ("rdma_ack_cm_event failed with %d\n", rc));
}
}
return;
}
int main(int argc, char *argv[]) {
struct pdata rep_pdata;
struct rdma_event_channel *cm_channel;
struct rdma_cm_id *listen_id;
struct rdma_cm_id *cm_id;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param = { };
struct ibv_pd *pd;
struct ibv_comp_channel *comp_chan;
struct ibv_cq *cq;
struct ibv_cq *evt_cq;
struct ibv_mr *mr;
struct ibv_qp_init_attr qp_attr = { };
struct ibv_sge sge;
struct ibv_send_wr send_wr = { };
struct ibv_send_wr *bad_send_wr;
struct ibv_recv_wr recv_wr = { };
struct ibv_recv_wr *bad_recv_wr;
struct ibv_wc wc;
void *cq_context;
struct sockaddr_in sin;
uint32_t *buf;
int err;
/* Set up RDMA CM structures */
cm_channel = rdma_create_event_channel();
if (!cm_channel)
return 1;
err = rdma_create_id(cm_channel, &listen_id, NULL, RDMA_PS_TCP);
if (err)
return err;
sin.sin_family = AF_INET;
sin.sin_port = htons(20079);
sin.sin_addr.s_addr = INADDR_ANY;
/* Bind to local port and listen for connection request */
err = rdma_bind_addr(listen_id, (struct sockaddr *) &sin);
if (err)
return 1;
err = rdma_listen(listen_id, 1);
if (err)
return 1;
err = rdma_get_cm_event(cm_channel, &event);
if (err)
return err;
printf("after get_cm_event\n");
if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST)
return 1;
cm_id = event->id;
rdma_ack_cm_event(event);
/* Create verbs objects now that we know which device to use */
pd = ibv_alloc_pd(cm_id->verbs);
if (!pd)
return 1;
comp_chan = ibv_create_comp_channel(cm_id->verbs);
if (!comp_chan)
return 1;
cq = ibv_create_cq(cm_id->verbs, 2, NULL, comp_chan, 0);
if (!cq)
return 1;
if (ibv_req_notify_cq(cq, 0))
return 1;
buf = calloc(2, sizeof(uint32_t));
if (!buf)
return 1;
mr = ibv_reg_mr(pd, buf, 2 * sizeof(uint32_t),
IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_READ
| IBV_ACCESS_REMOTE_WRITE);
if (!mr)
return 1;
qp_attr.cap.max_send_wr = 1;
qp_attr.cap.max_send_sge = 1;
qp_attr.cap.max_recv_wr = 1;
qp_attr.cap.max_recv_sge = 1;
qp_attr.send_cq = cq;
qp_attr.recv_cq = cq;
qp_attr.qp_type = IBV_QPT_RC;
err = rdma_create_qp(cm_id, pd, &qp_attr);
if (err)
return err;
/* Post receive before accepting connection */
sge.addr = (uintptr_t) buf + sizeof(uint32_t);
sge.length = sizeof(uint32_t);
sge.lkey = mr->lkey;
recv_wr.sg_list = &sge;
recv_wr.num_sge = 1;
if (ibv_post_recv(cm_id->qp, &recv_wr, &bad_recv_wr))
return 1;
rep_pdata.buf_va = htonll((uintptr_t) buf);
rep_pdata.buf_rkey = htonl(mr->rkey);
conn_param.responder_resources = 1;
conn_param.private_data = &rep_pdata;
conn_param.private_data_len = sizeof rep_pdata;
/* Accept connection */
printf("before accept\n");
err = rdma_accept(cm_id, &conn_param);
if (err)
return 1;
printf("after accept\n");
err = rdma_get_cm_event(cm_channel, &event);
if (err)
return err;
if (event->event != RDMA_CM_EVENT_ESTABLISHED)
return 1;
rdma_ack_cm_event(event);
/* Wait for receive completion */
if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context))
return 1;
if (ibv_req_notify_cq(cq, 0))
return 1;
if (ibv_poll_cq(cq, 1, &wc) < 1)
return 1;
if (wc.status != IBV_WC_SUCCESS)
return 1;
/* Add two integers and send reply back */
buf[0] = htonl(ntohl(buf[0]) + ntohl(buf[1]));
sge.addr = (uintptr_t) buf;
sge.length = sizeof(uint32_t);
sge.lkey = mr->lkey;
send_wr.opcode = IBV_WR_SEND;
send_wr.send_flags = IBV_SEND_SIGNALED;
send_wr.sg_list = &sge;
send_wr.num_sge = 1;
if (ibv_post_send(cm_id->qp, &send_wr, &bad_send_wr))
return 1;
/* Wait for send completion */
if (ibv_get_cq_event(comp_chan, &evt_cq, &cq_context))
return 1;
if (ibv_poll_cq(cq, 1, &wc) < 1)
return 1;
if (wc.status != IBV_WC_SUCCESS)
return 1;
printf("before ack cq 2\n");
ibv_ack_cq_events(cq, 2);
return 0;
}
JNIEXPORT jint JNICALL Java_com_madsys_rdma_RDMASelector_isSet(JNIEnv *env, jobject obj, jlong readfd, jlong ChannelStruct)
{
fd_set *fd = (fd_set *)readfd;
struct rdma_core *core = (struct rdma_core *)ChannelStruct;
int err;
struct ibv_wc wc[2];
struct ibv_cq *evt_cq;
void *cq_context;
if(core->cm_channel != NULL)
{
if(FD_ISSET(core->cm_channel->fd,fd))
{
if((err = rdma_get_cm_event(core->cm_channel,&core->event)) < 0)
{
printf("rdma_get_cm_event fails.\n");
return RDMA_INVALID;
}
switch(core->event->event)
{
case RDMA_CM_EVENT_CONNECT_REQUEST:
return RDMA_CONNECT;
case RDMA_CM_EVENT_ESTABLISHED:
rdma_ack_cm_event(core->event);
return RDMA_ESTABLISHED;
case RDMA_CM_EVENT_DISCONNECTED:
rdma_ack_cm_event(core->event);
return RDMA_DISCONNECT;
default:
rdma_ack_cm_event(core->event);
return RDMA_INVALID;
}
}
}
if(core->comp_chan != NULL)
{
int i,num;
int type = 0;
if(FD_ISSET(core->comp_chan->fd,fd))
{
while(ibv_get_cq_event(core->comp_chan,&evt_cq,&cq_context));
ibv_req_notify_cq(core->cq,0);
ibv_ack_cq_events(core->cq,1);
if( (num = ibv_poll_cq(core->cq,2,wc)) > 0 )
{
for(i=0;i<num;i++)
{
if(wc[0].status == IBV_WC_SUCCESS)
{
if(wc[0].opcode == IBV_WC_SEND)
type |= RDMA_WRITE;
else if(wc[0].opcode & IBV_WC_RECV)
type |= RDMA_READ;
}
}
if(type == 0)
type = RDMA_INVALID;
}
else
return RDMA_INVALID;
return type;
}
}
return RDMA_NOTHING;
}
int rdma_client_connect(struct pingpong_context *ctx,struct perftest_parameters *user_param)
{
char *service;
int temp,num_of_retry= NUM_OF_RETRIES;
struct sockaddr_in sin;
struct addrinfo *res;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param;
struct addrinfo hints;
memset(&hints, 0, sizeof hints);
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
if (check_add_port(&service,user_param->port,user_param->servername,&hints,&res)) {
fprintf(stderr, "Problem in resolving basic adress and port\n");
return FAILURE;
}
sin.sin_addr.s_addr = ((struct sockaddr_in*)res->ai_addr)->sin_addr.s_addr;
sin.sin_family = PF_INET;
sin.sin_port = htons((unsigned short)user_param->port);
while (1) {
if (num_of_retry == 0) {
fprintf(stderr, "Received %d times ADDR_ERROR\n",NUM_OF_RETRIES);
return FAILURE;
}
if (rdma_resolve_addr(ctx->cm_id, NULL,(struct sockaddr *)&sin,2000)) {
fprintf(stderr, "rdma_resolve_addr failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event == RDMA_CM_EVENT_ADDR_ERROR) {
num_of_retry--;
rdma_ack_cm_event(event);
continue;
}
if (event->event != RDMA_CM_EVENT_ADDR_RESOLVED) {
fprintf(stderr, "unexpected CM event %d\n",event->event);
rdma_ack_cm_event(event);
return FAILURE;
}
rdma_ack_cm_event(event);
break;
}
if (user_param->tos != DEF_TOS) {
if (rdma_set_option(ctx->cm_id,RDMA_OPTION_ID,RDMA_OPTION_ID_TOS,&user_param->tos,sizeof(uint8_t))) {
fprintf(stderr, " Set TOS option failed: %d\n",event->event);
return FAILURE;
}
}
while (1) {
if (num_of_retry <= 0) {
fprintf(stderr, "Received %d times ADDR_ERROR - aborting\n",NUM_OF_RETRIES);
return FAILURE;
}
if (rdma_resolve_route(ctx->cm_id,2000)) {
fprintf(stderr, "rdma_resolve_route failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event == RDMA_CM_EVENT_ROUTE_ERROR) {
num_of_retry--;
rdma_ack_cm_event(event);
continue;
}
if (event->event != RDMA_CM_EVENT_ROUTE_RESOLVED) {
fprintf(stderr, "unexpected CM event %d\n",event->event);
rdma_ack_cm_event(event);
return FAILURE;
}
rdma_ack_cm_event(event);
break;
}
ctx->context = ctx->cm_id->verbs;
temp = user_param->work_rdma_cm;
user_param->work_rdma_cm = ON;
if (ctx_init(ctx,user_param)) {
fprintf(stderr," Unable to create the resources needed by comm struct\n");
return FAILURE;
}
memset(&conn_param, 0, sizeof conn_param);
if (user_param->verb == READ || user_param->verb == ATOMIC) {
conn_param.responder_resources = user_param->out_reads;
conn_param.initiator_depth = user_param->out_reads;
}
user_param->work_rdma_cm = temp;
conn_param.retry_count = user_param->retry_count;
conn_param.rnr_retry_count = 7;
if (user_param->work_rdma_cm == OFF) {
if (post_one_recv_wqe(ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
}
if (rdma_connect(ctx->cm_id,&conn_param)) {
fprintf(stderr, "Function rdma_connect failed\n");
return FAILURE;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return FAILURE;
}
if (event->event != RDMA_CM_EVENT_ESTABLISHED) {
rdma_ack_cm_event(event);
fprintf(stderr, "Unexpected CM event bl blka %d\n", event->event);
return FAILURE;
}
if (user_param->connection_type == UD) {
user_param->rem_ud_qpn = event->param.ud.qp_num;
user_param->rem_ud_qkey = event->param.ud.qkey;
ctx->ah[0] = ibv_create_ah(ctx->pd,&event->param.ud.ah_attr);
if (!ctx->ah) {
printf(" Unable to create address handler for UD QP\n");
return FAILURE;
}
if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) {
if (send_qp_num_for_ah(ctx,user_param)) {
printf(" Unable to send my QP number\n");
return FAILURE;
}
}
}
rdma_ack_cm_event(event);
return SUCCESS;
}
/******************************************************************************
+ *
+ ******************************************************************************/
int rdma_server_connect(struct pingpong_context *ctx,
struct perftest_parameters *user_param)
{
int temp;
struct addrinfo *res;
struct rdma_cm_event *event;
struct rdma_conn_param conn_param;
struct addrinfo hints;
char *service;
struct sockaddr_in sin;
memset(&hints, 0, sizeof hints);
hints.ai_flags = AI_PASSIVE;
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
if (check_add_port(&service,user_param->port,user_param->servername,&hints,&res)) {
fprintf(stderr, "Problem in resolving basic adress and port\n");
return FAILURE;
}
sin.sin_addr.s_addr = 0;
sin.sin_family = PF_INET;
sin.sin_port = htons((unsigned short)user_param->port);
if (rdma_bind_addr(ctx->cm_id_control,(struct sockaddr *)&sin)) {
fprintf(stderr," rdma_bind_addr failed\n");
return 1;
}
if (rdma_listen(ctx->cm_id_control,0)) {
fprintf(stderr, "rdma_listen failed\n");
return 1;
}
if (rdma_get_cm_event(ctx->cm_channel,&event)) {
fprintf(stderr, "rdma_get_cm_events failed\n");
return 1;
}
if (event->event != RDMA_CM_EVENT_CONNECT_REQUEST) {
fprintf(stderr, "bad event waiting for connect request %d\n",event->event);
return 1;
}
ctx->cm_id = (struct rdma_cm_id*)event->id;
ctx->context = ctx->cm_id->verbs;
if (user_param->work_rdma_cm == ON)
alloc_ctx(ctx,user_param);
temp = user_param->work_rdma_cm;
user_param->work_rdma_cm = ON;
if (ctx_init(ctx,user_param)) {
fprintf(stderr," Unable to create the resources needed by comm struct\n");
return FAILURE;
}
memset(&conn_param, 0, sizeof conn_param);
if (user_param->verb == READ || user_param->verb == ATOMIC) {
conn_param.responder_resources = user_param->out_reads;
conn_param.initiator_depth = user_param->out_reads;
}
if (user_param->connection_type == UD)
conn_param.qp_num = ctx->qp[0]->qp_num;
conn_param.retry_count = user_param->retry_count;
conn_param.rnr_retry_count = 7;
user_param->work_rdma_cm = temp;
if (user_param->work_rdma_cm == OFF) {
if (post_one_recv_wqe(ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
} else if (user_param->connection_type == UD) {
if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) {
if (post_recv_to_get_ah(ctx)) {
fprintf(stderr, "Couldn't post send \n");
return 1;
}
}
}
if (rdma_accept(ctx->cm_id, &conn_param)) {
fprintf(stderr, "Function rdma_accept failed\n");
return 1;
}
if (user_param->work_rdma_cm && user_param->connection_type == UD) {
if (user_param->tst == LAT || (user_param->tst == BW && user_param->duplex)) {
if (create_ah_from_wc_recv(ctx,user_param)) {
fprintf(stderr, "Unable to create AH from WC\n");
return 1;
}
}
}
rdma_ack_cm_event(event);
rdma_destroy_id(ctx->cm_id_control);
freeaddrinfo(res);
return 0;
}
void client_test(char *ip, char *port) {
struct addrinfo *addr;
struct rdma_cm_event *event = NULL;
struct rdma_cm_id *conn= NULL;
struct rdma_event_channel *ec = NULL;
struct timeval t1, t2, t3, t11, t12;
struct timeval dt, dt1, dt2, dt11, dt12, dt13;
gettimeofday(&t1, NULL);
TEST_NZ(getaddrinfo(ip, port, NULL, &addr));
TEST_Z(ec = rdma_create_event_channel());
gettimeofday(&t11, NULL);
TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
gettimeofday(&t12, NULL);
TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
freeaddrinfo(addr);
gettimeofday(&t2, NULL);
while (rdma_get_cm_event(ec, &event) == 0) {
struct rdma_cm_event event_copy;
memcpy(&event_copy, event, sizeof(*event));
rdma_ack_cm_event(event);
if (on_event(&event_copy)) {
s_ctx->ec = ec;
s_ctx->id = conn;
on_connection(event_copy.id);//send our memory information to server using post_send
poll_cq(NULL);//wait for send_completion
poll_cq(NULL);//wait for recv_completion
break;
}
}
gettimeofday(&t3, NULL);
timersub(&t3, &t1, &dt);
timersub(&t3, &t2, &dt2);
timersub(&t2, &t1, &dt1);
timersub(&t2, &t12, &dt13);
timersub(&t12, &t11, &dt12);
timersub(&t11, &t1, &dt11);
long usec = dt.tv_usec + 10000 * dt.tv_sec;
printf("[dt]:\t%ld us.\n", usec);
printf("[dt1]:\t%ld us.\n", dt1.tv_usec+1000000 *dt1.tv_sec);
printf("Including the following steps: \n");
printf("[dt11]:\t%ld us.\n", dt11.tv_usec+1000000 *dt11.tv_sec);
printf("[dt12]:\t%ld us.\n", dt12.tv_usec+1000000 *dt12.tv_sec);
printf("[dt13]:\t%ld us.\n", dt13.tv_usec+1000000 *dt13.tv_sec);
printf("[dt2] takes %ld micro_secs.\n", dt2.tv_usec+1000000*dt2.tv_sec);
printf("[dt]:total time\t[dt1]:pre_setup\t[dt2]:send/recv\t.\n");
printf("[dt11]:create_event_channel\t[dt12]:create_id\t[dt13]:resolve_address.\n");
return;
};
