int get_rdma_addr(char *src, char *dst, char *port,
struct rdma_addrinfo *hints, struct rdma_addrinfo **rai)
{
struct rdma_addrinfo rai_hints, *res;
int ret;
if (hints->ai_flags & RAI_PASSIVE)
return rdma_getaddrinfo(src, port, hints, rai);
rai_hints = *hints;
if (src) {
rai_hints.ai_flags |= RAI_PASSIVE;
ret = rdma_getaddrinfo(src, NULL, &rai_hints, &res);
if (ret)
return ret;
rai_hints.ai_src_addr = res->ai_src_addr;
rai_hints.ai_src_len = res->ai_src_len;
rai_hints.ai_flags &= ~RAI_PASSIVE;
}
ret = rdma_getaddrinfo(dst, port, &rai_hints, rai);
if (src)
rdma_freeaddrinfo(res);
return ret;
}
int fi_ibv_get_rdma_rai(const char *node, const char *service, uint64_t flags,
const struct fi_info *hints, struct rdma_addrinfo **rai)
{
struct rdma_addrinfo rai_hints, *_rai;
struct rdma_addrinfo **rai_current;
int ret = fi_ibv_fi_to_rai(hints, flags, &rai_hints);
if (ret)
goto out;
if (!node && !rai_hints.ai_dst_addr) {
if ((!rai_hints.ai_src_addr && !service) ||
(!rai_hints.ai_src_addr && FI_IBV_EP_TYPE_IS_RDM(hints)))
{
node = local_node;
}
rai_hints.ai_flags |= RAI_PASSIVE;
}
ret = rdma_getaddrinfo((char *) node, (char *) service,
&rai_hints, &_rai);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_getaddrinfo", errno);
if (errno) {
ret = -errno;
}
goto out;
}
/*
* If caller requested rai, remove ib_rai entries added by IBACM to
* prevent wrong ib_connect_hdr from being sent in connect request.
*/
if (rai && hints && (hints->addr_format != FI_SOCKADDR_IB)) {
for (rai_current = &_rai; *rai_current;) {
struct rdma_addrinfo *rai_next;
if ((*rai_current)->ai_family == AF_IB) {
rai_next = (*rai_current)->ai_next;
(*rai_current)->ai_next = NULL;
rdma_freeaddrinfo(*rai_current);
*rai_current = rai_next;
continue;
}
rai_current = &(*rai_current)->ai_next;
}
}
if (rai)
*rai = _rai;
out:
if (rai_hints.ai_src_addr)
free(rai_hints.ai_src_addr);
if (rai_hints.ai_dst_addr)
free(rai_hints.ai_dst_addr);
return ret;
}
static int server_listen(void)
{
struct rdma_addrinfo *rai = NULL;
struct addrinfo *ai;
int val, ret;
if (use_rgai) {
rai_hints.ai_flags |= RAI_PASSIVE;
ret = rdma_getaddrinfo(src_addr, port, &rai_hints, &rai);
} else {
ai_hints.ai_flags |= AI_PASSIVE;
ret = getaddrinfo(src_addr, port, &ai_hints, &ai);
}
if (ret) {
perror("getaddrinfo");
return ret;
}
lrs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) :
rs_socket(ai->ai_family, SOCK_STREAM, 0);
if (lrs < 0) {
perror("rsocket");
ret = lrs;
goto free;
}
val = 1;
ret = rs_setsockopt(lrs, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val);
if (ret) {
perror("rsetsockopt SO_REUSEADDR");
goto close;
}
ret = rai ? rs_bind(lrs, rai->ai_src_addr, rai->ai_src_len) :
rs_bind(lrs, ai->ai_addr, ai->ai_addrlen);
if (ret) {
perror("rbind");
goto close;
}
ret = rs_listen(lrs, 1);
if (ret)
perror("rlisten");
close:
if (ret)
rs_close(lrs);
free:
if (rai)
rdma_freeaddrinfo(rai);
else
freeaddrinfo(ai);
return ret;
}
RDMACMSocket* RDMACMSocket::connect(const HostAndPort& hp) {
struct rdma_cm_id* client_id = NULL;
struct rdma_addrinfo hints, *res;
memset(&hints, 0, sizeof(hints));
hints.ai_port_space = RDMA_PS_TCP;
res = NULL;
char* hostname = const_cast<char*>(hp.hostname);
char* port_str = const_cast<char*>(hp.port_str);
if (rdma_getaddrinfo(hostname, port_str, &hints, &res) < 0) {
perror("rdma_getaddrinfo");
exit(1);
}
struct ibv_qp_init_attr attr;
memset(&attr, 0, sizeof(attr));
attr.cap.max_send_wr = PACKET_WINDOW_SIZE;
attr.cap.max_recv_wr = PACKET_WINDOW_SIZE;
attr.cap.max_send_sge = 1;
attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = 0;
attr.sq_sig_all = 1;
if (rdma_create_ep(&client_id, res, NULL, &attr) < 0) {
rdma_freeaddrinfo(res);
perror("rdma_create_ep");
exit(1);
}
rdma_freeaddrinfo(res);
if (rdma_connect(client_id, NULL) < 0) {
rdma_destroy_ep(client_id);
perror("rdma_connect");
exit(1);
}
return new RDMACMSocket(client_id);
}
/*
* Listen for XRC RECV QP connection request.
*/
static struct rdma_cm_id * xrc_listen_recv(void)
{
struct rdma_addrinfo hints, *res;
struct rdma_cm_id *id;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_flags = RAI_PASSIVE;
hints.ai_port_space = RDMA_PS_IB;
hints.ai_qp_type = IBV_QPT_XRC_RECV;
ret = rdma_getaddrinfo(NULL, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo listen recv %d\n", errno);
return NULL;
}
ret = rdma_create_ep(&listen_id, res, NULL, NULL);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep listen recv %d\n", errno);
return NULL;
}
ret = rdma_listen(listen_id, 0);
if (ret) {
printf("rdma_listen %d\n", errno);
return NULL;
}
ret = rdma_get_request(listen_id, &id);
if (ret) {
printf("rdma_get_request %d\n", errno);
return NULL;
}
return id;
}
static int client_connect(void)
{
struct rdma_addrinfo *rai = NULL;
struct addrinfo *ai;
struct pollfd fds;
int ret, err;
socklen_t len;
ret = use_rgai ? rdma_getaddrinfo(dst_addr, port, &rai_hints, &rai) :
getaddrinfo(dst_addr, port, &ai_hints, &ai);
if (ret) {
perror("getaddrinfo");
return ret;
}
rs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) :
rs_socket(ai->ai_family, SOCK_STREAM, 0);
if (rs < 0) {
perror("rsocket");
ret = rs;
goto free;
}
set_options(rs);
/* TODO: bind client to src_addr */
if (rai && rai->ai_route) {
ret = rs_setsockopt(rs, SOL_RDMA, RDMA_ROUTE, rai->ai_route,
rai->ai_route_len);
if (ret) {
perror("rsetsockopt RDMA_ROUTE");
goto close;
}
}
ret = rai ? rs_connect(rs, rai->ai_dst_addr, rai->ai_dst_len) :
rs_connect(rs, ai->ai_addr, ai->ai_addrlen);
if (ret && (errno != EINPROGRESS)) {
perror("rconnect");
goto close;
}
if (ret && (errno == EINPROGRESS)) {
fds.fd = rs;
fds.events = POLLOUT;
ret = do_poll(&fds, poll_timeout);
if (ret)
goto close;
len = sizeof err;
ret = rs_getsockopt(rs, SOL_SOCKET, SO_ERROR, &err, &len);
if (ret)
goto close;
if (err) {
ret = -1;
errno = err;
perror("async rconnect");
}
}
close:
if (ret)
rs_close(rs);
free:
if (rai)
rdma_freeaddrinfo(rai);
else
freeaddrinfo(ai);
return ret;
}
int
kiro_server_start (KiroServer *self, const char *address, const char *port, void *mem, size_t mem_size)
{
g_return_val_if_fail (self != NULL, -1);
KiroServerPrivate *priv = KIRO_SERVER_GET_PRIVATE (self);
if (priv->base) {
g_debug ("Server already started.");
return -1;
}
if (!mem || mem_size == 0) {
g_warning ("Invalid memory given to provide.");
return -1;
}
struct rdma_addrinfo hints, *res_addrinfo;
memset (&hints, 0, sizeof (hints));
hints.ai_port_space = RDMA_PS_IB;
hints.ai_flags = RAI_PASSIVE;
char *addr_c = g_strdup (address);
char *port_c = g_strdup (port);
int rtn = rdma_getaddrinfo (addr_c, port_c, &hints, &res_addrinfo);
g_free (addr_c);
g_free (port_c);
if (rtn) {
g_critical ("Failed to create address information: %s", strerror (errno));
return -1;
}
struct ibv_qp_init_attr qp_attr;
memset (&qp_attr, 0, sizeof (qp_attr));
qp_attr.cap.max_send_wr = 10;
qp_attr.cap.max_recv_wr = 10;
qp_attr.cap.max_send_sge = 1;
qp_attr.cap.max_recv_sge = 1;
qp_attr.qp_context = priv->base;
qp_attr.sq_sig_all = 1;
if (rdma_create_ep (& (priv->base), res_addrinfo, NULL, &qp_attr)) {
g_critical ("Endpoint creation failed: %s", strerror (errno));
g_free (res_addrinfo);
return -1;
}
g_free (res_addrinfo); // No longer needed
g_debug ("Endpoint created");
char *addr_local = NULL;
struct sockaddr *src_addr = rdma_get_local_addr (priv->base);
if (!src_addr) {
addr_local = "NONE";
}
else {
addr_local = inet_ntoa (((struct sockaddr_in *)src_addr)->sin_addr);
/*
if(src_addr->sa_family == AF_INET)
addr_local = &(((struct sockaddr_in*)src_addr)->sin_addr);
else
addr_local = &(((struct sockaddr_in6*)src_addr)->sin6_addr);
*/
}
g_message ("Server bound to address %s:%s", addr_local, port);
if (rdma_listen (priv->base, 0)) {
g_critical ("Failed to put server into listening state: %s", strerror (errno));
rdma_destroy_ep (priv->base);
return -1;
}
priv->mem = mem;
priv->mem_size = mem_size;
priv->ec = rdma_create_event_channel();
if (rdma_migrate_id (priv->base, priv->ec)) {
g_critical ("Was unable to migrate connection to new Event Channel: %s", strerror (errno));
rdma_destroy_ep (priv->base);
return -1;
}
priv->main_loop = g_main_loop_new (NULL, FALSE);
priv->conn_ec = g_io_channel_unix_new (priv->ec->fd);
g_io_add_watch (priv->conn_ec, G_IO_IN | G_IO_PRI, process_cm_event, (gpointer)priv);
priv->main_thread = g_thread_new ("KIRO Server main loop", start_server_main_loop, priv->main_loop);
// We gave control to the main_loop (with add_watch) and don't need our ref
// any longer
g_io_channel_unref (priv->conn_ec);
g_message ("Enpoint listening");
return 0;
}
static int rc_test(void)
{
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_flags = RAI_PASSIVE;
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(NULL, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = sizeof send_msg;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&listen_id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
ret = rdma_listen(listen_id, 0);
if (ret) {
printf("rdma_listen %d\n", errno);
return ret;
}
ret = rdma_get_request(listen_id, &id);
if (ret) {
printf("rdma_get_request %d\n", errno);
return ret;
}
mr = rdma_reg_msgs(id, recv_msg, sizeof recv_msg);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
ret = rdma_post_recv(id, NULL, recv_msg, sizeof recv_msg, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
ret = rdma_accept(id, NULL);
if (ret) {
printf("rdma_accept %d\n", errno);
return ret;
}
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
ret = rdma_post_send(id, NULL, send_msg, sizeof send_msg, NULL, IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
ret = rdma_get_send_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_send_comp %d\n", ret);
return ret;
}
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
rdma_destroy_ep(listen_id);
return 0;
}
static int run(void)
{
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(server, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = 16;
attr.qp_context = id;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
mr = rdma_reg_msgs(id, recv_msg, 16);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
ret = rdma_connect(id, NULL);
if (ret) {
printf("rdma_connect %d\n", errno);
return ret;
}
s = get_dtime();
ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
e = get_dtime();
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
printf("time %f\n", e - s);
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
return 0;
}
