int get_rdma_addr(char *src, char *dst, char *port, struct rdma_addrinfo *hints, struct rdma_addrinfo **rai) { struct rdma_addrinfo rai_hints, *res; int ret; if (hints->ai_flags & RAI_PASSIVE) return rdma_getaddrinfo(src, port, hints, rai); rai_hints = *hints; if (src) { rai_hints.ai_flags |= RAI_PASSIVE; ret = rdma_getaddrinfo(src, NULL, &rai_hints, &res); if (ret) return ret; rai_hints.ai_src_addr = res->ai_src_addr; rai_hints.ai_src_len = res->ai_src_len; rai_hints.ai_flags &= ~RAI_PASSIVE; } ret = rdma_getaddrinfo(dst, port, &rai_hints, rai); if (src) rdma_freeaddrinfo(res); return ret; }
int fi_ibv_get_rdma_rai(const char *node, const char *service, uint64_t flags, const struct fi_info *hints, struct rdma_addrinfo **rai) { struct rdma_addrinfo rai_hints, *_rai; struct rdma_addrinfo **rai_current; int ret = fi_ibv_fi_to_rai(hints, flags, &rai_hints); if (ret) goto out; if (!node && !rai_hints.ai_dst_addr) { if ((!rai_hints.ai_src_addr && !service) || (!rai_hints.ai_src_addr && FI_IBV_EP_TYPE_IS_RDM(hints))) { node = local_node; } rai_hints.ai_flags |= RAI_PASSIVE; } ret = rdma_getaddrinfo((char *) node, (char *) service, &rai_hints, &_rai); if (ret) { VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_getaddrinfo", errno); if (errno) { ret = -errno; } goto out; } /* * If caller requested rai, remove ib_rai entries added by IBACM to * prevent wrong ib_connect_hdr from being sent in connect request. */ if (rai && hints && (hints->addr_format != FI_SOCKADDR_IB)) { for (rai_current = &_rai; *rai_current;) { struct rdma_addrinfo *rai_next; if ((*rai_current)->ai_family == AF_IB) { rai_next = (*rai_current)->ai_next; (*rai_current)->ai_next = NULL; rdma_freeaddrinfo(*rai_current); *rai_current = rai_next; continue; } rai_current = &(*rai_current)->ai_next; } } if (rai) *rai = _rai; out: if (rai_hints.ai_src_addr) free(rai_hints.ai_src_addr); if (rai_hints.ai_dst_addr) free(rai_hints.ai_dst_addr); return ret; }
static int server_listen(void) { struct rdma_addrinfo *rai = NULL; struct addrinfo *ai; int val, ret; if (use_rgai) { rai_hints.ai_flags |= RAI_PASSIVE; ret = rdma_getaddrinfo(src_addr, port, &rai_hints, &rai); } else { ai_hints.ai_flags |= AI_PASSIVE; ret = getaddrinfo(src_addr, port, &ai_hints, &ai); } if (ret) { perror("getaddrinfo"); return ret; } lrs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) : rs_socket(ai->ai_family, SOCK_STREAM, 0); if (lrs < 0) { perror("rsocket"); ret = lrs; goto free; } val = 1; ret = rs_setsockopt(lrs, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val); if (ret) { perror("rsetsockopt SO_REUSEADDR"); goto close; } ret = rai ? rs_bind(lrs, rai->ai_src_addr, rai->ai_src_len) : rs_bind(lrs, ai->ai_addr, ai->ai_addrlen); if (ret) { perror("rbind"); goto close; } ret = rs_listen(lrs, 1); if (ret) perror("rlisten"); close: if (ret) rs_close(lrs); free: if (rai) rdma_freeaddrinfo(rai); else freeaddrinfo(ai); return ret; }
RDMACMSocket* RDMACMSocket::connect(const HostAndPort& hp) { struct rdma_cm_id* client_id = NULL; struct rdma_addrinfo hints, *res; memset(&hints, 0, sizeof(hints)); hints.ai_port_space = RDMA_PS_TCP; res = NULL; char* hostname = const_cast<char*>(hp.hostname); char* port_str = const_cast<char*>(hp.port_str); if (rdma_getaddrinfo(hostname, port_str, &hints, &res) < 0) { perror("rdma_getaddrinfo"); exit(1); } struct ibv_qp_init_attr attr; memset(&attr, 0, sizeof(attr)); attr.cap.max_send_wr = PACKET_WINDOW_SIZE; attr.cap.max_recv_wr = PACKET_WINDOW_SIZE; attr.cap.max_send_sge = 1; attr.cap.max_recv_sge = 1; attr.cap.max_inline_data = 0; attr.sq_sig_all = 1; if (rdma_create_ep(&client_id, res, NULL, &attr) < 0) { rdma_freeaddrinfo(res); perror("rdma_create_ep"); exit(1); } rdma_freeaddrinfo(res); if (rdma_connect(client_id, NULL) < 0) { rdma_destroy_ep(client_id); perror("rdma_connect"); exit(1); } return new RDMACMSocket(client_id); }
/* * Listen for XRC RECV QP connection request. */ static struct rdma_cm_id * xrc_listen_recv(void) { struct rdma_addrinfo hints, *res; struct rdma_cm_id *id; int ret; memset(&hints, 0, sizeof hints); hints.ai_flags = RAI_PASSIVE; hints.ai_port_space = RDMA_PS_IB; hints.ai_qp_type = IBV_QPT_XRC_RECV; ret = rdma_getaddrinfo(NULL, port, &hints, &res); if (ret) { printf("rdma_getaddrinfo listen recv %d\n", errno); return NULL; } ret = rdma_create_ep(&listen_id, res, NULL, NULL); rdma_freeaddrinfo(res); if (ret) { printf("rdma_create_ep listen recv %d\n", errno); return NULL; } ret = rdma_listen(listen_id, 0); if (ret) { printf("rdma_listen %d\n", errno); return NULL; } ret = rdma_get_request(listen_id, &id); if (ret) { printf("rdma_get_request %d\n", errno); return NULL; } return id; }
static int client_connect(void) { struct rdma_addrinfo *rai = NULL; struct addrinfo *ai; struct pollfd fds; int ret, err; socklen_t len; ret = use_rgai ? rdma_getaddrinfo(dst_addr, port, &rai_hints, &rai) : getaddrinfo(dst_addr, port, &ai_hints, &ai); if (ret) { perror("getaddrinfo"); return ret; } rs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) : rs_socket(ai->ai_family, SOCK_STREAM, 0); if (rs < 0) { perror("rsocket"); ret = rs; goto free; } set_options(rs); /* TODO: bind client to src_addr */ if (rai && rai->ai_route) { ret = rs_setsockopt(rs, SOL_RDMA, RDMA_ROUTE, rai->ai_route, rai->ai_route_len); if (ret) { perror("rsetsockopt RDMA_ROUTE"); goto close; } } ret = rai ? rs_connect(rs, rai->ai_dst_addr, rai->ai_dst_len) : rs_connect(rs, ai->ai_addr, ai->ai_addrlen); if (ret && (errno != EINPROGRESS)) { perror("rconnect"); goto close; } if (ret && (errno == EINPROGRESS)) { fds.fd = rs; fds.events = POLLOUT; ret = do_poll(&fds, poll_timeout); if (ret) goto close; len = sizeof err; ret = rs_getsockopt(rs, SOL_SOCKET, SO_ERROR, &err, &len); if (ret) goto close; if (err) { ret = -1; errno = err; perror("async rconnect"); } } close: if (ret) rs_close(rs); free: if (rai) rdma_freeaddrinfo(rai); else freeaddrinfo(ai); return ret; }
int kiro_server_start (KiroServer *self, const char *address, const char *port, void *mem, size_t mem_size) { g_return_val_if_fail (self != NULL, -1); KiroServerPrivate *priv = KIRO_SERVER_GET_PRIVATE (self); if (priv->base) { g_debug ("Server already started."); return -1; } if (!mem || mem_size == 0) { g_warning ("Invalid memory given to provide."); return -1; } struct rdma_addrinfo hints, *res_addrinfo; memset (&hints, 0, sizeof (hints)); hints.ai_port_space = RDMA_PS_IB; hints.ai_flags = RAI_PASSIVE; char *addr_c = g_strdup (address); char *port_c = g_strdup (port); int rtn = rdma_getaddrinfo (addr_c, port_c, &hints, &res_addrinfo); g_free (addr_c); g_free (port_c); if (rtn) { g_critical ("Failed to create address information: %s", strerror (errno)); return -1; } struct ibv_qp_init_attr qp_attr; memset (&qp_attr, 0, sizeof (qp_attr)); qp_attr.cap.max_send_wr = 10; qp_attr.cap.max_recv_wr = 10; qp_attr.cap.max_send_sge = 1; qp_attr.cap.max_recv_sge = 1; qp_attr.qp_context = priv->base; qp_attr.sq_sig_all = 1; if (rdma_create_ep (& (priv->base), res_addrinfo, NULL, &qp_attr)) { g_critical ("Endpoint creation failed: %s", strerror (errno)); g_free (res_addrinfo); return -1; } g_free (res_addrinfo); // No longer needed g_debug ("Endpoint created"); char *addr_local = NULL; struct sockaddr *src_addr = rdma_get_local_addr (priv->base); if (!src_addr) { addr_local = "NONE"; } else { addr_local = inet_ntoa (((struct sockaddr_in *)src_addr)->sin_addr); /* if(src_addr->sa_family == AF_INET) addr_local = &(((struct sockaddr_in*)src_addr)->sin_addr); else addr_local = &(((struct sockaddr_in6*)src_addr)->sin6_addr); */ } g_message ("Server bound to address %s:%s", addr_local, port); if (rdma_listen (priv->base, 0)) { g_critical ("Failed to put server into listening state: %s", strerror (errno)); rdma_destroy_ep (priv->base); return -1; } priv->mem = mem; priv->mem_size = mem_size; priv->ec = rdma_create_event_channel(); if (rdma_migrate_id (priv->base, priv->ec)) { g_critical ("Was unable to migrate connection to new Event Channel: %s", strerror (errno)); rdma_destroy_ep (priv->base); return -1; } priv->main_loop = g_main_loop_new (NULL, FALSE); priv->conn_ec = g_io_channel_unix_new (priv->ec->fd); g_io_add_watch (priv->conn_ec, G_IO_IN | G_IO_PRI, process_cm_event, (gpointer)priv); priv->main_thread = g_thread_new ("KIRO Server main loop", start_server_main_loop, priv->main_loop); // We gave control to the main_loop (with add_watch) and don't need our ref // any longer g_io_channel_unref (priv->conn_ec); g_message ("Enpoint listening"); return 0; }
static int rc_test(void) { struct rdma_addrinfo hints, *res; struct ibv_qp_init_attr attr; struct ibv_wc wc; int ret; memset(&hints, 0, sizeof hints); hints.ai_flags = RAI_PASSIVE; hints.ai_port_space = RDMA_PS_TCP; ret = rdma_getaddrinfo(NULL, port, &hints, &res); if (ret) { printf("rdma_getaddrinfo %d\n", errno); return ret; } memset(&attr, 0, sizeof attr); attr.cap.max_send_wr = attr.cap.max_recv_wr = 1; attr.cap.max_send_sge = attr.cap.max_recv_sge = 1; attr.cap.max_inline_data = sizeof send_msg; attr.sq_sig_all = 1; ret = rdma_create_ep(&listen_id, res, NULL, &attr); rdma_freeaddrinfo(res); if (ret) { printf("rdma_create_ep %d\n", errno); return ret; } ret = rdma_listen(listen_id, 0); if (ret) { printf("rdma_listen %d\n", errno); return ret; } ret = rdma_get_request(listen_id, &id); if (ret) { printf("rdma_get_request %d\n", errno); return ret; } mr = rdma_reg_msgs(id, recv_msg, sizeof recv_msg); if (!mr) { printf("rdma_reg_msgs %d\n", errno); return ret; } ret = rdma_post_recv(id, NULL, recv_msg, sizeof recv_msg, mr); if (ret) { printf("rdma_post_recv %d\n", errno); return ret; } ret = rdma_accept(id, NULL); if (ret) { printf("rdma_accept %d\n", errno); return ret; } ret = rdma_get_recv_comp(id, &wc); if (ret <= 0) { printf("rdma_get_recv_comp %d\n", ret); return ret; } ret = rdma_post_send(id, NULL, send_msg, sizeof send_msg, NULL, IBV_SEND_INLINE); if (ret) { printf("rdma_post_send %d\n", errno); return ret; } ret = rdma_get_send_comp(id, &wc); if (ret <= 0) { printf("rdma_get_send_comp %d\n", ret); return ret; } rdma_disconnect(id); rdma_dereg_mr(mr); rdma_destroy_ep(id); rdma_destroy_ep(listen_id); return 0; }
static int run(void) { struct rdma_addrinfo hints, *res; struct ibv_qp_init_attr attr; struct ibv_wc wc; int ret; memset(&hints, 0, sizeof hints); hints.ai_port_space = RDMA_PS_TCP; ret = rdma_getaddrinfo(server, port, &hints, &res); if (ret) { printf("rdma_getaddrinfo %d\n", errno); return ret; } memset(&attr, 0, sizeof attr); attr.cap.max_send_wr = attr.cap.max_recv_wr = 1; attr.cap.max_send_sge = attr.cap.max_recv_sge = 1; attr.cap.max_inline_data = 16; attr.qp_context = id; attr.sq_sig_all = 1; ret = rdma_create_ep(&id, res, NULL, &attr); rdma_freeaddrinfo(res); if (ret) { printf("rdma_create_ep %d\n", errno); return ret; } mr = rdma_reg_msgs(id, recv_msg, 16); if (!mr) { printf("rdma_reg_msgs %d\n", errno); return ret; } ret = rdma_post_recv(id, NULL, recv_msg, 16, mr); if (ret) { printf("rdma_post_recv %d\n", errno); return ret; } ret = rdma_connect(id, NULL); if (ret) { printf("rdma_connect %d\n", errno); return ret; } s = get_dtime(); ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE); if (ret) { printf("rdma_post_send %d\n", errno); return ret; } e = get_dtime(); ret = rdma_get_recv_comp(id, &wc); if (ret <= 0) { printf("rdma_get_recv_comp %d\n", ret); return ret; } printf("time %f\n", e - s); rdma_disconnect(id); rdma_dereg_mr(mr); rdma_destroy_ep(id); return 0; }