int fi_ibv_create_ep(const char *node, const char *service,
uint64_t flags, const struct fi_info *hints,
struct rdma_addrinfo **rai, struct rdma_cm_id **id)
{
struct rdma_addrinfo *_rai = NULL;
int ret;
ret = fi_ibv_get_rdma_rai(node, service, flags, hints, &_rai);
if (ret) {
return ret;
}
ret = rdma_create_ep(id, _rai, NULL, NULL);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_create_ep", errno);
ret = -errno;
goto err1;
}
if (rai) {
*rai = _rai;
} else {
rdma_freeaddrinfo(_rai);
}
return ret;
err1:
rdma_freeaddrinfo(_rai);
return ret;
}
int fi_ibv_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct rdma_cm_id *id;
struct rdma_addrinfo *rai;
int ret;
ret = fi_ibv_init_info();
if (ret)
goto out;
ret = fi_ibv_create_ep(node, service, flags, hints, &rai, &id);
if (ret)
goto out;
if (id->verbs) {
ret = fi_ibv_get_matching_info(ibv_get_device_name(id->verbs->device),
hints, rai, info);
} else {
ret = fi_ibv_get_matching_info(NULL, hints, rai, info);
}
rdma_destroy_ep(id);
rdma_freeaddrinfo(rai);
out:
if (!ret || ret == -FI_ENOMEM)
return ret;
else
return -FI_ENODATA;
}
int get_rdma_addr(char *src, char *dst, char *port,
struct rdma_addrinfo *hints, struct rdma_addrinfo **rai)
{
struct rdma_addrinfo rai_hints, *res;
int ret;
if (hints->ai_flags & RAI_PASSIVE)
return rdma_getaddrinfo(src, port, hints, rai);
rai_hints = *hints;
if (src) {
rai_hints.ai_flags |= RAI_PASSIVE;
ret = rdma_getaddrinfo(src, NULL, &rai_hints, &res);
if (ret)
return ret;
rai_hints.ai_src_addr = res->ai_src_addr;
rai_hints.ai_src_len = res->ai_src_len;
rai_hints.ai_flags &= ~RAI_PASSIVE;
}
ret = rdma_getaddrinfo(dst, port, &rai_hints, rai);
if (src)
rdma_freeaddrinfo(res);
return ret;
}
int fi_ibv_get_rdma_rai(const char *node, const char *service, uint64_t flags,
const struct fi_info *hints, struct rdma_addrinfo **rai)
{
struct rdma_addrinfo rai_hints, *_rai;
struct rdma_addrinfo **rai_current;
int ret = fi_ibv_fi_to_rai(hints, flags, &rai_hints);
if (ret)
goto out;
if (!node && !rai_hints.ai_dst_addr) {
if ((!rai_hints.ai_src_addr && !service) ||
(!rai_hints.ai_src_addr && FI_IBV_EP_TYPE_IS_RDM(hints)))
{
node = local_node;
}
rai_hints.ai_flags |= RAI_PASSIVE;
}
ret = rdma_getaddrinfo((char *) node, (char *) service,
&rai_hints, &_rai);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_getaddrinfo", errno);
if (errno) {
ret = -errno;
}
goto out;
}
/*
* If caller requested rai, remove ib_rai entries added by IBACM to
* prevent wrong ib_connect_hdr from being sent in connect request.
*/
if (rai && hints && (hints->addr_format != FI_SOCKADDR_IB)) {
for (rai_current = &_rai; *rai_current;) {
struct rdma_addrinfo *rai_next;
if ((*rai_current)->ai_family == AF_IB) {
rai_next = (*rai_current)->ai_next;
(*rai_current)->ai_next = NULL;
rdma_freeaddrinfo(*rai_current);
*rai_current = rai_next;
continue;
}
rai_current = &(*rai_current)->ai_next;
}
}
if (rai)
*rai = _rai;
out:
if (rai_hints.ai_src_addr)
free(rai_hints.ai_src_addr);
if (rai_hints.ai_dst_addr)
free(rai_hints.ai_dst_addr);
return ret;
}
int fi_ibv_create_ep(const char *node, const char *service,
uint64_t flags, const struct fi_info *hints,
struct rdma_addrinfo **rai, struct rdma_cm_id **id)
{
struct rdma_addrinfo *_rai;
struct sockaddr *local_addr;
int ret;
ret = fi_ibv_get_rdma_rai(node, service, flags, hints, &_rai);
if (ret) {
return ret;
}
ret = rdma_create_ep(id, _rai, NULL, NULL);
if (ret) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "rdma_create_ep", errno);
ret = -errno;
goto err1;
}
if (rai && !_rai->ai_src_addr) {
local_addr = rdma_get_local_addr(*id);
_rai->ai_src_len = fi_ibv_sockaddr_len(local_addr);
if (!(_rai->ai_src_addr = malloc(_rai->ai_src_len))) {
ret = -FI_ENOMEM;
goto err2;
}
memcpy(_rai->ai_src_addr, local_addr, _rai->ai_src_len);
}
if (rai) {
*rai = _rai;
} else {
rdma_freeaddrinfo(_rai);
}
return ret;
err2:
rdma_destroy_ep(*id);
err1:
rdma_freeaddrinfo(_rai);
return ret;
}
RDMACMSocket* RDMACMSocket::connect(const HostAndPort& hp) {
struct rdma_cm_id* client_id = NULL;
struct rdma_addrinfo hints, *res;
memset(&hints, 0, sizeof(hints));
hints.ai_port_space = RDMA_PS_TCP;
res = NULL;
char* hostname = const_cast<char*>(hp.hostname);
char* port_str = const_cast<char*>(hp.port_str);
if (rdma_getaddrinfo(hostname, port_str, &hints, &res) < 0) {
perror("rdma_getaddrinfo");
exit(1);
}
struct ibv_qp_init_attr attr;
memset(&attr, 0, sizeof(attr));
attr.cap.max_send_wr = PACKET_WINDOW_SIZE;
attr.cap.max_recv_wr = PACKET_WINDOW_SIZE;
attr.cap.max_send_sge = 1;
attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = 0;
attr.sq_sig_all = 1;
if (rdma_create_ep(&client_id, res, NULL, &attr) < 0) {
rdma_freeaddrinfo(res);
perror("rdma_create_ep");
exit(1);
}
rdma_freeaddrinfo(res);
if (rdma_connect(client_id, NULL) < 0) {
rdma_destroy_ep(client_id);
perror("rdma_connect");
exit(1);
}
return new RDMACMSocket(client_id);
}
static int server_listen(void)
{
struct rdma_addrinfo *rai = NULL;
struct addrinfo *ai;
int val, ret;
if (use_rgai) {
rai_hints.ai_flags |= RAI_PASSIVE;
ret = rdma_getaddrinfo(src_addr, port, &rai_hints, &rai);
} else {
ai_hints.ai_flags |= AI_PASSIVE;
ret = getaddrinfo(src_addr, port, &ai_hints, &ai);
}
if (ret) {
perror("getaddrinfo");
return ret;
}
lrs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) :
rs_socket(ai->ai_family, SOCK_STREAM, 0);
if (lrs < 0) {
perror("rsocket");
ret = lrs;
goto free;
}
val = 1;
ret = rs_setsockopt(lrs, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val);
if (ret) {
perror("rsetsockopt SO_REUSEADDR");
goto close;
}
ret = rai ? rs_bind(lrs, rai->ai_src_addr, rai->ai_src_len) :
rs_bind(lrs, ai->ai_addr, ai->ai_addrlen);
if (ret) {
perror("rbind");
goto close;
}
ret = rs_listen(lrs, 1);
if (ret)
perror("rlisten");
close:
if (ret)
rs_close(lrs);
free:
if (rai)
rdma_freeaddrinfo(rai);
else
freeaddrinfo(ai);
return ret;
}
int rdma_getaddrinfo(char *node, char *service,
struct rdma_addrinfo *hints,
struct rdma_addrinfo **res)
{
struct rdma_addrinfo *rai;
int ret;
if (!service && !node && !hints)
return ERR(EINVAL);
ret = ucma_init();
if (ret)
return ret;
rai = calloc(1, sizeof(*rai));
if (!rai)
return ERR(ENOMEM);
if (!hints)
hints = &nohints;
if (node || service) {
ret = ucma_getaddrinfo(node, service, hints, rai);
} else {
rai->ai_flags = hints->ai_flags;
rai->ai_family = hints->ai_family;
rai->ai_qp_type = hints->ai_qp_type;
rai->ai_port_space = hints->ai_port_space;
if (hints->ai_dst_len) {
ret = ucma_copy_addr(&rai->ai_dst_addr, &rai->ai_dst_len,
hints->ai_dst_addr, hints->ai_dst_len);
}
}
if (ret)
goto err;
if (!rai->ai_src_len && hints->ai_src_len) {
ret = ucma_copy_addr(&rai->ai_src_addr, &rai->ai_src_len,
hints->ai_src_addr, hints->ai_src_len);
if (ret)
goto err;
}
if (!(rai->ai_flags & RAI_PASSIVE))
ucma_ib_resolve(&rai, hints);
*res = rai;
return 0;
err:
rdma_freeaddrinfo(rai);
return ret;
}
static void fi_ibv_verbs_devs_free(struct dlist_entry *verbs_devs)
{
struct verbs_dev_info *dev;
struct verbs_addr *addr;
while (!dlist_empty(verbs_devs)) {
dlist_pop_front(verbs_devs, struct verbs_dev_info, dev, entry);
while (!dlist_empty(&dev->addrs)) {
dlist_pop_front(&dev->addrs, struct verbs_addr, addr, entry);
rdma_freeaddrinfo(addr->rai);
free(addr);
}
free(dev->name);
free(dev);
}
}
static int fi_ibv_copy_ifaddr(const char *name, const char *service, uint64_t flags,
struct fi_info *info)
{
struct rdma_addrinfo *rai;
struct fi_info *fi;
struct rdma_cm_id *id;
int ret;
ret = fi_ibv_get_rdma_rai(name, service, flags, NULL, &rai);
if (ret) {
FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC,
"rdma_getaddrinfo failed for name:%s\n", name);
return ret;
}
ret = rdma_create_ep(&id, rai, NULL, NULL);
if (!ret) {
for (fi = info; fi; fi = fi->next)
if (!strncmp(id->verbs->device->name, fi->domain_attr->name,
strlen(id->verbs->device->name)))
break;
if (!fi) {
FI_WARN(&fi_ibv_prov, FI_LOG_FABRIC,
"No matching fi_info for device: "
"%s with address: %s\n",
id->verbs->device->name, name);
} else {
if (fi->src_addr) {
free(fi->src_addr);
fi->src_addr = NULL;
}
fi_ibv_rai_to_fi(rai, fi);
}
rdma_destroy_ep(id);
}
rdma_freeaddrinfo(rai);
return 0;
}
/*
* Listen for XRC RECV QP connection request.
*/
static struct rdma_cm_id * xrc_listen_recv(void)
{
struct rdma_addrinfo hints, *res;
struct rdma_cm_id *id;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_flags = RAI_PASSIVE;
hints.ai_port_space = RDMA_PS_IB;
hints.ai_qp_type = IBV_QPT_XRC_RECV;
ret = rdma_getaddrinfo(NULL, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo listen recv %d\n", errno);
return NULL;
}
ret = rdma_create_ep(&listen_id, res, NULL, NULL);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep listen recv %d\n", errno);
return NULL;
}
ret = rdma_listen(listen_id, 0);
if (ret) {
printf("rdma_listen %d\n", errno);
return NULL;
}
ret = rdma_get_request(listen_id, &id);
if (ret) {
printf("rdma_get_request %d\n", errno);
return NULL;
}
return id;
}
int main(int argc, char **argv)
{
int op, ret;
hints.ai_port_space = RDMA_PS_UDP;
while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:P:f:")) != -1) {
switch (op) {
case 's':
dst_addr = optarg;
break;
case 'b':
src_addr = optarg;
break;
case 'c':
connections = atoi(optarg);
break;
case 'C':
message_count = atoi(optarg);
break;
case 'S':
message_size = atoi(optarg);
break;
case 't':
set_tos = 1;
tos = (uint8_t) strtoul(optarg, NULL, 0);
break;
case 'p': /* for backwards compatibility - use -P */
hints.ai_port_space = strtol(optarg, NULL, 0);
break;
case 'f':
if (!strncasecmp("ip", optarg, 2)) {
hints.ai_flags = RAI_NUMERICHOST;
} else if (!strncasecmp("gid", optarg, 3)) {
hints.ai_flags = RAI_NUMERICHOST | RAI_FAMILY;
hints.ai_family = AF_IB;
} else if (strncasecmp("name", optarg, 4)) {
fprintf(stderr, "Warning: unknown address format\n");
}
break;
case 'P':
if (!strncasecmp("ipoib", optarg, 5)) {
hints.ai_port_space = RDMA_PS_IPOIB;
} else if (strncasecmp("udp", optarg, 3)) {
fprintf(stderr, "Warning: unknown port space format\n");
}
break;
default:
printf("usage: %s\n", argv[0]);
printf("\t[-s server_address]\n");
printf("\t[-b bind_address]\n");
printf("\t[-f address_format]\n");
printf("\t name, ip, ipv6, or gid\n");
printf("\t[-P port_space]\n");
printf("\t udp or ipoib\n");
printf("\t[-c connections]\n");
printf("\t[-C message_count]\n");
printf("\t[-S message_size]\n");
printf("\t[-t type_of_service]\n");
printf("\t[-p port_space - %#x for UDP (default), "
"%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);
exit(1);
}
}
test.connects_left = connections;
test.channel = rdma_create_event_channel();
if (!test.channel) {
perror("failed to create event channel");
exit(1);
}
if (alloc_nodes())
exit(1);
if (dst_addr) {
ret = run_client();
} else {
hints.ai_flags |= RAI_PASSIVE;
ret = run_server();
}
printf("test complete\n");
destroy_nodes();
rdma_destroy_event_channel(test.channel);
if (test.rai)
rdma_freeaddrinfo(test.rai);
printf("return status %d\n", ret);
return ret;
}
int main(int argc, char **argv)
{
int op, ret;
hints.ai_port_space = RDMA_PS_TCP;
while ((op = getopt(argc, argv, "s:b:f:P:c:C:S:t:p:m")) != -1) {
switch (op) {
case 's':
dst_addr = optarg;
break;
case 'b':
src_addr = optarg;
break;
case 'f':
if (!strncasecmp("ip", optarg, 2)) {
hints.ai_flags = RAI_NUMERICHOST;
} else if (!strncasecmp("gid", optarg, 3)) {
hints.ai_flags = RAI_NUMERICHOST | RAI_FAMILY;
hints.ai_family = AF_IB;
} else if (strncasecmp("name", optarg, 4)) {
fprintf(stderr, "Warning: unknown address format\n");
}
break;
case 'P':
if (!strncasecmp("ib", optarg, 2)) {
hints.ai_port_space = RDMA_PS_IB;
} else if (strncasecmp("tcp", optarg, 3)) {
fprintf(stderr, "Warning: unknown port space format\n");
}
break;
case 'c':
connections = atoi(optarg);
break;
case 'C':
message_count = atoi(optarg);
break;
case 'S':
message_size = atoi(optarg);
break;
case 't':
set_tos = 1;
tos = (uint8_t) strtoul(optarg, NULL, 0);
break;
case 'p':
port = optarg;
break;
case 'm':
migrate = 1;
break;
default:
printf("usage: %s\n", argv[0]);
printf("\t[-s server_address]\n");
printf("\t[-b bind_address]\n");
printf("\t[-f address_format]\n");
printf("\t name, ip, ipv6, or gid\n");
printf("\t[-P port_space]\n");
printf("\t tcp or ib\n");
printf("\t[-c connections]\n");
printf("\t[-C message_count]\n");
printf("\t[-S message_size]\n");
printf("\t[-t type_of_service]\n");
printf("\t[-p port_number]\n");
printf("\t[-m(igrate)]\n");
exit(1);
}
}
test.connects_left = connections;
test.channel = create_first_event_channel();
if (!test.channel) {
exit(1);
}
if (alloc_nodes())
exit(1);
if (dst_addr) {
ret = run_client();
} else {
hints.ai_flags |= RAI_PASSIVE;
ret = run_server();
}
printf("test complete\n");
destroy_nodes();
rdma_destroy_event_channel(test.channel);
if (test.rai)
rdma_freeaddrinfo(test.rai);
printf("return status %d\n", ret);
return ret;
}
static int client_connect(void)
{
struct rdma_addrinfo *rai = NULL;
struct addrinfo *ai;
struct pollfd fds;
int ret, err;
socklen_t len;
ret = use_rgai ? rdma_getaddrinfo(dst_addr, port, &rai_hints, &rai) :
getaddrinfo(dst_addr, port, &ai_hints, &ai);
if (ret) {
perror("getaddrinfo");
return ret;
}
rs = rai ? rs_socket(rai->ai_family, SOCK_STREAM, 0) :
rs_socket(ai->ai_family, SOCK_STREAM, 0);
if (rs < 0) {
perror("rsocket");
ret = rs;
goto free;
}
set_options(rs);
/* TODO: bind client to src_addr */
if (rai && rai->ai_route) {
ret = rs_setsockopt(rs, SOL_RDMA, RDMA_ROUTE, rai->ai_route,
rai->ai_route_len);
if (ret) {
perror("rsetsockopt RDMA_ROUTE");
goto close;
}
}
ret = rai ? rs_connect(rs, rai->ai_dst_addr, rai->ai_dst_len) :
rs_connect(rs, ai->ai_addr, ai->ai_addrlen);
if (ret && (errno != EINPROGRESS)) {
perror("rconnect");
goto close;
}
if (ret && (errno == EINPROGRESS)) {
fds.fd = rs;
fds.events = POLLOUT;
ret = do_poll(&fds, poll_timeout);
if (ret)
goto close;
len = sizeof err;
ret = rs_getsockopt(rs, SOL_SOCKET, SO_ERROR, &err, &len);
if (ret)
goto close;
if (err) {
ret = -1;
errno = err;
perror("async rconnect");
}
}
close:
if (ret)
rs_close(rs);
free:
if (rai)
rdma_freeaddrinfo(rai);
else
freeaddrinfo(ai);
return ret;
}
void fi_ibv_destroy_ep(struct rdma_addrinfo *rai, struct rdma_cm_id **id)
{
rdma_freeaddrinfo(rai);
rdma_destroy_ep(*id);
}
static int run(void)
{
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(server, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = 16;
attr.qp_context = id;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
mr = rdma_reg_msgs(id, recv_msg, 16);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
ret = rdma_post_recv(id, NULL, recv_msg, 16, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
ret = rdma_connect(id, NULL);
if (ret) {
printf("rdma_connect %d\n", errno);
return ret;
}
s = get_dtime();
ret = rdma_post_send(id, NULL, send_msg, 16, NULL, IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
e = get_dtime();
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
printf("time %f\n", e - s);
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
return 0;
}
int main(int argc, char **argv)
{
int op, ret;
while ((op = getopt(argc, argv, "s:b:c:C:S:t:p:mT")) != -1) {
switch (op) {
case 's':
dst_addr = optarg;
break;
case 'b':
src_addr = optarg;
break;
case 'c':
connections = atoi(optarg);
break;
case 'C':
message_count = atoi(optarg);
break;
case 'S':
message_size = atoi(optarg);
break;
case 't':
set_tos = 1;
tos = (uint8_t) strtoul(optarg, NULL, 0);
break;
case 'p':
port = optarg;
break;
case 'm':
migrate = 1;
break;
case 'T':
set_ts = 1;
break;
default:
printf("usage: %s\n", argv[0]);
printf("\t[-s server_address]\n");
printf("\t[-b bind_address]\n");
printf("\t[-c connections]\n");
printf("\t[-C message_count]\n");
printf("\t[-S message_size]\n");
printf("\t[-t type_of_service]\n");
printf("\t[-p port_number]\n");
printf("\t[-m(igrate)]\n");
printf("\t[-T(imestamping)]\n");
exit(1);
}
}
test.connects_left = connections;
test.channel = rdma_create_event_channel();
if (!test.channel) {
printf("failed to create event channel\n");
exit(1);
}
if (alloc_nodes())
exit(1);
if (dst_addr)
ret = run_client();
else
ret = run_server();
printf("test complete\n");
destroy_nodes();
rdma_destroy_event_channel(test.channel);
if (test.rai)
rdma_freeaddrinfo(test.rai);
printf("return status %d\n", ret);
return ret;
}
static int rc_test(void)
{
struct rdma_addrinfo hints, *res;
struct ibv_qp_init_attr attr;
struct ibv_wc wc;
int ret;
memset(&hints, 0, sizeof hints);
hints.ai_flags = RAI_PASSIVE;
hints.ai_port_space = RDMA_PS_TCP;
ret = rdma_getaddrinfo(NULL, port, &hints, &res);
if (ret) {
printf("rdma_getaddrinfo %d\n", errno);
return ret;
}
memset(&attr, 0, sizeof attr);
attr.cap.max_send_wr = attr.cap.max_recv_wr = 1;
attr.cap.max_send_sge = attr.cap.max_recv_sge = 1;
attr.cap.max_inline_data = sizeof send_msg;
attr.sq_sig_all = 1;
ret = rdma_create_ep(&listen_id, res, NULL, &attr);
rdma_freeaddrinfo(res);
if (ret) {
printf("rdma_create_ep %d\n", errno);
return ret;
}
ret = rdma_listen(listen_id, 0);
if (ret) {
printf("rdma_listen %d\n", errno);
return ret;
}
ret = rdma_get_request(listen_id, &id);
if (ret) {
printf("rdma_get_request %d\n", errno);
return ret;
}
mr = rdma_reg_msgs(id, recv_msg, sizeof recv_msg);
if (!mr) {
printf("rdma_reg_msgs %d\n", errno);
return ret;
}
ret = rdma_post_recv(id, NULL, recv_msg, sizeof recv_msg, mr);
if (ret) {
printf("rdma_post_recv %d\n", errno);
return ret;
}
ret = rdma_accept(id, NULL);
if (ret) {
printf("rdma_accept %d\n", errno);
return ret;
}
ret = rdma_get_recv_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_recv_comp %d\n", ret);
return ret;
}
ret = rdma_post_send(id, NULL, send_msg, sizeof send_msg, NULL, IBV_SEND_INLINE);
if (ret) {
printf("rdma_post_send %d\n", errno);
return ret;
}
ret = rdma_get_send_comp(id, &wc);
if (ret <= 0) {
printf("rdma_get_send_comp %d\n", ret);
return ret;
}
rdma_disconnect(id);
rdma_dereg_mr(mr);
rdma_destroy_ep(id);
rdma_destroy_ep(listen_id);
return 0;
}
