static int run_client(void)
{
struct rdma_addrinfo hints;
int i, ret, ret2;
printf("cmatose: starting client\n");
memset(&hints, 0, sizeof hints);
hints.ai_port_space = RDMA_PS_TCP;
ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai);
if (ret) {
perror("cmatose: getaddrinfo error");
return ret;
}
printf("cmatose: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id, test.rai->ai_src_addr,
test.rai->ai_dst_addr, 2000);
if (ret) {
perror("cmatose: failure getting addr");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto disc;
if (message_count) {
printf("receiving data transfers\n");
ret = poll_cqs(RECV_CQ_INDEX);
if (ret)
goto disc;
printf("sending replies\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i]);
if (ret)
goto disc;
}
printf("data transfers complete\n");
}
ret = 0;
if (migrate) {
ret = migrate_channel(NULL);
if (ret)
goto out;
}
disc:
ret2 = disconnect_events();
if (ret2)
ret = ret2;
out:
return ret;
}
static int run_server(void)
{
struct rdma_cm_id *listen_id;
int i, ret;
printf("udaddy: starting server\n");
ret = rdma_create_id(test.channel, &listen_id, &test, port_space);
if (ret) {
perror("udaddy: listen request failed");
return ret;
}
if (src_addr) {
ret = get_addr(src_addr, &test.src_in);
if (ret)
goto out;
} else
test.src_in.sin_family = PF_INET;
test.src_in.sin_port = port;
ret = rdma_bind_addr(listen_id, test.src_addr);
if (ret) {
perror("udaddy: bind address failed");
return ret;
}
ret = rdma_listen(listen_id, 0);
if (ret) {
perror("udaddy: failure trying to listen");
goto out;
}
connect_events();
if (message_count) {
printf("receiving data transfers\n");
ret = poll_cqs();
if (ret)
goto out;
printf("sending replies\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i], IBV_SEND_SIGNALED);
if (ret)
goto out;
}
ret = poll_cqs();
if (ret)
goto out;
printf("data transfers complete\n");
}
out:
rdma_destroy_id(listen_id);
return ret;
}
static int run_client(void)
{
int i, ret;
printf("udaddy: starting client\n");
if (src_addr) {
ret = get_addr(src_addr, &test.src_in);
if (ret)
return ret;
}
ret = get_addr(dst_addr, &test.dst_in);
if (ret)
return ret;
test.dst_in.sin_port = port;
printf("udaddy: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id,
src_addr ? test.src_addr : NULL,
test.dst_addr, 2000);
if (ret) {
perror("udaddy: failure getting addr");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto out;
if (message_count) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i], 0);
if (ret)
goto out;
}
printf("receiving data transfers\n");
ret = poll_cqs();
if (ret)
goto out;
printf("data transfers complete\n");
}
out:
return ret;
}
static int run_client(void)
{
int i, ret, ret2;
printf("cmatose: starting client\n");
printf("cmatose: connecting\n");
for (i = 0; i < connections; i++) {
ret = fi_connect(nodes[i].ep, info->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
connects_left--;
return ret;
}
}
ret = connect_events();
if (ret)
goto disc;
if (hints->tx_attr->size) {
printf("receiving data transfers\n");
ret = poll_cqs(RECV_CQ_INDEX);
if (ret)
goto disc;
printf("sending replies\n");
for (i = 0; i < connections; i++) {
ret = post_sends(nodes + i);
if (ret)
goto disc;
}
printf("completing sends\n");
ret = poll_cqs(SEND_CQ_INDEX);
if (ret)
goto disc;
printf("data transfers complete\n");
}
ret = 0;
disc:
ret2 = shutdown_events();
printf("disconnected\n");
if (ret2)
ret = ret2;
return ret;
}
static int run_client(void)
{
int i, ret;
printf("udaddy: starting client\n");
ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai);
if (ret) {
perror("udaddy: getaddrinfo error");
return ret;
}
printf("udaddy: connecting\n");
for (i = 0; i < connections; i++) {
ret = rdma_resolve_addr(test.nodes[i].cma_id, test.rai->ai_src_addr,
test.rai->ai_dst_addr, 2000);
if (ret) {
perror("udaddy: failure getting addr");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto out;
if (message_count) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i], 0);
if (ret)
goto out;
}
printf("receiving data transfers\n");
ret = poll_cqs();
if (ret)
goto out;
printf("data transfers complete\n");
}
out:
return ret;
}
static int run_server(void)
{
struct rdma_cm_id *listen_id;
int i, ret;
printf("cmatose: starting server\n");
ret = rdma_create_id(test.channel, &listen_id, &test, hints.ai_port_space);
if (ret) {
perror("cmatose: listen request failed");
return ret;
}
ret = get_rdma_addr(src_addr, dst_addr, port, &hints, &test.rai);
if (ret) {
printf("cmatose: getrdmaaddr error: %s\n", gai_strerror(ret));
goto out;
}
ret = rdma_bind_addr(listen_id, test.rai->ai_src_addr);
if (ret) {
perror("cmatose: bind address failed");
goto out;
}
ret = rdma_listen(listen_id, 0);
if (ret) {
perror("cmatose: failure trying to listen");
goto out;
}
ret = connect_events();
if (ret)
goto out;
if (message_count) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i]);
if (ret)
goto out;
}
printf("completing sends\n");
ret = poll_cqs(SEND_CQ_INDEX);
if (ret)
goto out;
printf("receiving data transfers\n");
ret = poll_cqs(RECV_CQ_INDEX);
if (ret)
goto out;
printf("data transfers complete\n");
}
if (migrate) {
ret = migrate_channel(listen_id);
if (ret)
goto out;
}
printf("cmatose: disconnecting\n");
for (i = 0; i < connections; i++) {
if (!test.nodes[i].connected)
continue;
test.nodes[i].connected = 0;
rdma_disconnect(test.nodes[i].cma_id);
}
ret = disconnect_events();
printf("disconnected\n");
out:
rdma_destroy_id(listen_id);
return ret;
}
static int run_server(void)
{
int i, ret;
printf("cmatose: starting server\n");
ret = fi_passive_ep(fabric, info, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
return ret;
}
ret = fi_pep_bind(pep, &eq->fid, 0);
if (ret) {
FT_PRINTERR("fi_ep_bind", ret);
goto out;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto out;
}
ret = connect_events();
if (ret)
goto out;
if (hints->tx_attr->size) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&nodes[i]);
if (ret)
goto out;
}
printf("completing sends\n");
ret = poll_cqs(SEND_CQ_INDEX);
if (ret)
goto out;
printf("receiving data transfers\n");
ret = poll_cqs(RECV_CQ_INDEX);
if (ret)
goto out;
printf("data transfers complete\n");
}
printf("cmatose: disconnecting\n");
for (i = 0; i < connections; i++) {
if (!nodes[i].connected)
continue;
nodes[i].connected = 0;
fi_shutdown(nodes[i].ep, 0);
}
ret = shutdown_events();
printf("disconnected\n");
out:
fi_close(&pep->fid);
return ret;
}
static int run(void)
{
int i, ret;
printf("mckey: starting %s\n", is_sender ? "client" : "server");
if (src_addr) {
ret = get_addr(src_addr, (struct sockaddr *) &test.src_in);
if (ret)
return ret;
}
ret = get_addr(dst_addr, (struct sockaddr *) &test.dst_in);
if (ret)
return ret;
printf("mckey: joining\n");
for (i = 0; i < connections; i++) {
if (src_addr) {
ret = rdma_bind_addr(test.nodes[i].cma_id,
test.src_addr);
if (ret) {
perror("mckey: addr bind failure");
connect_error();
return ret;
}
}
if (unmapped_addr)
ret = addr_handler(&test.nodes[i]);
else
ret = rdma_resolve_addr(test.nodes[i].cma_id,
test.src_addr, test.dst_addr,
2000);
if (ret) {
perror("mckey: resolve addr failure");
connect_error();
return ret;
}
}
ret = connect_events();
if (ret)
goto out;
pthread_create(&test.cmathread, NULL, cma_thread, NULL);
/*
* Pause to give SM chance to configure switches. We don't want to
* handle reliability issue in this simple test program.
*/
sleep(3);
if (message_count) {
if (is_sender) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&test.nodes[i], 0);
if (ret)
goto out;
}
} else {
printf("receiving data transfers\n");
ret = poll_cqs();
if (ret)
goto out;
}
printf("data transfers complete\n");
}
out:
for (i = 0; i < connections; i++) {
ret = rdma_leave_multicast(test.nodes[i].cma_id,
test.dst_addr);
if (ret)
perror("mckey: failure leaving");
}
return ret;
}
static int poll_send_cqs(void)
{
struct ibv_wc wc[POLL_BATCH];
int done, i, ret, poll_ret;
long total_counter = 0;
struct timespec ts0;
ts0.tv_sec = -1;
ts0.tv_nsec = -1;
struct timespec tslow, remp;
tslow.tv_sec = 0;
tslow.tv_nsec = 1;
if(!is_sender) {
printf(" called poll_send_cqs() with receiving side \n");
return -1;
}
if (print_base == -1) print_base = message_buffer * message_batch;
for (i = 0; i < connections; i++) {
if (!test.nodes[i].connected)
continue;
do {
for (done = 0; done < message_buffer * message_batch; done += poll_ret) {
poll_ret = ibv_poll_cq(test.nodes[i].cq, POLL_BATCH, wc);
total_counter += poll_ret;
if (poll_ret < 0) {
printf("rxe_send_mc: failed polling CQ: %d\n", poll_ret);
return poll_ret;
}
else if (poll_ret > 0) {
if(ts0.tv_sec == -1 && ts0.tv_nsec == -1) {
clock_gettime(CLOCK_REALTIME, &ts0);
}
if(wc->opcode == IBV_WC_SEND && wc->status == IBV_WC_SUCCESS ) {
/* if(done % 20 == 0) {
if(nanosleep(&tslow,&remp)) {
printf("sleep error\n");
return -1;
}
}
*/
ret = post_sends(&test.nodes[i],IBV_SEND_SIGNALED,poll_ret);
if (ret < 0) {
printf("rxe_send_mc: failed post sends after polling CQ: %d\n", ret);
return ret;
}
if(done != 0 && done % print_base == 0) {
print_line(&ts0, done, print_base);
}
}
}
}
printf ("Have the last message %u of %lu \n", done, total_counter);
// fprintf (pFile,"Have the last message %u of %lu \n", done, total_counter);
} while (message_batch >= 1000 && if_continue());
}
return 0;
}
