int cm_client_start_connect(void)
{
int ret;
struct sockaddr_in loc_sa;
cm_local_ip(&loc_sa);
cli_hints = fi_allocinfo();
cli_hints->fabric_attr->name = strdup("gni");
cli_hints->caps = GNIX_EP_PRIMARY_CAPS;
cli_hints->ep_attr->type = FI_EP_MSG;
cli_hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
ret = fi_getinfo(fi_version(), inet_ntoa(loc_sa.sin_addr),
DEF_PORT, 0, cli_hints, &cli_fi);
cr_assert(!ret);
ret = fi_fabric(cli_fi->fabric_attr, &cli_fab, NULL);
cr_assert(!ret);
ret = fi_eq_open(cli_fab, &eq_attr, &cli_eq, NULL);
cr_assert(!ret);
ret = fi_domain(cli_fab, cli_fi, &cli_dom, NULL);
cr_assert(!ret);
ret = fi_endpoint(cli_dom, cli_fi, &cli_ep, NULL);
cr_assert(!ret, "fi_endpoint");
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
ret = fi_cq_open(cli_dom, &cq_attr, &cli_cq, &cli_cq);
cr_assert(!ret);
ret = fi_ep_bind(cli_ep, &cli_eq->fid, 0);
cr_assert(!ret);
ret = fi_ep_bind(cli_ep, &cli_cq->fid, FI_SEND | FI_RECV);
cr_assert(!ret);
ret = fi_enable(cli_ep);
cr_assert(!ret);
ret = fi_connect(cli_ep, cli_fi->dest_addr, cli_cm_in_data,
GNIX_CM_DATA_MAX_SIZE+1);
cr_assert(ret == -FI_EINVAL);
ret = fi_connect(cli_ep, cli_fi->dest_addr, cli_cm_in_data,
strlen(cli_cm_in_data));
cr_assert(!ret);
dbg_printf("Client connect complete.\n");
return 0;
}
static int hook_connect(struct fid_ep *ep, const void *addr,
const void *param, size_t paramlen)
{
struct hook_ep *myep = container_of(ep, struct hook_ep, ep);
return fi_connect(myep->hep, addr, param, paramlen);
}
static int client_connect(size_t paramlen)
{
ft_fill_buf(cm_data, paramlen);
/* Connect to server */
return fi_connect(ep, fi->dest_addr, cm_data, paramlen);
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(buf);
if (ret)
return ret;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
return -FI_EOTHER;
}
return 0;
}
static int run_client(void)
{
int i, ret, ret2;
printf("cmatose: starting client\n");
printf("cmatose: connecting\n");
for (i = 0; i < connections; i++) {
ret = fi_connect(nodes[i].ep, info->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
connects_left--;
return ret;
}
}
ret = connect_events();
if (ret)
goto disc;
if (hints->tx_attr->size) {
printf("receiving data transfers\n");
ret = poll_cqs(RECV_CQ_INDEX);
if (ret)
goto disc;
printf("sending replies\n");
for (i = 0; i < connections; i++) {
ret = post_sends(nodes + i);
if (ret)
goto disc;
}
printf("completing sends\n");
ret = poll_cqs(SEND_CQ_INDEX);
if (ret)
goto disc;
printf("data transfers complete\n");
}
ret = 0;
disc:
ret2 = shutdown_events();
printf("disconnected\n");
if (ret2)
ret = ret2;
return ret;
}
int ft_client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = ft_alloc_active_res(fi);
if (ret)
return ret;
ret = ft_init_ep();
if (ret)
return ret;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
ret = (int) rd;
return ret;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
return ret;
}
return 0;
}
static int client_connect(void)
{
int ret;
socklen_t addrlen;
struct sockaddr *sin;
ret = common_setup();
if (ret != 0)
goto err;
ret = getaddr(dst_addr, port, (struct sockaddr **) &sin,
(socklen_t *) &addrlen);
if (ret != 0)
goto err;
ret = fi_connect(ep, sin, NULL, 0);
if (ret) {
printf("fi_connect %s\n", fi_strerror(-ret));
goto err;
}
// send initial message to server
ret = send_xfer(4);
if (ret != 0)
goto err;
// wait for reply to know server is ready
ret = recv_xfer(4);
if (ret != 0)
goto err;
return 0;
err:
free_ep_res();
fi_close(&av->fid);
fi_close(&ep->fid);
fi_close(&dom->fid);
fi_close(&fab->fid);
return ret;
}
static int ft_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = fi_connect(ep, fabric_info->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = ft_get_event(&event, &entry, sizeof entry,
FI_CONNECTED, sizeof entry);
if (rd < 0)
return (int) rd;
return 0;
}
/*
* rpmem_fip_connect -- connect to remote peer
*/
int
rpmem_fip_connect(struct rpmem_fip *fip)
{
int ret;
struct fi_eq_cm_entry entry;
ret = rpmem_fip_init_cq(fip);
if (ret)
goto err_init_cq;
ret = rpmem_fip_init_ep(fip);
if (ret)
goto err_init_ep;
ret = fip->ops->lanes_post(fip);
if (ret)
goto err_lanes_post;
ret = fi_connect(fip->ep, fip->fi->dest_addr, NULL, 0);
if (ret) {
RPMEM_FI_ERR(ret, "initiating connection request");
goto err_fi_connect;
}
ret = rpmem_fip_read_eq(fip->eq, &entry, FI_CONNECTED,
&fip->ep->fid, -1);
if (ret)
goto err_fi_eq_read;
return 0;
err_fi_eq_read:
err_fi_connect:
err_lanes_post:
rpmem_fip_fini_ep(fip);
err_init_ep:
rpmem_fip_fini_cq(fip);
err_init_cq:
return ret;
}
int client_connect(struct fi_info *prov, simple_context_t *ctx)
{
struct fi_eq_attr eq_attr = { 0 };
struct fi_cq_attr cq_attr = { 0 };
struct sockaddr_in addr = { 0 };
int ret;
print_trace("in\n");
connected = 0;
ret = fi_fabric(prov->fabric_attr, &ctx->fabric, NULL);
if (ret) {
print_err("fi_fabric returned %d\n", ret);
ctx->fabric = NULL;
return ret;
}
ret = fi_domain(ctx->fabric, prov, &ctx->domain, NULL);
if (ret) {
print_err("fi_fdomain returned %d\n", ret);
ctx->domain = NULL;
return ret;
}
/* set QP WR depth */
prov->ep_attr->tx_ctx_cnt = (size_t) (post_depth + 1);
prov->ep_attr->rx_ctx_cnt = (size_t) (post_depth + 1);
/* set ScatterGather max depth */
prov->tx_attr->iov_limit = 1;
prov->rx_attr->iov_limit = 1;
prov->tx_attr->inject_size = 0; /* no INLINE support */
ret = fi_endpoint(ctx->domain, prov, &ctx->ep, CONTEXT);
if (ret) {
print_err("fi_endpoint returned %d\n", ret);
ctx->ep = NULL;
return ret;
}
eq_attr.wait_obj = FI_WAIT_NONE;
ret = fi_eq_open(ctx->fabric, &eq_attr, &ctx->eq, NULL);
if (ret) {
print_err("fi_eq_open returned %d\n", ret);
ctx->eq = NULL;
return ret;
}
cq_attr.size = post_depth * 4;
cq_attr.flags = FI_SEND;
cq_attr.format = FI_CQ_FORMAT_MSG;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.wait_cond = FI_CQ_COND_NONE;
ret = fi_cq_open(ctx->domain, &cq_attr, &ctx->scq, NULL);
if (ret) {
print_err("fi_cq_open returned %d\n", ret);
ctx->scq = NULL;
return ret;
}
cq_attr.flags = FI_RECV;
ret = fi_cq_open(ctx->domain, &cq_attr, &ctx->rcq, NULL);
if (ret) {
print_err("fi_cq_open returned %d\n", ret);
ctx->rcq = NULL;
return ret;
}
ret = fi_ep_bind(ctx->ep, &ctx->eq->fid, 0);
if (ret) {
print_err("fi_ep_bind returned %d\n", ret);
return ret;
}
ret = fi_ep_bind(ctx->ep, &ctx->scq->fid, FI_SEND);
if (ret) {
print_err("fi_ep_bind returned %d\n", ret);
return ret;
}
ret = fi_ep_bind(ctx->ep, &ctx->rcq->fid, FI_RECV);
if (ret) {
print_err("fi_ep_bind returned %d\n", ret);
return ret;
}
ret = fi_enable(ctx->ep);
if (ret) {
print_err("fi_enable returned %d\n", ret);
return ret;
}
addr.sin_family = AF_INET;
addr.sin_port = htons(TEST_PORT);
ret = in4_pton(svr_ipaddr, strlen(svr_ipaddr),
(u8 *)&addr.sin_addr.s_addr, '\0', NULL);
if (ret != 1) {
print_err("Err converting target server IP address '%s'?\n",
svr_ipaddr);
return -EINVAL;
}
ret = fi_connect(ctx->ep, &addr, PRIVATE_DATA, sizeof(PRIVATE_DATA));
if (ret) {
print_err("fi_connect returned %d\n", ret);
return ret;
}
connected = 1;
return 0;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err0;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_cm_res();
if (ret)
goto err4;
ret = alloc_ep_res(fi);
if (ret)
goto err5;
ret = bind_ep_res();
if (ret)
goto err6;
/* Connect to server */
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
goto err6;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "connect");
ret = (int) rd;
goto err6;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err6;
}
fi_freeinfo(fi);
return 0;
err6:
free_ep_res();
err5:
fi_close(&cmeq->fid);
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int i, ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
ret = get_dupinfo();
if (ret)
return ret;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = alloc_ep();
if (ret)
return ret;
ret = bind_ep_array_res();
if (ret)
return ret;
for (i = 0; i < ep_cnt; i++) {
ret = fi_connect(ep_array[i], fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
ret = (int) rd;
return ret;
}
if (event != FI_CONNECTED || entry.fid != &ep_array[i]->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
return ret;
}
}
/* Post recv */
if (rx_shared_ctx)
ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
else
ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
if (ret)
return ret;
return 0;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
if (src_addr) {
ret = getaddr(src_addr, NULL, (struct sockaddr **) &hints.src_addr,
(socklen_t *) &hints.src_addrlen);
if (ret)
printf("source address error %s\n", gai_strerror(ret));
}
ret = fi_getinfo(FI_VERSION(1, 0), dst_addr, port, 0, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
goto err0;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
printf("fi_domain %s %s\n", fi_strerror(-ret),
fi->domain_attr->name);
goto err2;
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto err3;
}
ret = alloc_ep_res(fi);
if (ret)
goto err4;
ret = bind_ep_res();
if (ret)
goto err5;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
printf("fi_connect %s\n", fi_strerror(-ret));
goto err5;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
return (int) rd;
}
if (event != FI_COMPLETE || entry.fid != &ep->fid) {
printf("Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err1;
}
if (hints.src_addr)
free(hints.src_addr);
fi_freeinfo(fi);
return 0;
err5:
free_ep_res();
err4:
fi_close(&ep->fid);
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
if (hints.src_addr)
free(hints.src_addr);
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err0;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
goto err3;
}
ret = alloc_ep_res(fi);
if (ret)
goto err3;
ret = bind_ep_res();
if (ret)
goto err5;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
goto err5;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err1;
}
fi_freeinfo(fi);
return 0;
err5:
free_ep_res();
err3:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
return ret;
}
static int pp_connect_ctx(struct pingpong_context *ctx)
{
struct fi_eq_cm_entry entry;
uint32_t event;
int rc = 0;
ssize_t rd;
/* Open domain */
rc = fi_domain(ctx->fabric, ctx->info, &ctx->dom, NULL);
if (rc) {
FT_PRINTERR("fi_fdomain", rc);
return 1;
}
if (pp_eq_create(ctx)) {
fprintf(stderr, "Unable to create event queue\n");
return 1;
}
rc = fi_mr_reg(ctx->dom, ctx->buf, ctx->size, FI_SEND | FI_RECV, 0, 0, 0, &ctx->mr, NULL);
if (rc) {
FT_PRINTERR("fi_mr_reg", rc);
return 1;
}
/* Open endpoint */
rc = fi_endpoint(ctx->dom, ctx->info, &ctx->ep, NULL);
if (rc) {
FT_PRINTERR("fi_endpoint", rc);
return 1;
}
/* Create event queue */
if (pp_cq_create(ctx)) {
fprintf(stderr, "Unable to create event queue\n");
return 1;
}
/* Bind eq to ep */
rc = fi_ep_bind(ctx->ep, &ctx->cq->fid, FI_SEND | FI_RECV);
if (rc) {
FT_PRINTERR("fi_ep_bind", rc);
return 1;
}
rc = fi_ep_bind(ctx->ep, &ctx->eq->fid, 0);
if (rc) {
FT_PRINTERR("fi_ep_bind", rc);
return 1;
}
rc = fi_enable(ctx->ep);
if (rc) {
FT_PRINTERR("fi_enable", rc);
return EXIT_FAILURE;
}
ctx->routs = pp_post_recv(ctx, ctx->rx_depth);
if (ctx->routs < ctx->rx_depth) {
FT_ERR("Couldn't post receive (%d)", ctx->routs);
return 1;
}
printf("Connecting to server\n");
rc = fi_connect(ctx->ep, ctx->info->dest_addr, NULL, 0);
if (rc) {
FT_PRINTERR("fi_connect", rc);
return 1;
}
rd = fi_eq_sread(ctx->eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, ctx->eq, "fi_eq_sread", "connect");
return 1;
}
if (event != FI_CONNECTED) {
fprintf(stderr, "Unexpected CM event %d\n", event);
return 1;
}
printf("Connection successful\n");
return 0;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = check_address(&pep->fid, "fi_endpoint (pep)");
if (ret)
return ret;
assert(fi->handle == &pep->fid);
ret = ft_alloc_active_res(fi);
if (ret)
return ret;
/* Close the passive endpoint that we "stole" the source address
* from */
FT_CLOSE_FID(pep);
ret = check_address(&ep->fid, "fi_endpoint (ep)");
if (ret)
return ret;
ret = ft_init_ep();
if (ret)
return ret;
/* Connect to server */
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)\n", event, entry.fid, ep);
return -FI_EOTHER;
}
ret = check_address(&ep->fid, "connect");
if (ret) {
return ret;
}
return 0;
}
static void test_connect_with_accept_blocking_on_eq_fq_CLIENT(void)
{
int ret;
printf("CLIENT running\n");
// Get the server's node (IP addr) and service (port)
MPI_Recv(ofi_node, sizeof(ofi_node) - 1, MPI_CHAR,
0, 101, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Recv(ofi_service, sizeof(ofi_service) - 1, MPI_CHAR,
0, 102, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
printf("CLIENT received via MPI: %s / %s\n", ofi_node, ofi_service);
//setup_ofi(ofi_node, ofi_service);
setup_ofi(NULL, NULL, 0);
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
inet_aton(ofi_node, &sin.sin_addr);
sin.sin_port = htons(atoi(ofi_service));
printf("CLIENT translated: %s\n", addrstr(&sin));
setup_ofi_active(fidev.info, &ficonn.ep);
// Print server addr
printf("CLIENT connecting to %s\n", addrstr(&sin));
// Connect!
printf("Client connecting...\n");
ret = fi_connect(ficonn.ep,
//fidev.info->dest_addr,
&sin,
(void*) client_data, sizeof(client_data));
if (ret < 0) {
error("fi_connect failed");
}
#if WANT_FDS
// Now wait for the listen to complete
int nevents;
#define NEVENTS 32
struct epoll_event events[NEVENTS];
int timeout = 10000;
while (1) {
printf("CLIENT blocking on epoll\n");
nevents = epoll_wait(epoll_fd, events, NEVENTS, timeout);
if (nevents < 0) {
if (errno != EINTR) {
error("client epoll wait failed");
} else {
continue;
}
} else {
printf("CLIENT successfully woke up from epoll! %d events\n", nevents);
for (int i = 0; i < nevents; ++i) {
if (events[i].data.u32 != 2222) {
error("CLIENT unexpected epoll return type");
}
}
// If we got the expected event, then go read from the EQ
break;
}
}
#endif
// Wait for FI_CONNECTED event
uint32_t event;
uint8_t *entry_buffer;
size_t expected_len = sizeof(struct fi_eq_cm_entry) +
sizeof(client_data);
entry_buffer = (uint8_t*) calloc(1, expected_len);
if (NULL == entry_buffer) {
error("calloc failed");
}
struct fi_eq_cm_entry *entry = (struct fi_eq_cm_entry*) entry_buffer;
while (1) {
printf("CLIENT waiting for FI_CONNECTED\n");
#if WANT_FDS
ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0);
#else
ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0);
#endif
if (-FI_EAVAIL == ret) {
fprintf(stderr, "client fi_eq_sread failed because there's something in the error queue\n");
char buffer[2048];
struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer;
ret = fi_eq_readerr(fidev.eq, err_entry, 0);
fprintf(stderr, "error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno));
error("sad panda");
} else if (ret == -EAGAIN) {
fprintf(stderr, "CLIENT fi_eq_sread fail got -EAGAIN... trying again...\n");
sleep(1);
continue;
} else if (ret < 0) {
fprintf(stderr, "SERVER fi_eq_sread fail: %s, ret = %d)\n", fi_strerror(-ret), ret);
error("client fi_eq_sread failed for some random reason");
} else if (event != FI_CONNECTED) {
error("client got some unexpected event");
} else if (ret != expected_len) {
error("client got wrong length back from fi_eq_sread");
}
uint32_t *d = (uint32_t*) entry->data;
for (int i = 0; i < (sizeof(server_data) / sizeof(uint32_t)); ++i) {
if (d[i] != server_data[i]) {
printf("CLIENT got wrong CM client data: d[%d]=%d, should be %d\n",
i, d[i], server_data[i]);
}
}
printf("client got FI_CONNECTED, correct size, and correct data -- yay!\n");
break;
}
printf("CLIENT connecting -- waiting for server before sending\n");
MPI_Barrier(MPI_COMM_WORLD);
sleep(1);
int msg[4] = { 99, 100, 101, 102 };
int len = sizeof(msg);
printf("CLIENT sending len of %d\n", len);
struct fid_mr no_mr;
struct fid_mr *mr;
void *send_context = (void*) 0x42;
#if 0
fi_mr_reg(fidev.domain, msg, len, FI_SEND | FI_RECV,
0, (uint64_t)(uintptr_t) msg, 0, &mr, NULL);
#else
// Try using no mr, like fi_msg_pingpong...
memset(&no_mr, 0, sizeof(no_mr));
mr = &no_mr;
#endif
ret = fi_send(ficonn.ep, msg, len,
fi_mr_desc(mr), 0, send_context);
if (ret < 0) {
printf("fi_Send failed! %d, %s\n", ret, fi_strerror(-ret));
MPI_Abort(MPI_COMM_WORLD, 37);
}
// Wait for send completion
struct fi_cq_entry cqe;
while (1) {
ret = fi_cq_sread(ficonn.cq, &cqe, 1, 0, -1);
if (cqe.op_context == send_context) {
printf("CLIENT send completed\n");
break;
} else {
printf("CLIENT got some other completion... continuing\n");
}
}
printf("CLIENT sent -- waiting for server before teardown\n");
MPI_Barrier(MPI_COMM_WORLD);
printf("CLIENT tearing down\n");
fi_close(&(mr->fid));
teardown_ofi();
}
