int cm_server_accept(void)
{
uint32_t event;
ssize_t rd;
int ret;
struct fi_eq_cm_entry *entry;
void *eqe_buf[EQE_SIZE] = {0};
rd = fi_eq_sread(srv_eq, &event, &eqe_buf, EQE_SIZE, -1, 0);
cr_assert(rd == (sizeof(*entry) + strlen(cli_cm_in_data)));
cr_assert(event == FI_CONNREQ);
entry = (struct fi_eq_cm_entry *)eqe_buf;
cr_assert(!memcmp(cli_cm_in_data, entry->data,
strlen(cli_cm_in_data)));
ret = fi_domain(srv_fab, entry->info, &srv_dom, NULL);
cr_assert(!ret);
ret = fi_endpoint(srv_dom, entry->info, &srv_ep, NULL);
cr_assert(!ret, "fi_endpoint");
fi_freeinfo(entry->info);
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
ret = fi_cq_open(srv_dom, &cq_attr, &srv_cq, &srv_cq);
cr_assert(!ret);
ret = fi_ep_bind(srv_ep, &srv_eq->fid, 0);
cr_assert(!ret);
ret = fi_ep_bind(srv_ep, &srv_cq->fid, FI_SEND | FI_RECV);
cr_assert(!ret);
ret = fi_enable(srv_ep);
cr_assert(!ret);
ret = fi_accept(srv_ep, srv_cm_in_data, GNIX_CM_DATA_MAX_SIZE+1);
cr_assert(ret == -FI_EINVAL);
ret = fi_accept(srv_ep, srv_cm_in_data, strlen(srv_cm_in_data));
cr_assert(!ret);
dbg_printf("Server accept complete.\n");
return 0;
}
static int connreq_handler(struct fi_info *info)
{
struct cma_node *node;
int ret;
if (conn_index == connections) {
ret = -ENOMEM;
goto err1;
}
node = &nodes[conn_index++];
ret = init_node(node, info);
if (ret)
goto err2;
ret = fi_accept(node->ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err2;
}
return 0;
err2:
connects_left--;
err1:
printf("cmatose: failing connection request\n");
fi_reject(pep, info->handle, NULL, 0);
return ret;
}
int ft_server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "listen");
return (int) rd;
}
fi = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = ft_alloc_active_res(fi);
if (ret)
goto err;
ret = ft_init_ep();
if (ret)
goto err;
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "accept");
ret = (int) rd;
goto err;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
return 0;
err:
fi_reject(pep, fi->handle, NULL, 0);
return ret;
}
static int ft_accept(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
rd = ft_get_event(&event, &entry, sizeof entry,
FI_CONNREQ, sizeof entry);
if (rd < 0)
return (int) rd;
fabric_info = entry.info;
ret = ft_open_active();
if (ret)
return ret;
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
return ret;
}
rd = ft_get_event(&event, &entry, sizeof entry,
FI_CONNECTED, sizeof entry);
if (rd < 0)
return (int) rd;
return 0;
}
/*
* rpmemd_fip_accept -- accept a single connection request
*
* XXX
*
* We probably need some timeouts for connection related events.
*/
int
rpmemd_fip_accept(struct rpmemd_fip *fip)
{
int ret;
struct fi_eq_cm_entry entry;
ret = rpmem_fip_read_eq(fip->eq, &entry,
FI_CONNREQ, &fip->pep->fid, -1);
if (ret)
goto err_event_connreq;
ret = rpmemd_fip_init_cq(fip);
if (ret)
goto err_init_cq;
ret = rpmemd_fip_init_ep(fip, entry.info);
if (ret)
goto err_init_ep;
ret = fip->ops->post(fip);
if (ret)
goto err_post;
ret = fi_accept(fip->ep, NULL, 0);
if (ret) {
RPMEMD_FI_ERR(ret, "accepting connection request");
goto err_accept;
}
ret = rpmem_fip_read_eq(fip->eq, &entry,
FI_CONNECTED, &fip->ep->fid, -1);
if (ret)
goto err_event_connected;
return 0;
err_event_connected:
err_accept:
err_post:
rpmemd_fip_fini_ep(fip);
err_init_ep:
rpmemd_fip_fini_cq(fip);
err_init_cq:
err_event_connreq:
return -1;
}
void Connection::on_connect_request(struct fi_eq_cm_entry* event,
struct fid_domain* pd,
struct fid_cq* cq) {
int err = fi_endpoint(pd, event->info, &ep_, this);
if (err) {
L_(fatal) << "fi_endpoint failed: " << err << "=" << fi_strerror(-err);
throw LibfabricException("fi_endpoint failed");
}
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
err = fi_ep_bind(ep_, (::fid_t)eq_, 0);
if (err) {
L_(fatal) << "fi_ep_bind failed to eq: " << err << "=" << fi_strerror(-err);
throw LibfabricException("fi_ep_bind failed to eq");
}
err = fi_ep_bind(ep_, (fid_t)cq, FI_SEND | FI_RECV | FI_SELECTIVE_COMPLETION);
if (err) {
L_(fatal) << "fi_ep_bind failed to cq: " << err << "=" << fi_strerror(-err);
throw LibfabricException("fi_ep_bind failed to cq");
}
#pragma GCC diagnostic pop
// setup(pd);
setup_mr(pd);
auto private_data = get_private_data();
assert(private_data->size() <= 255);
err = fi_enable(ep_);
if (err) {
L_(fatal) << "fi_enable failed: " << err << "=" << fi_strerror(-err);
throw LibfabricException("fi_enable failed");
}
// accept_connect_request();
err = fi_accept(ep_, private_data->data(), private_data->size());
if (err) {
L_(fatal) << "fi_accept failed: " << err << "=" << fi_strerror(-err);
throw LibfabricException("fi_accept failed");
}
// setup(pd);
setup();
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
/* Wait for connection request from client */
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
FT_ERR("Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
ret = fi_domain(fabric, info, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = ft_alloc_active_res(info);
if (ret)
goto err;
ret = ft_init_ep();
if (ret)
goto err;
/* Accept the incoming connection. Also transitions endpoint to active state */
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
goto err;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)\n", event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
ret = check_address(&ep->fid, "accept");
if (ret) {
goto err;
}
fi_freeinfo(info);
return 0;
err:
fi_reject(pep, info->handle, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int hook_accept(struct fid_ep *ep, const void *param, size_t paramlen)
{
struct hook_ep *myep = container_of(ep, struct hook_ep, ep);
return fi_accept(myep->hep, param, paramlen);
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
/* Wait for connection request from client */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "listen");
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err1;
ret = bind_ep_res();
if (ret)
goto err3;
/* Accept the incoming connection. Also transitions endpoint to active state */
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err3;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "accept");
ret = (int) rd;
goto err3;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err1:
fi_reject(pep, info->handle, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int server_accept(size_t paramlen)
{
uint32_t event;
int ret;
ret = server_listen(paramlen);
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = ft_alloc_active_res(fi);
if (ret) {
FT_PRINTERR("alloc_active_res", ret);
goto err;
}
ret = ft_init_ep();
if (ret) {
FT_PRINTERR("init_ep", ret);
goto err;
}
/* Data will apppear on accept event on remote end. */
ft_fill_buf(cm_data, paramlen);
/* Accept the incoming connection. Also transitions endpoint to active
* state.
*/
ret = fi_accept(ep, cm_data, paramlen);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
/* Local FI_CONNECTED event does not have data associated. */
memset(entry, 0, sizeof(*entry));
ret = fi_eq_sread(eq, &event, entry, sizeof(*entry), -1, 0);
if (ret != sizeof(*entry)) {
FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "accept");
goto err;
}
if (event != FI_CONNECTED || entry->fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)", event,
entry->fid, ep);
ret = -FI_EOTHER;
goto err;
}
fi_shutdown(ep, 0);
ret = read_shutdown_event();
if (ret)
goto err;
FT_CLOSE_FID(ep);
FT_CLOSE_FID(rxcq);
FT_CLOSE_FID(txcq);
FT_CLOSE_FID(rxcntr);
FT_CLOSE_FID(txcntr);
FT_CLOSE_FID(av);
FT_CLOSE_FID(domain);
return 0;
err:
fi_reject(pep, fi->handle, NULL, 0);
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret, k;
int num_conn_reqs = 0, num_connected = 0;
struct ep_info *ep_state_array = NULL;
ep_array = calloc(ep_cnt, sizeof(*ep_array));
if (!ep_array)
return -FI_ENOMEM;
ep_state_array = calloc(ep_cnt, sizeof(*ep_state_array));
if (!ep_state_array)
return -FI_ENOMEM;
while (num_connected != ep_cnt) {
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "cm-event");
ret = (int) rd;
goto err;
}
switch(event) {
case FI_CONNREQ:
if (num_conn_reqs == ep_cnt) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
fi = ep_state_array[num_conn_reqs].fi = entry.info;
ep_state_array[num_conn_reqs].state = FT_EP_CONNECT_RCVD;
if (num_conn_reqs == 0) {
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = alloc_ep_res(fi);
if (ret)
goto err;
}
ret = fi_endpoint(domain, fi, &ep_array[num_conn_reqs], NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
goto err;
}
ep_state_array[num_conn_reqs].ep = ep_array[num_conn_reqs];
ret = bind_ep_res(ep_array[num_conn_reqs]);
if (ret)
goto err;
ret = fi_accept(ep_array[num_conn_reqs], NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
ep_state_array[num_conn_reqs].state = FT_EP_CONNECTING;
num_conn_reqs++;
break;
case FI_CONNECTED:
if (num_conn_reqs <= num_connected) {
ret = -FI_EOTHER;
goto err;
}
for (k = 0; k < num_conn_reqs; k++) {
if (ep_state_array[k].state != FT_EP_CONNECTING)
continue;
if (&ep_state_array[k].ep->fid == entry.fid) {
ep_state_array[k].state = FT_EP_CONNECTED;
num_connected++;
if (num_connected != ep_cnt)
fi_freeinfo(ep_state_array[k].fi);
break;
}
}
if (k == num_conn_reqs) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
break;
default:
ret = -FI_EOTHER;
goto err;
}
}
/* Post recv */
if (rx_shared_ctx)
ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
else
ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
if (ret)
goto err;
free(ep_state_array);
return 0;
err:
for (k = 0; k < ep_cnt; k++) {
switch(ep_state_array[k].state) {
case FT_EP_CONNECT_RCVD:
fi_reject(pep, ep_state_array[k].fi->handle, NULL, 0);
break;
case FT_EP_CONNECTING:
case FT_EP_CONNECTED:
fi_shutdown(ep_state_array[k].ep, 0);
break;
case FT_EP_STATE_INIT:
default:
break;
}
}
free(ep_state_array);
return ret;
}
static void test_connect_with_accept_blocking_on_eq_fq_SERVER(void)
{
int ret;
printf("SERVER running\n");
setup_ofi(NULL, NULL, FI_SOURCE);
#if WANT_FDS
// Add the EQ FD to the epoll fd
static struct epoll_event edt;
memset(&edt, 0, sizeof(edt));
edt.events = EPOLLIN;
edt.data.u32 = 2222;
ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fidev.eq_fd, &edt);
if (ret < 0) {
error("server epoll_ctl failed");
}
#endif
// Make a PEP
ret = fi_passive_ep(fidev.fabric, fidev.info, &fidev.pep, NULL);
if (0 != ret) {
error("fi_passive_ep failed");
}
#if WANT_FIXED_PORT
size_t ss = sizeof(sin);
ret = fi_getname(&(fidev.pep->fid), &sin, &ss);
if (0 != ret) {
error("fi_setname failed");
}
sin.sin_port = htons(listen_port);
// Bind the PEP to listen on a specific port
ret = fi_setname(&(fidev.pep->fid), &sin, sizeof(sin));
if (0 != ret) {
error("fi_setname failed");
}
#endif
// Bind the EQ to the PEP
ret = fi_pep_bind(fidev.pep, &fidev.eq->fid, 0);
if (0 != ret) {
error("fi_pep_bind failed");
}
// Listen
ret = fi_listen(fidev.pep);
if (0 != ret) {
error("fi_listen failed");
}
// Get the actual address of this PEP
struct sockaddr_in sinout;
size_t s = sizeof(sinout);
ret = fi_getname(&(fidev.pep->fid), &sinout, &s);
if (0 != ret) {
error("fi_setname failed");
}
sin.sin_family = sinout.sin_family;
sin.sin_addr = sinout.sin_addr;
sin.sin_port = sinout.sin_port;
// Print server addr
printf("SERVER listening on %s\n", addrstr(&sin));
// Send our node (IP addr) and service (port) to the client
snprintf(ofi_node, sizeof(ofi_node) - 1, "%s",
inet_ntoa(sin.sin_addr));
snprintf(ofi_service, sizeof(ofi_service) - 1, "%d",
ntohs(sin.sin_port));
MPI_Send(ofi_node, sizeof(ofi_node) - 1, MPI_CHAR,
1, 101, MPI_COMM_WORLD);
MPI_Send(ofi_service, sizeof(ofi_service) - 1, MPI_CHAR,
1, 102, MPI_COMM_WORLD);
printf("SERVER sent via MPI to client: %s / %s\n", ofi_node, ofi_service);
#if WANT_FDS
// Now wait for the listen to complete
int nevents;
#define NEVENTS 32
struct epoll_event events[NEVENTS];
int timeout = 10000;
while (1) {
printf("SERVER blocking on epoll\n");
nevents = epoll_wait(epoll_fd, events, NEVENTS, timeout);
if (nevents < 0) {
if (errno != EINTR) {
error("server epoll wait failed");
} else {
continue;
}
} else {
printf("SERVER successfully woke up from epoll! %d events\n", nevents);
for (int i = 0; i < nevents; ++i) {
if (events[i].data.u32 != 2222) {
error("server unexpected epoll return type");
}
}
// If we got the expected event, then go read from the EQ
break;
}
}
#endif
// Wait for the FI_CONNREQ event
uint32_t event;
uint8_t *entry_buffer;
size_t expected_len = sizeof(struct fi_eq_cm_entry) +
sizeof(client_data);
entry_buffer = (uint8_t*) calloc(1, expected_len);
if (NULL == entry_buffer) {
error("calloc failed");
}
struct fi_eq_cm_entry *entry = (struct fi_eq_cm_entry*) entry_buffer;
while (1) {
printf("SERVER waiting for FI_CONNREQ\n");
#if WANT_FDS
ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0);
#else
ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0);
#endif
if (-FI_EAVAIL == ret) {
printf("server fi_eq_sread failed because there's something in the error queue\n");
char buffer[2048];
struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer;
ret = fi_eq_readerr(fidev.eq, err_entry, 0);
printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno));
error("sad panda");
} else if (-EAGAIN == ret) {
fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n");
sleep(1);
continue;
} else if (ret < 0) {
fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret);
error("SERVER fi_eq_sread failed for some random reason");
} else if (event != FI_CONNREQ) {
error("SERVER got some unexpected event");
} else if (ret != expected_len) {
error("SERVER got wrong length back from fi_eq_sread");
}
uint32_t *d = (uint32_t*) entry->data;
for (int i = 0; i < (sizeof(client_data) / sizeof(uint32_t)); ++i) {
if (d[i] != client_data[i]) {
printf("SERVER got wrong CM client data: d[%d]=%d, should be %d\n",
i, d[i], client_data[i]);
}
}
printf("SERVER got FI_CONNREQ, correct size, and correct data -- yay!\n");
break;
}
// Silly logistics: setup_ofi_active adds the fd to the epoll set.
// But we already added it. So for simplicity, just remove it
// here so that setup_ofi_active() can re-add it.
#if WANT_FDS
// Remove the EQ FD from the epoll fd
ret = epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fidev.eq_fd, &edt);
if (ret < 0) {
error("server epoll_ctl DEL failed");
}
#endif
// Make an active endpoint
setup_ofi_active(entry->info, &ficonn.ep);
// Accept the incoming connection
ret = fi_accept(ficonn.ep, (void*) server_data, sizeof(server_data));
if (ret != 0) {
printf("fi_accept: ret=%d, %s\n", ret, fi_strerror(-ret));
error("SERVER fi_accept failed\n");
}
// Need to read and get a FI_CONNECTED event
while (1) {
printf("SERVER waiting for FI_CONNECTED\n");
#if WANT_FDS
ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0);
#else
ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0);
#endif
if (-FI_EAVAIL == ret) {
printf("server fi_eq_sread failed because there's something in the error queue\n");
char buffer[2048];
struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer;
ret = fi_eq_readerr(fidev.eq, err_entry, 0);
printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno));
error("sad panda");
} else if (-EAGAIN == ret) {
fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n");
sleep(1);
continue;
} else if (ret < 0) {
fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret);
error("SERVER fi_eq_sread failed for some random reason");
} else if (event != FI_CONNECTED) {
error("SERVER got some unexpected event");
}
printf("SERVER got FI_CONNECTED -- yay!\n");
break;
}
// Post a recv buffer for the client to send
int msg[4] = { 0 };
int len = sizeof(msg);
printf("SERVER receiving len of %d\n", len);
struct fid_mr no_mr;
struct fid_mr *mr;
void *recv_context = (void*) 0x17;
#if 0
fi_mr_reg(fidev.domain, msg, len, FI_SEND | FI_RECV,
0, (uint64_t)(uintptr_t) msg, 0, &mr, NULL);
#else
// Try using no mr, like fi_msg_pingpong...
memset(&no_mr, 0, sizeof(no_mr));
mr = &no_mr;
#endif
ret = fi_recv(ficonn.ep, msg, len,
fi_mr_desc(mr), 0, recv_context);
if (ret < 0) {
printf("fi_recv failed! %d, %s\n", ret, fi_strerror(-ret));
MPI_Abort(MPI_COMM_WORLD, 37);
}
sleep(1);
printf("SERVER posted receive -- waiting for client to send\n");
MPI_Barrier(MPI_COMM_WORLD);
// Wait for receive completion
struct fi_cq_entry cqe;
while (1) {
ret = fi_cq_sread(ficonn.cq, &cqe, 1, 0, -1);
if (cqe.op_context == recv_context) {
printf("SERVER receive completed\n");
break;
} else {
printf("SERVER got some other completion... continuing\n");
}
}
printf("SERVER finished -- waiting for client before teardown\n");
MPI_Barrier(MPI_COMM_WORLD);
printf("SERVER tearing down\n");
fi_close(&(mr->fid));
teardown_ofi();
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
return (int) rd;
}
if (event != FI_CONNREQ) {
printf("Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
info = entry.info;
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
printf("fi_domain %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_endpoint(dom, info, &ep, NULL);
if (ret) {
printf("fi_endpoint for req %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err2;
ret = bind_ep_res();
if (ret)
goto err3;
ret = fi_accept(ep, NULL, 0);
if (ret) {
printf("fi_accept %s\n", fi_strerror(-ret));
goto err3;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
goto err3;
}
if (event != FI_COMPLETE || entry.fid != &ep->fid) {
printf("Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err2:
fi_close(&ep->fid);
err1:
fi_reject(pep, info->connreq, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = fi_endpoint(dom, info, &ep, NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", -ret);
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err1;
ret = bind_ep_res();
if (ret)
goto err3;
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err3;
}
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
goto err3;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err1:
fi_reject(pep, info->connreq, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int pp_accept_ctx(struct pingpong_context *ctx)
{
struct fi_eq_cm_entry entry;
uint32_t event;
int rc = 0;
ssize_t rd;
rd = fi_eq_sread(ctx->eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, ctx->eq, "fi_eq_sread", "listen");
return 1;
}
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
return 1;
}
rc = fi_domain(ctx->fabric, entry.info, &ctx->dom, NULL);
if (rc) {
FT_PRINTERR("fi_fdomain", rc);
return 1;
}
rc = fi_mr_reg(ctx->dom, ctx->buf, ctx->size, FI_SEND | FI_RECV, 0, 0, 0, &ctx->mr, NULL);
if (rc) {
FT_PRINTERR("fi_mr_reg", rc);
return 1;
}
rc = fi_endpoint(ctx->dom, entry.info, &ctx->ep, NULL);
if (rc) {
FT_PRINTERR("fi_endpoint", rc);
return 1;
}
fi_freeinfo(entry.info);
/* Create event queue */
if (pp_cq_create(ctx)) {
fprintf(stderr, "Unable to create event queue\n");
return 1;
}
rc = fi_ep_bind(ctx->ep, &ctx->cq->fid, FI_SEND | FI_RECV);
if (rc) {
FT_PRINTERR("fi_ep_bind", rc);
return 1;
}
rc = fi_ep_bind(ctx->ep, &ctx->eq->fid, 0);
if (rc) {
FT_PRINTERR("fi_ep_bind", rc);
return 1;
}
rc = fi_enable(ctx->ep);
if (rc) {
FT_PRINTERR("fi_enable", rc);
return EXIT_FAILURE;
}
ctx->routs = pp_post_recv(ctx, ctx->rx_depth);
if (ctx->routs < ctx->rx_depth) {
FT_ERR("Couldn't post receive (%d)", ctx->routs);
return 1;
}
rc = fi_accept(ctx->ep, NULL, 0);
if (rc) {
FT_PRINTERR("fi_accept", rc);
return 1;
}
rd = fi_eq_sread(ctx->eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, ctx->eq, "fi_eq_sread", "accept");
return 1;
}
if (event != FI_CONNECTED) {
fprintf(stderr, "Unexpected CM event %d\n", event);
return 1;
}
printf("Connection accepted\n");
return 0;
}