static int pp_listen_ctx(struct pingpong_context *ctx)
{
int rc = 0;
rc = fi_passive_ep(ctx->fabric, ctx->info, &ctx->lep, NULL);
if (rc) {
fprintf(stderr, "Unable to open listener endpoint\n");
return 1;
}
/* Create listener EQ */
rc = pp_eq_create(ctx);
if (rc) {
fprintf(stderr, "Unable to allocate listener resources\n");
return 1;
}
rc = fi_pep_bind(ctx->lep, &ctx->eq->fid, 0);
if (rc) {
FT_PRINTERR("fi_pep_bind", rc);
return 1;
}
rc = fi_listen(ctx->lep);
if (rc) {
FT_PRINTERR("fi_listen", rc);
return 1;
}
printf("Listening for incoming connections...\n");
return 0;
}
int cm_server_start(void)
{
int ret;
struct sockaddr_in loc_sa;
cm_local_ip(&loc_sa);
srv_hints = fi_allocinfo();
srv_hints->fabric_attr->name = strdup("gni");
srv_hints->ep_attr->type = FI_EP_MSG;
srv_hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
ret = fi_getinfo(fi_version(), inet_ntoa(loc_sa.sin_addr),
DEF_PORT, FI_SOURCE, srv_hints, &srv_fi);
cr_assert(!ret);
ret = fi_fabric(srv_fi->fabric_attr, &srv_fab, NULL);
cr_assert(!ret);
ret = fi_eq_open(srv_fab, &eq_attr, &srv_eq, NULL);
cr_assert(!ret);
ret = fi_passive_ep(srv_fab, srv_fi, &srv_pep, NULL);
cr_assert(!ret);
ret = fi_pep_bind(srv_pep, &srv_eq->fid, 0);
cr_assert(!ret);
ret = fi_listen(srv_pep);
cr_assert(!ret);
dbg_printf("Server start complete.\n");
return 0;
}
static int rxm_ep_msg_res_open(struct fi_info *rxm_info,
struct util_domain *util_domain, struct rxm_ep *rxm_ep)
{
struct rxm_fabric *rxm_fabric;
struct rxm_domain *rxm_domain;
struct fi_cq_attr cq_attr;
int ret;
ret = ofix_getinfo(rxm_prov.version, NULL, NULL, 0, &rxm_util_prov,
rxm_info, rxm_alter_layer_info, rxm_alter_base_info,
1, &rxm_ep->msg_info);
if (ret)
return ret;
rxm_domain = container_of(util_domain, struct rxm_domain, util_domain);
rxm_fabric = container_of(util_domain->fabric, struct rxm_fabric, util_fabric);
ret = fi_passive_ep(rxm_fabric->msg_fabric, rxm_ep->msg_info, &rxm_ep->msg_pep, rxm_ep);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open msg PEP\n");
goto err1;
}
memset(&cq_attr, 0, sizeof(cq_attr));
cq_attr.size = rxm_info->tx_attr->size + rxm_info->rx_attr->size;
cq_attr.format = FI_CQ_FORMAT_MSG;
ret = fi_cq_open(rxm_domain->msg_domain, &cq_attr, &rxm_ep->msg_cq, NULL);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_CQ, "Unable to open MSG CQ\n");
goto err1;
}
ret = fi_srx_context(rxm_domain->msg_domain, rxm_ep->msg_info->rx_attr,
&rxm_ep->srx_ctx, NULL);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open shared receive context\n");
goto err2;
}
/* We don't care what's in the dest_addr at this point. We go by AV. */
if (rxm_ep->msg_info->dest_addr) {
free(rxm_ep->msg_info->dest_addr);
rxm_ep->msg_info->dest_addr = NULL;
rxm_ep->msg_info->dest_addrlen = 0;
}
/* Zero out the port as we would be creating multiple MSG EPs for a single
* RXM EP and we don't want address conflicts. */
if (rxm_ep->msg_info->src_addr)
((struct sockaddr_in *)(rxm_ep->msg_info->src_addr))->sin_port = 0;
return 0;
err2:
fi_close(&rxm_ep->msg_pep->fid);
err1:
fi_freeinfo(rxm_ep->msg_info);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, NULL, opts.src_port, FI_SOURCE, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Open the fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
/* Open a passive endpoint */
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
/* Allocate connection management resources */
ret = alloc_cm_res();
if (ret)
goto err2;
/* Bind EQ to passive endpoint */
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
/* Listen for incoming connections */
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
fi_close(&cmeq->fid);
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, FI_SOURCE, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
cq_data_size = fi->domain_attr->cq_data_size;
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
printf("fi_passive_ep %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_bind(&pep->fid, &cmeq->fid, 0);
if (ret) {
printf("fi_bind %s\n", fi_strerror(-ret));
goto err3;
}
ret = fi_listen(pep);
if (ret) {
printf("fi_listen %s\n", fi_strerror(-ret));
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE,
hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
int ft_start_server(void)
{
int ret;
ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE,
hints, &fi_pep);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_fabric(fi_pep->fabric_attr, &fabric, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
return ret;
}
ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
if (ret) {
FT_PRINTERR("fi_eq_open", ret);
return ret;
}
ret = fi_passive_ep(fabric, fi_pep, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
return ret;
}
ret = fi_pep_bind(pep, &eq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
return ret;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
return ret;
}
return 0;
}
int ft_start_server(void)
{
int ret;
ret = ft_getinfo(hints, &fi_pep);
if (ret)
return ret;
ret = fi_fabric(fi_pep->fabric_attr, &fabric, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
return ret;
}
ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
if (ret) {
FT_PRINTERR("fi_eq_open", ret);
return ret;
}
ret = fi_passive_ep(fabric, fi_pep, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
return ret;
}
ret = fi_pep_bind(pep, &eq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
return ret;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
return ret;
}
return 0;
}
static void fas_ep_setup(void)
{
int ret, i, j;
size_t addrlen = 0;
fas_setup_common(fi_version());
ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt);
ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt);
for (i = 0; i < NUMEPS; i++) {
fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt;
fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt;
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain returned: %s", fi_strerror(-ret));
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));
switch (ep_type) {
case EP:
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint returned: %s",
fi_strerror(-ret));
break;
case SEP:
ret = fi_scalable_ep(dom[i], fi[i], ep + i,
NULL);
cr_assert(!ret, "fi_endpoint returned: %s",
fi_strerror(-ret));
break;
case PEP:
ret = fi_passive_ep(fab, fi[i], pep + i,
NULL);
cr_assert(!ret, "fi_endpoint returned: %s",
fi_strerror(-ret));
ret = fi_getname(get_fid[ep_type](i), NULL,
&addrlen);
if (use_str_fmt) {
cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN,
"fi_getname returned: %s",
fi_strerror(-ret));
} else {
cr_assert(addrlen ==
sizeof(struct gnix_ep_name),
"fi_getname returned: %s",
fi_strerror(-ret));
}
ep_name_len[i] = addrlen;
continue;
default:
cr_assert_fail("Unknown endpoint type.");
}
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open returned: %s", fi_strerror(-ret));
switch (ep_type) {
case EP:
case PEP:
ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i,
0);
cr_assert(!ret, "fi_cq_open returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(ep[i], &msg_cq[i]->fid,
FI_SEND | FI_RECV);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
break;
case SEP:
dbg_printf(BLUE
"ctx_cnt = %d\n"
COLOR_RESET,
ctx_cnt);
for (j = 0; j < ctx_cnt; j++) {
ret = fi_tx_context(ep[i], j, NULL,
&tx_ep[i][j], NULL);
cr_assert(!ret,
"fi_tx_context returned: %s",
fi_strerror(-ret));
ret = fi_cq_open(dom[i], &cq_attr,
&tx_cq[i][j],
NULL);
cr_assert(!ret,
"fi_cq_open returned: %s",
fi_strerror(-ret));
ret = fi_rx_context(ep[i], j, NULL,
&rx_ep[i][j], NULL);
cr_assert(!ret,
"fi_rx_context returned: %s",
fi_strerror(-ret));
ret = fi_cq_open(dom[i], &cq_attr,
&rx_cq[i][j],
NULL);
cr_assert(!ret,
"fi_cq_open returned: %s",
fi_strerror(-ret));
}
break;
default:
cr_assert_fail("Unknown endpoint type.");
}
ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen);
if (use_str_fmt) {
cr_assert(addrlen > sizeof(struct gnix_ep_name),
"fi_getname returned: %s",
fi_strerror(-ret));
} else {
cr_assert(addrlen == sizeof(struct gnix_ep_name),
"fi_getname returned: %s",
fi_strerror(-ret));
}
ep_name[i] = malloc(addrlen);
ep_name_len[i] = addrlen;
dbg_printf(BLUE
"ep_name_len[%d] = %lu\n"
COLOR_RESET, i,
ep_name_len[i]);
cr_assert(ep_name[i] != NULL, "malloc returned: %s",
strerror(errno));
ret = fi_getname(get_fid[ep_type](i), ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS, "fi_getname returned: %s",
fi_strerror(-ret));
}
/* Just testing setname / getname for passive endpoints */
if (ep_type == PEP)
return;
for (i = 0; i < NUMEPS; i++) {
/*Insert all gni addresses into each av*/
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1, "fi_av_insert returned: %s",
fi_strerror(-ret));
}
switch (ep_type) {
case EP:
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(ep[i], &send_cntr[i]->fid,
FI_SEND);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid,
FI_RECV);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
break;
case SEP:
ret = fi_scalable_ep_bind(ep[i], &av[i]->fid,
0);
cr_assert(!ret,
"fi_scalable_ep_bind returned: %s",
fi_strerror(-ret));
dbg_printf(BLUE
"ctx_cnt = %d\n"
COLOR_RESET,
ctx_cnt);
for (j = 0; j < ctx_cnt; j++) {
ret = fi_ep_bind(tx_ep[i][j],
&tx_cq[i][j]->fid,
FI_TRANSMIT);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(tx_ep[i][j],
&send_cntr[i]->fid,
FI_SEND);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_enable(tx_ep[i][j]);
cr_assert(!ret,
"fi_enable returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(rx_ep[i][j],
&rx_cq[i][j]->fid,
FI_RECV);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(rx_ep[i][j],
&recv_cntr[i]->fid,
FI_RECV);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_enable(rx_ep[i][j]);
cr_assert(!ret,
"fi_enable returned: %s",
fi_strerror(-ret));
}
break;
case PEP:
break;
default:
cr_assert_fail("Unknown endpoint type.");
}
ret = fi_enable(ep[i]);
cr_assert(!ret, "fi_ep_enable returned: %s", fi_strerror(-ret));
if (ep_type != SEP) {
ret = fi_enable(ep[i]);
cr_assert_eq(ret, -FI_EOPBADSTATE,
"fi_enable returned: %s",
fi_strerror(-ret));
}
}
}
static int setup_handle(void)
{
static char buf[BUFSIZ];
struct addrinfo *ai, aihints;
const char *bound_addr_str;
int ret;
memset(&aihints, 0, sizeof aihints);
aihints.ai_flags = AI_PASSIVE;
ret = getaddrinfo(opts.src_addr, opts.src_port, &aihints, &ai);
if (ret == EAI_SYSTEM) {
FT_ERR("getaddrinfo for %s:%s: %s\n",
opts.src_addr, opts.src_port, strerror(errno));
return -ret;
} else if (ret) {
FT_ERR("getaddrinfo: %s\n", gai_strerror(ret));
return -FI_ENODATA;
}
switch (ai->ai_family) {
case AF_INET:
hints->addr_format = FI_SOCKADDR_IN;
break;
case AF_INET6:
hints->addr_format = FI_SOCKADDR_IN6;
break;
}
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, opts.src_addr, NULL, FI_SOURCE, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto out;
}
free(fi->src_addr);
fi->src_addr = NULL;
fi->src_addrlen = 0;
ret = fi_fabric(fi->fabric_attr, &fabric, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto out;
}
ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
if (ret) {
FT_PRINTERR("fi_eq_open", ret);
goto out;
}
/* Open a passive endpoint */
ret = fi_passive_ep(fabric, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto out;
}
ret = fi_setname(&pep->fid, ai->ai_addr, ai->ai_addrlen);
if (ret) {
FT_PRINTERR("fi_setname", ret);
goto out;
}
ret = fi_getname(&pep->fid, &bound_addr, &bound_addr_len);
if (ret) {
FT_PRINTERR("fi_getname", ret);
goto out;
}
/* Verify port number */
switch (ai->ai_family) {
case AF_INET:
if (bound_addr.sin.sin_port == 0) {
FT_ERR("port number is 0 after fi_setname()\n");
ret = -FI_EINVAL;
goto out;
}
break;
case AF_INET6:
if (bound_addr.sin6.sin6_port == 0) {
FT_ERR("port number is 0 after fi_setname()\n");
ret = -FI_EINVAL;
goto out;
}
break;
}
bound_addr_str = sockaddrstr(&bound_addr, bound_addr_len, buf, BUFSIZ);
if (!bound_addr_str) {
FT_ERR("Unable to get bound_addr as string!\n");
ret = -FI_EINVAL;
goto out;
}
printf("bound_addr: \"%s\"\n", bound_addr_str);
hints->handle = &pep->fid;
out:
freeaddrinfo(ai);
return ret;
}
/*
* rpmemd_fip_init_fabric_res -- initialize common fabric's resources
*/
static int
rpmemd_fip_init_fabric_res(struct rpmemd_fip *fip)
{
int ret;
ret = fi_fabric(fip->fi->fabric_attr, &fip->fabric, NULL);
if (ret) {
RPMEMD_FI_ERR(ret, "opening fabric domain");
goto err_fi_fabric;
}
ret = fi_domain(fip->fabric, fip->fi, &fip->domain, NULL);
if (ret) {
RPMEMD_FI_ERR(ret, "opening fabric access domain");
goto err_fi_domain;
}
struct fi_eq_attr eq_attr = {
.size = 0, /* use default */
.flags = 0,
.wait_obj = FI_WAIT_UNSPEC,
.signaling_vector = 0,
.wait_set = NULL,
};
ret = fi_eq_open(fip->fabric, &eq_attr, &fip->eq, NULL);
if (ret) {
RPMEMD_FI_ERR(ret, "opening event queue");
goto err_eq_open;
}
ret = fi_passive_ep(fip->fabric, fip->fi, &fip->pep, NULL);
if (ret) {
RPMEMD_FI_ERR(ret, "allocating passive endpoint");
goto err_pep;
}
ret = fi_pep_bind(fip->pep, &fip->eq->fid, 0);
if (ret) {
RPMEMD_FI_ERR(ret, "binding event queue to passive endpoint");
goto err_pep_bind_eq;
}
return 0;
err_pep_bind_eq:
RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint");
err_pep:
RPMEMD_FI_CLOSE(fip->eq, "closing event queue");
err_eq_open:
RPMEMD_FI_CLOSE(fip->domain, "closing fabric access domain");
err_fi_domain:
RPMEMD_FI_CLOSE(fip->fabric, "closing fabric domain");
err_fi_fabric:
return ret;
}
/*
* rpmemd_fip_fini_fabric_res -- deinitialize common fabric resources
*/
static void
rpmemd_fip_fini_fabric_res(struct rpmemd_fip *fip)
{
RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint");
RPMEMD_FI_CLOSE(fip->eq, "closing event queue");
RPMEMD_FI_CLOSE(fip->domain, "closing fabric access domain");
RPMEMD_FI_CLOSE(fip->fabric, "closing fabric domain");
}
static void test_connect_with_accept_blocking_on_eq_fq_SERVER(void)
{
int ret;
printf("SERVER running\n");
setup_ofi(NULL, NULL, FI_SOURCE);
#if WANT_FDS
// Add the EQ FD to the epoll fd
static struct epoll_event edt;
memset(&edt, 0, sizeof(edt));
edt.events = EPOLLIN;
edt.data.u32 = 2222;
ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fidev.eq_fd, &edt);
if (ret < 0) {
error("server epoll_ctl failed");
}
#endif
// Make a PEP
ret = fi_passive_ep(fidev.fabric, fidev.info, &fidev.pep, NULL);
if (0 != ret) {
error("fi_passive_ep failed");
}
#if WANT_FIXED_PORT
size_t ss = sizeof(sin);
ret = fi_getname(&(fidev.pep->fid), &sin, &ss);
if (0 != ret) {
error("fi_setname failed");
}
sin.sin_port = htons(listen_port);
// Bind the PEP to listen on a specific port
ret = fi_setname(&(fidev.pep->fid), &sin, sizeof(sin));
if (0 != ret) {
error("fi_setname failed");
}
#endif
// Bind the EQ to the PEP
ret = fi_pep_bind(fidev.pep, &fidev.eq->fid, 0);
if (0 != ret) {
error("fi_pep_bind failed");
}
// Listen
ret = fi_listen(fidev.pep);
if (0 != ret) {
error("fi_listen failed");
}
// Get the actual address of this PEP
struct sockaddr_in sinout;
size_t s = sizeof(sinout);
ret = fi_getname(&(fidev.pep->fid), &sinout, &s);
if (0 != ret) {
error("fi_setname failed");
}
sin.sin_family = sinout.sin_family;
sin.sin_addr = sinout.sin_addr;
sin.sin_port = sinout.sin_port;
// Print server addr
printf("SERVER listening on %s\n", addrstr(&sin));
// Send our node (IP addr) and service (port) to the client
snprintf(ofi_node, sizeof(ofi_node) - 1, "%s",
inet_ntoa(sin.sin_addr));
snprintf(ofi_service, sizeof(ofi_service) - 1, "%d",
ntohs(sin.sin_port));
MPI_Send(ofi_node, sizeof(ofi_node) - 1, MPI_CHAR,
1, 101, MPI_COMM_WORLD);
MPI_Send(ofi_service, sizeof(ofi_service) - 1, MPI_CHAR,
1, 102, MPI_COMM_WORLD);
printf("SERVER sent via MPI to client: %s / %s\n", ofi_node, ofi_service);
#if WANT_FDS
// Now wait for the listen to complete
int nevents;
#define NEVENTS 32
struct epoll_event events[NEVENTS];
int timeout = 10000;
while (1) {
printf("SERVER blocking on epoll\n");
nevents = epoll_wait(epoll_fd, events, NEVENTS, timeout);
if (nevents < 0) {
if (errno != EINTR) {
error("server epoll wait failed");
} else {
continue;
}
} else {
printf("SERVER successfully woke up from epoll! %d events\n", nevents);
for (int i = 0; i < nevents; ++i) {
if (events[i].data.u32 != 2222) {
error("server unexpected epoll return type");
}
}
// If we got the expected event, then go read from the EQ
break;
}
}
#endif
// Wait for the FI_CONNREQ event
uint32_t event;
uint8_t *entry_buffer;
size_t expected_len = sizeof(struct fi_eq_cm_entry) +
sizeof(client_data);
entry_buffer = (uint8_t*) calloc(1, expected_len);
if (NULL == entry_buffer) {
error("calloc failed");
}
struct fi_eq_cm_entry *entry = (struct fi_eq_cm_entry*) entry_buffer;
while (1) {
printf("SERVER waiting for FI_CONNREQ\n");
#if WANT_FDS
ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0);
#else
ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0);
#endif
if (-FI_EAVAIL == ret) {
printf("server fi_eq_sread failed because there's something in the error queue\n");
char buffer[2048];
struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer;
ret = fi_eq_readerr(fidev.eq, err_entry, 0);
printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno));
error("sad panda");
} else if (-EAGAIN == ret) {
fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n");
sleep(1);
continue;
} else if (ret < 0) {
fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret);
error("SERVER fi_eq_sread failed for some random reason");
} else if (event != FI_CONNREQ) {
error("SERVER got some unexpected event");
} else if (ret != expected_len) {
error("SERVER got wrong length back from fi_eq_sread");
}
uint32_t *d = (uint32_t*) entry->data;
for (int i = 0; i < (sizeof(client_data) / sizeof(uint32_t)); ++i) {
if (d[i] != client_data[i]) {
printf("SERVER got wrong CM client data: d[%d]=%d, should be %d\n",
i, d[i], client_data[i]);
}
}
printf("SERVER got FI_CONNREQ, correct size, and correct data -- yay!\n");
break;
}
// Silly logistics: setup_ofi_active adds the fd to the epoll set.
// But we already added it. So for simplicity, just remove it
// here so that setup_ofi_active() can re-add it.
#if WANT_FDS
// Remove the EQ FD from the epoll fd
ret = epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fidev.eq_fd, &edt);
if (ret < 0) {
error("server epoll_ctl DEL failed");
}
#endif
// Make an active endpoint
setup_ofi_active(entry->info, &ficonn.ep);
// Accept the incoming connection
ret = fi_accept(ficonn.ep, (void*) server_data, sizeof(server_data));
if (ret != 0) {
printf("fi_accept: ret=%d, %s\n", ret, fi_strerror(-ret));
error("SERVER fi_accept failed\n");
}
// Need to read and get a FI_CONNECTED event
while (1) {
printf("SERVER waiting for FI_CONNECTED\n");
#if WANT_FDS
ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0);
#else
ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0);
#endif
if (-FI_EAVAIL == ret) {
printf("server fi_eq_sread failed because there's something in the error queue\n");
char buffer[2048];
struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer;
ret = fi_eq_readerr(fidev.eq, err_entry, 0);
printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno));
error("sad panda");
} else if (-EAGAIN == ret) {
fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n");
sleep(1);
continue;
} else if (ret < 0) {
fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret);
error("SERVER fi_eq_sread failed for some random reason");
} else if (event != FI_CONNECTED) {
error("SERVER got some unexpected event");
}
printf("SERVER got FI_CONNECTED -- yay!\n");
break;
}
// Post a recv buffer for the client to send
int msg[4] = { 0 };
int len = sizeof(msg);
printf("SERVER receiving len of %d\n", len);
struct fid_mr no_mr;
struct fid_mr *mr;
void *recv_context = (void*) 0x17;
#if 0
fi_mr_reg(fidev.domain, msg, len, FI_SEND | FI_RECV,
0, (uint64_t)(uintptr_t) msg, 0, &mr, NULL);
#else
// Try using no mr, like fi_msg_pingpong...
memset(&no_mr, 0, sizeof(no_mr));
mr = &no_mr;
#endif
ret = fi_recv(ficonn.ep, msg, len,
fi_mr_desc(mr), 0, recv_context);
if (ret < 0) {
printf("fi_recv failed! %d, %s\n", ret, fi_strerror(-ret));
MPI_Abort(MPI_COMM_WORLD, 37);
}
sleep(1);
printf("SERVER posted receive -- waiting for client to send\n");
MPI_Barrier(MPI_COMM_WORLD);
// Wait for receive completion
struct fi_cq_entry cqe;
while (1) {
ret = fi_cq_sread(ficonn.cq, &cqe, 1, 0, -1);
if (cqe.op_context == recv_context) {
printf("SERVER receive completed\n");
break;
} else {
printf("SERVER got some other completion... continuing\n");
}
}
printf("SERVER finished -- waiting for client before teardown\n");
MPI_Barrier(MPI_COMM_WORLD);
printf("SERVER tearing down\n");
fi_close(&(mr->fid));
teardown_ofi();
}
static int run_server(void)
{
int i, ret;
printf("cmatose: starting server\n");
ret = fi_passive_ep(fabric, info, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
return ret;
}
ret = fi_pep_bind(pep, &eq->fid, 0);
if (ret) {
FT_PRINTERR("fi_ep_bind", ret);
goto out;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto out;
}
ret = connect_events();
if (ret)
goto out;
if (hints->tx_attr->size) {
printf("initiating data transfers\n");
for (i = 0; i < connections; i++) {
ret = post_sends(&nodes[i]);
if (ret)
goto out;
}
printf("completing sends\n");
ret = poll_cqs(SEND_CQ_INDEX);
if (ret)
goto out;
printf("receiving data transfers\n");
ret = poll_cqs(RECV_CQ_INDEX);
if (ret)
goto out;
printf("data transfers complete\n");
}
printf("cmatose: disconnecting\n");
for (i = 0; i < connections; i++) {
if (!nodes[i].connected)
continue;
nodes[i].connected = 0;
fi_shutdown(nodes[i].ep, 0);
}
ret = shutdown_events();
printf("disconnected\n");
out:
fi_close(&pep->fid);
return ret;
}
