int ft_server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "listen");
return (int) rd;
}
fi = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = ft_alloc_active_res(fi);
if (ret)
goto err;
ret = ft_init_ep();
if (ret)
goto err;
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "accept");
ret = (int) rd;
goto err;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
return 0;
err:
fi_reject(pep, fi->handle, NULL, 0);
return ret;
}
static int client_expect_reject(size_t paramlen)
{
uint32_t event;
int ret;
ret = client_connect(paramlen);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
ret = fi_eq_sread(eq, &event, entry, sizeof(*entry), -1, 0);
if (ret != -FI_EAVAIL) {
FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "connect");
return ret;
}
ret = fi_eq_readerr(eq, &err_entry, 0);
if (ret != sizeof(err_entry)) {
FT_EQ_ERR(eq, err_entry, NULL, 0);
return err_entry.err;
}
if (err_entry.err != FI_ECONNREFUSED)
return err_entry.err;
/* Check data on FI_ECONNREFUSED error event. */
return ft_check_buf(err_entry.err_data, err_entry.err_data_size);
}
static int server_listen(size_t paramlen)
{
size_t expected;
uint32_t event;
int ret;
expected = paramlen + sizeof(*entry);
memset(entry, 0, expected);
ret = fi_eq_sread(eq, &event, entry, expected, -1, 0);
if (ret != expected) {
FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "listen");
return ret;
}
if (event != FI_CONNREQ) {
FT_ERR("Unexpected CM event %d", event);
return -FI_EOTHER;
}
ret = ft_check_buf(entry->data, paramlen);
if (ret)
return ret;
fi = entry->info;
return 0;
}
static int client_expect_accept(size_t paramlen)
{
size_t expected;
uint32_t event;
int ret;
expected = paramlen + sizeof(*entry);
ret = client_connect(paramlen);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
ret = fi_eq_sread(eq, &event, entry, expected, -1, 0);
if (ret != expected) {
FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "connect");
return ret;
}
if (event != FI_CONNECTED || entry->fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)", event,
entry->fid, ep);
return -FI_EOTHER;
}
/* Check data on FI_CONNECTED event. */
ret = ft_check_buf(entry->data, paramlen);
if (ret)
return ret;
fi_shutdown(ep, 0);
return read_shutdown_event();
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
if (ret)
return ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = ft_init_ep(buf);
if (ret)
return ret;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
return -FI_EOTHER;
}
return 0;
}
int ft_client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = ft_alloc_active_res(fi);
if (ret)
return ret;
ret = ft_init_ep();
if (ret)
return ret;
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
ret = (int) rd;
return ret;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
return ret;
}
return 0;
}
static int read_shutdown_event()
{
int ret;
uint32_t event;
memset(entry, 0, sizeof(*entry));
ret = fi_eq_sread(eq, &event, entry, sizeof(*entry), -1, 0);
if (ret < 0) {
FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "shutdown");
return ret;
}
if (event != FI_SHUTDOWN || entry->fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)", event,
entry->fid, ep);
ret = -FI_EOTHER;
return ret;
}
return 0;
}
static int shutdown_events(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
int ret = 0;
ssize_t rd;
while (disconnects_left && !ret) {
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
printf("unexpected event during shutdown\n");
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "shutdown");
ret = -FI_EIO;
break;
}
ret = cma_handler(event, &entry);
}
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err0;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_cm_res();
if (ret)
goto err4;
ret = alloc_ep_res(fi);
if (ret)
goto err5;
ret = bind_ep_res();
if (ret)
goto err6;
/* Connect to server */
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
goto err6;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "connect");
ret = (int) rd;
goto err6;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err6;
}
fi_freeinfo(fi);
return 0;
err6:
free_ep_res();
err5:
fi_close(&cmeq->fid);
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
/* Wait for connection request from client */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "listen");
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err1;
}
ret = fi_domain(fab, info, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err1;
}
ret = alloc_ep_res(info);
if (ret)
goto err1;
ret = bind_ep_res();
if (ret)
goto err3;
/* Accept the incoming connection. Also transitions endpoint to active state */
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err3;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "accept");
ret = (int) rd;
goto err3;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err3;
}
fi_freeinfo(info);
return 0;
err3:
free_ep_res();
err1:
fi_reject(pep, info->handle, NULL, 0);
fi_freeinfo(info);
return ret;
}
static int server_accept(size_t paramlen)
{
uint32_t event;
int ret;
ret = server_listen(paramlen);
if (ret)
return ret;
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = ft_alloc_active_res(fi);
if (ret) {
FT_PRINTERR("alloc_active_res", ret);
goto err;
}
ret = ft_init_ep();
if (ret) {
FT_PRINTERR("init_ep", ret);
goto err;
}
/* Data will apppear on accept event on remote end. */
ft_fill_buf(cm_data, paramlen);
/* Accept the incoming connection. Also transitions endpoint to active
* state.
*/
ret = fi_accept(ep, cm_data, paramlen);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
/* Local FI_CONNECTED event does not have data associated. */
memset(entry, 0, sizeof(*entry));
ret = fi_eq_sread(eq, &event, entry, sizeof(*entry), -1, 0);
if (ret != sizeof(*entry)) {
FT_PROCESS_EQ_ERR(ret, eq, "fi_eq_sread", "accept");
goto err;
}
if (event != FI_CONNECTED || entry->fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)", event,
entry->fid, ep);
ret = -FI_EOTHER;
goto err;
}
fi_shutdown(ep, 0);
ret = read_shutdown_event();
if (ret)
goto err;
FT_CLOSE_FID(ep);
FT_CLOSE_FID(rxcq);
FT_CLOSE_FID(txcq);
FT_CLOSE_FID(rxcntr);
FT_CLOSE_FID(txcntr);
FT_CLOSE_FID(av);
FT_CLOSE_FID(domain);
return 0;
err:
fi_reject(pep, fi->handle, NULL, 0);
return ret;
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret, k;
int num_conn_reqs = 0, num_connected = 0;
struct ep_info *ep_state_array = NULL;
ep_array = calloc(ep_cnt, sizeof(*ep_array));
if (!ep_array)
return -FI_ENOMEM;
ep_state_array = calloc(ep_cnt, sizeof(*ep_state_array));
if (!ep_state_array)
return -FI_ENOMEM;
while (num_connected != ep_cnt) {
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "cm-event");
ret = (int) rd;
goto err;
}
switch(event) {
case FI_CONNREQ:
if (num_conn_reqs == ep_cnt) {
fprintf(stderr, "Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
fi = ep_state_array[num_conn_reqs].fi = entry.info;
ep_state_array[num_conn_reqs].state = FT_EP_CONNECT_RCVD;
if (num_conn_reqs == 0) {
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = alloc_ep_res(fi);
if (ret)
goto err;
}
ret = fi_endpoint(domain, fi, &ep_array[num_conn_reqs], NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
goto err;
}
ep_state_array[num_conn_reqs].ep = ep_array[num_conn_reqs];
ret = bind_ep_res(ep_array[num_conn_reqs]);
if (ret)
goto err;
ret = fi_accept(ep_array[num_conn_reqs], NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
ep_state_array[num_conn_reqs].state = FT_EP_CONNECTING;
num_conn_reqs++;
break;
case FI_CONNECTED:
if (num_conn_reqs <= num_connected) {
ret = -FI_EOTHER;
goto err;
}
for (k = 0; k < num_conn_reqs; k++) {
if (ep_state_array[k].state != FT_EP_CONNECTING)
continue;
if (&ep_state_array[k].ep->fid == entry.fid) {
ep_state_array[k].state = FT_EP_CONNECTED;
num_connected++;
if (num_connected != ep_cnt)
fi_freeinfo(ep_state_array[k].fi);
break;
}
}
if (k == num_conn_reqs) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
break;
default:
ret = -FI_EOTHER;
goto err;
}
}
/* Post recv */
if (rx_shared_ctx)
ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
else
ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
if (ret)
goto err;
free(ep_state_array);
return 0;
err:
for (k = 0; k < ep_cnt; k++) {
switch(ep_state_array[k].state) {
case FT_EP_CONNECT_RCVD:
fi_reject(pep, ep_state_array[k].fi->handle, NULL, 0);
break;
case FT_EP_CONNECTING:
case FT_EP_CONNECTED:
fi_shutdown(ep_state_array[k].ep, 0);
break;
case FT_EP_STATE_INIT:
default:
break;
}
}
free(ep_state_array);
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int i, ret;
ret = ft_getinfo(hints, &fi);
if (ret)
return ret;
ret = get_dupinfo();
if (ret)
return ret;
ret = ft_open_fabric_res();
if (ret)
return ret;
ret = alloc_ep_res(fi);
if (ret)
return ret;
ret = alloc_ep();
if (ret)
return ret;
ret = bind_ep_array_res();
if (ret)
return ret;
for (i = 0; i < ep_cnt; i++) {
ret = fi_connect(ep_array[i], fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "connect");
ret = (int) rd;
return ret;
}
if (event != FI_CONNECTED || entry.fid != &ep_array[i]->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
return ret;
}
}
/* Post recv */
if (rx_shared_ctx)
ret = ft_post_rx(srx_ctx, MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
else
ret = ft_post_rx(ep_array[0], MAX(rx_size, FT_MAX_CTRL_MSG), &rx_ctx);
if (ret)
return ret;
return 0;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
ssize_t rd;
int ret;
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_domain(fabric, fi, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
return ret;
}
ret = check_address(&pep->fid, "fi_endpoint (pep)");
if (ret)
return ret;
assert(fi->handle == &pep->fid);
ret = ft_alloc_active_res(fi);
if (ret)
return ret;
/* Close the passive endpoint that we "stole" the source address
* from */
FT_CLOSE_FID(pep);
ret = ft_init_ep();
if (ret)
return ret;
ret = check_address(&ep->fid, "fi_endpoint (ep)");
if (ret)
return ret;
/* Connect to server */
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
return ret;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, eq, "fi_eq_sread", "listen");
return (int) rd;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)", event, entry.fid, ep);
return -FI_EOTHER;
}
ret = check_address(&ep->fid, "connect");
if (ret) {
return ret;
}
return 0;
}
static int pp_connect_ctx(struct pingpong_context *ctx)
{
struct fi_eq_cm_entry entry;
uint32_t event;
int rc = 0;
ssize_t rd;
/* Open domain */
rc = fi_domain(ctx->fabric, ctx->info, &ctx->dom, NULL);
if (rc) {
FT_PRINTERR("fi_fdomain", rc);
return 1;
}
if (pp_eq_create(ctx)) {
fprintf(stderr, "Unable to create event queue\n");
return 1;
}
rc = fi_mr_reg(ctx->dom, ctx->buf, ctx->size, FI_SEND | FI_RECV, 0, 0, 0, &ctx->mr, NULL);
if (rc) {
FT_PRINTERR("fi_mr_reg", rc);
return 1;
}
/* Open endpoint */
rc = fi_endpoint(ctx->dom, ctx->info, &ctx->ep, NULL);
if (rc) {
FT_PRINTERR("fi_endpoint", rc);
return 1;
}
/* Create event queue */
if (pp_cq_create(ctx)) {
fprintf(stderr, "Unable to create event queue\n");
return 1;
}
/* Bind eq to ep */
rc = fi_ep_bind(ctx->ep, &ctx->cq->fid, FI_SEND | FI_RECV);
if (rc) {
FT_PRINTERR("fi_ep_bind", rc);
return 1;
}
rc = fi_ep_bind(ctx->ep, &ctx->eq->fid, 0);
if (rc) {
FT_PRINTERR("fi_ep_bind", rc);
return 1;
}
rc = fi_enable(ctx->ep);
if (rc) {
FT_PRINTERR("fi_enable", rc);
return EXIT_FAILURE;
}
ctx->routs = pp_post_recv(ctx, ctx->rx_depth);
if (ctx->routs < ctx->rx_depth) {
FT_ERR("Couldn't post receive (%d)", ctx->routs);
return 1;
}
printf("Connecting to server\n");
rc = fi_connect(ctx->ep, ctx->info->dest_addr, NULL, 0);
if (rc) {
FT_PRINTERR("fi_connect", rc);
return 1;
}
rd = fi_eq_sread(ctx->eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, ctx->eq, "fi_eq_sread", "connect");
return 1;
}
if (event != FI_CONNECTED) {
fprintf(stderr, "Unexpected CM event %d\n", event);
return 1;
}
printf("Connection successful\n");
return 0;
}
