static int server_listen(void)
{
struct fi_info *fi;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, NULL, opts.src_port, FI_SOURCE, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
/* Open the fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
/* Open a passive endpoint */
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
/* Allocate connection management resources */
ret = alloc_cm_res();
if (ret)
goto err2;
/* Bind EQ to passive endpoint */
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
/* Listen for incoming connections */
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
fi_close(&cmeq->fid);
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cq_attr cq_attr;
int ret;
buffer_size = !custom ? test_size[TEST_CNT - 1].size : transfer_size;
if (buffer_size < MIN_BUF_SIZE) {
buffer_size = MIN_BUF_SIZE;
}
buf = malloc(buffer_size);
if (!buf) {
perror("malloc");
return -1;
}
memset(&cq_attr, 0, sizeof cq_attr);
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.size = max_credits << 1;
ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
if (ret) {
printf("fi_eq_open send comp %s\n", fi_strerror(-ret));
goto err1;
}
ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
if (ret) {
printf("fi_eq_open recv comp %s\n", fi_strerror(-ret));
goto err2;
}
ret = fi_mr_reg(dom, buf, buffer_size, FI_REMOTE_WRITE, 0, 0, 0, &mr, NULL);
if (ret) {
printf("fi_mr_reg %s\n", fi_strerror(-ret));
goto err3;
}
if (!cmeq) {
ret = alloc_cm_res();
if (ret)
goto err4;
}
return 0;
err4:
fi_close(&mr->fid);
err3:
fi_close(&rcq->fid);
err2:
fi_close(&scq->fid);
err1:
free(buf);
return ret;
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cq_attr cq_attr;
int ret;
buffer_size = test_size[TEST_CNT - 1].size;
buf = malloc(buffer_size);
if (!buf) {
perror("malloc");
return -1;
}
memset(&cq_attr, 0, sizeof cq_attr);
cq_attr.format = FI_CQ_FORMAT_DATA;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.size = rx_depth;
ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
if (ret) {
printf("fi_cq_open send comp %s\n", fi_strerror(-ret));
goto err1;
}
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
if (ret) {
printf("fi_cq_open recv comp %s\n", fi_strerror(-ret));
goto err2;
}
ret = fi_mr_reg(dom, buf, buffer_size, 0, 0, 0, 0, &mr, NULL);
if (ret) {
printf("fi_mr_reg %s\n", fi_strerror(-ret));
goto err3;
}
if (!cmeq) {
ret = alloc_cm_res();
if (ret)
goto err4;
}
return 0;
err4:
fi_close(&mr->fid);
err3:
fi_close(&rcq->fid);
err2:
fi_close(&scq->fid);
err1:
free(buf);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, FI_SOURCE, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
cq_data_size = fi->domain_attr->cq_data_size;
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
printf("fi_passive_ep %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_bind(&pep->fid, &cmeq->fid, 0);
if (ret) {
printf("fi_bind %s\n", fi_strerror(-ret));
goto err3;
}
ret = fi_listen(pep);
if (ret) {
printf("fi_listen %s\n", fi_strerror(-ret));
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE,
hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
return ret;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err0;
}
ret = fi_passive_ep(fab, fi, &pep, NULL);
if (ret) {
FT_PRINTERR("fi_passive_ep", ret);
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = fi_pep_bind(pep, &cmeq->fid, 0);
if (ret) {
FT_PRINTERR("fi_pep_bind", ret);
goto err3;
}
ret = fi_listen(pep);
if (ret) {
FT_PRINTERR("fi_listen", ret);
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int server_listen(void)
{
struct fi_info *fi;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, 0, &hints, &fi);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
printf("fi_fabric %s\n", fi_strerror(-ret));
goto err0;
}
ret = fi_pendpoint(fab, fi, &pep, NULL);
if (ret) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto err1;
}
ret = alloc_cm_res();
if (ret)
goto err2;
ret = bind_fid(&pep->fid, &cmeq->fid, 0);
if (ret)
goto err3;
ret = fi_listen(pep);
if (ret) {
printf("fi_listen %s\n", fi_strerror(-ret));
goto err3;
}
fi_freeinfo(fi);
return 0;
err3:
free_lres();
err2:
fi_close(&pep->fid);
err1:
fi_close(&fab->fid);
err0:
fi_freeinfo(fi);
return ret;
}
static int client_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *fi;
ssize_t rd;
int ret;
/* Get fabric info */
ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
if (ret) {
FT_PRINTERR("fi_getinfo", ret);
goto err0;
}
/* Open fabric */
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
if (ret) {
FT_PRINTERR("fi_fabric", ret);
goto err1;
}
/* Open domain */
ret = fi_domain(fab, fi, &dom, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err2;
}
ret = alloc_cm_res();
if (ret)
goto err4;
ret = alloc_ep_res(fi);
if (ret)
goto err5;
ret = bind_ep_res();
if (ret)
goto err6;
/* Connect to server */
ret = fi_connect(ep, fi->dest_addr, NULL, 0);
if (ret) {
FT_PRINTERR("fi_connect", ret);
goto err6;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PROCESS_EQ_ERR(rd, cmeq, "fi_eq_sread", "connect");
ret = (int) rd;
goto err6;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
event, entry.fid, ep);
ret = -FI_EOTHER;
goto err6;
}
fi_freeinfo(fi);
return 0;
err6:
free_ep_res();
err5:
fi_close(&cmeq->fid);
err4:
fi_close(&dom->fid);
err2:
fi_close(&fab->fid);
err1:
fi_freeinfo(fi);
err0:
return ret;
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cq_attr cq_attr;
uint64_t access_mode;
int ret;
buffer_size = opts.user_options & FT_OPT_SIZE ?
opts.transfer_size : test_size[TEST_CNT - 1].size;
buf = malloc(MAX(buffer_size, sizeof(uint64_t)));
if (!buf) {
perror("malloc");
return -1;
}
memset(&cq_attr, 0, sizeof cq_attr);
cq_attr.format = FI_CQ_FORMAT_DATA;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.size = max_credits << 1;
ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
goto err1;
}
ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
goto err2;
}
switch (op_type) {
case FT_RMA_READ:
access_mode = FI_REMOTE_READ;
break;
case FT_RMA_WRITE:
case FT_RMA_WRITEDATA:
access_mode = FI_REMOTE_WRITE;
break;
default:
assert(0);
ret = -FI_EINVAL;
goto err3;
}
ret = fi_mr_reg(dom, buf, MAX(buffer_size, sizeof(uint64_t)),
access_mode, 0, 0, 0, &mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
goto err3;
}
if (!cmeq) {
ret = alloc_cm_res();
if (ret)
goto err4;
}
return 0;
err4:
fi_close(&mr->fid);
err3:
fi_close(&rcq->fid);
err2:
fi_close(&scq->fid);
err1:
free(buf);
return ret;
}
