static void cntr_setup_mt(void)
{
int i, ret;
cntr_setup_eps();
cntr_setup_av();
cntr_setup_cqs();
cntr_setup_bind_cqs();
for (i = 0; i < NUM_EPS; i++) {
ret = fi_cntr_open(dom, &cntr_attr, &ep_write_cntrs[i], 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &ep_write_cntrs[i]->fid,
FI_WRITE | FI_SEND);
cr_assert(!ret, "fi_ep_bind cntr");
ret = fi_cntr_open(dom, &cntr_attr, &ep_read_cntrs[i], 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &ep_read_cntrs[i]->fid, FI_READ);
cr_assert(!ret, "fi_ep_bind cntr");
}
cntr_setup_enable_ep();
cntr_setup_mr();
atomic_initialize(&cntr_test_next_tid, 0);
}
static inline void cntr_setup_cntrs(void)
{
int ret;
ret = fi_cntr_open(dom, &cntr_attr, &write_cntr, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom, &cntr_attr, &read_cntr, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom, &cntr_attr, &rcv_cntr, 0);
cr_assert(!ret, "fi_cntr_open");
}
static inline int allocate_cntr_and_cq(void)
{
int ret = 0;
struct fi_cntr_attr cntr_attr = {0};
struct fi_cq_attr cq_attr = {0};
cntr_attr.events = FI_CNTR_EVENTS_COMP;
/* -------------------------------------------------------*/
/* Define Completion tracking Resources to Attach to EP */
/* -------------------------------------------------------*/
// Create counter for counting completions of outgoing writes
ret = fi_cntr_open(shmem_transport_ofi_domainfd, &cntr_attr,
&shmem_transport_ofi_put_cntrfd, NULL);
if(ret!=0){
OFI_ERRMSG("put cntr_open failed\n");
return ret;
}
// Create counter for counting completions of outbound reads
ret = fi_cntr_open(shmem_transport_ofi_domainfd, &cntr_attr,
&shmem_transport_ofi_get_cntrfd, NULL);
if(ret!=0){
OFI_ERRMSG("get cntr_open failed\n");
return ret;
}
/* Create CQ to be used for NB puts */
cq_attr.format = FI_CQ_FORMAT_CONTEXT;//event type for CQ,only context stored/reported
cq_attr.size = shmem_transport_ofi_queue_slots;
ret = fi_cq_open(shmem_transport_ofi_domainfd, &cq_attr,
&shmem_transport_ofi_put_nb_cqfd, NULL);
if(ret!=0){
OFI_ERRMSG("cq_open failed\n");
return ret;
}
return ret;
}
static void fas_ep_setup(void)
{
int ret, i, j;
size_t addrlen = 0;
fas_setup_common(fi_version());
ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt);
ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt);
for (i = 0; i < NUMEPS; i++) {
fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt;
fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt;
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain returned: %s", fi_strerror(-ret));
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));
switch (ep_type) {
case EP:
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint returned: %s",
fi_strerror(-ret));
break;
case SEP:
ret = fi_scalable_ep(dom[i], fi[i], ep + i,
NULL);
cr_assert(!ret, "fi_endpoint returned: %s",
fi_strerror(-ret));
break;
case PEP:
ret = fi_passive_ep(fab, fi[i], pep + i,
NULL);
cr_assert(!ret, "fi_endpoint returned: %s",
fi_strerror(-ret));
ret = fi_getname(get_fid[ep_type](i), NULL,
&addrlen);
if (use_str_fmt) {
cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN,
"fi_getname returned: %s",
fi_strerror(-ret));
} else {
cr_assert(addrlen ==
sizeof(struct gnix_ep_name),
"fi_getname returned: %s",
fi_strerror(-ret));
}
ep_name_len[i] = addrlen;
continue;
default:
cr_assert_fail("Unknown endpoint type.");
}
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open returned: %s", fi_strerror(-ret));
switch (ep_type) {
case EP:
case PEP:
ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i,
0);
cr_assert(!ret, "fi_cq_open returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(ep[i], &msg_cq[i]->fid,
FI_SEND | FI_RECV);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
break;
case SEP:
dbg_printf(BLUE
"ctx_cnt = %d\n"
COLOR_RESET,
ctx_cnt);
for (j = 0; j < ctx_cnt; j++) {
ret = fi_tx_context(ep[i], j, NULL,
&tx_ep[i][j], NULL);
cr_assert(!ret,
"fi_tx_context returned: %s",
fi_strerror(-ret));
ret = fi_cq_open(dom[i], &cq_attr,
&tx_cq[i][j],
NULL);
cr_assert(!ret,
"fi_cq_open returned: %s",
fi_strerror(-ret));
ret = fi_rx_context(ep[i], j, NULL,
&rx_ep[i][j], NULL);
cr_assert(!ret,
"fi_rx_context returned: %s",
fi_strerror(-ret));
ret = fi_cq_open(dom[i], &cq_attr,
&rx_cq[i][j],
NULL);
cr_assert(!ret,
"fi_cq_open returned: %s",
fi_strerror(-ret));
}
break;
default:
cr_assert_fail("Unknown endpoint type.");
}
ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen);
if (use_str_fmt) {
cr_assert(addrlen > sizeof(struct gnix_ep_name),
"fi_getname returned: %s",
fi_strerror(-ret));
} else {
cr_assert(addrlen == sizeof(struct gnix_ep_name),
"fi_getname returned: %s",
fi_strerror(-ret));
}
ep_name[i] = malloc(addrlen);
ep_name_len[i] = addrlen;
dbg_printf(BLUE
"ep_name_len[%d] = %lu\n"
COLOR_RESET, i,
ep_name_len[i]);
cr_assert(ep_name[i] != NULL, "malloc returned: %s",
strerror(errno));
ret = fi_getname(get_fid[ep_type](i), ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS, "fi_getname returned: %s",
fi_strerror(-ret));
}
/* Just testing setname / getname for passive endpoints */
if (ep_type == PEP)
return;
for (i = 0; i < NUMEPS; i++) {
/*Insert all gni addresses into each av*/
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1, "fi_av_insert returned: %s",
fi_strerror(-ret));
}
switch (ep_type) {
case EP:
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(ep[i], &send_cntr[i]->fid,
FI_SEND);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid,
FI_RECV);
cr_assert(!ret, "fi_ep_bind returned: %s",
fi_strerror(-ret));
break;
case SEP:
ret = fi_scalable_ep_bind(ep[i], &av[i]->fid,
0);
cr_assert(!ret,
"fi_scalable_ep_bind returned: %s",
fi_strerror(-ret));
dbg_printf(BLUE
"ctx_cnt = %d\n"
COLOR_RESET,
ctx_cnt);
for (j = 0; j < ctx_cnt; j++) {
ret = fi_ep_bind(tx_ep[i][j],
&tx_cq[i][j]->fid,
FI_TRANSMIT);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(tx_ep[i][j],
&send_cntr[i]->fid,
FI_SEND);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_enable(tx_ep[i][j]);
cr_assert(!ret,
"fi_enable returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(rx_ep[i][j],
&rx_cq[i][j]->fid,
FI_RECV);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_ep_bind(rx_ep[i][j],
&recv_cntr[i]->fid,
FI_RECV);
cr_assert(!ret,
"fi_ep_bind returned: %s",
fi_strerror(-ret));
ret = fi_enable(rx_ep[i][j]);
cr_assert(!ret,
"fi_enable returned: %s",
fi_strerror(-ret));
}
break;
case PEP:
break;
default:
cr_assert_fail("Unknown endpoint type.");
}
ret = fi_enable(ep[i]);
cr_assert(!ret, "fi_ep_enable returned: %s", fi_strerror(-ret));
if (ep_type != SEP) {
ret = fi_enable(ep[i]);
cr_assert_eq(ret, -FI_EOPBADSTATE,
"fi_enable returned: %s",
fi_strerror(-ret));
}
}
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cntr_attr cntr_attr;
struct fi_av_attr av_attr;
uint64_t flags = 0;
int ret;
buffer_size = MAX(sizeof(char *) * strlen(welcome_text),
sizeof(uint64_t));
buf = malloc(buffer_size);
if (!buf) {
perror("malloc");
return -1;
}
memset(&cntr_attr, 0, sizeof cntr_attr);
cntr_attr.events = FI_CNTR_EVENTS_COMP;
ret = fi_cntr_open(dom, &cntr_attr, &scntr, NULL);
if (ret) {
FT_PRINTERR("fi_cntr_open", ret);
goto err1;
}
ret = fi_cntr_open(dom, &cntr_attr, &rcntr, NULL);
if (ret) {
FT_PRINTERR("fi_cntr_open", ret);
goto err2;
}
/* Set FI_MR_KEY to associate the memory region with the specified key
* Set FI_MR_OFFSET to use specified offset as the base address */
flags = FI_MR_KEY | FI_MR_OFFSET;
ret = fi_mr_reg(dom, buf, buffer_size, FI_REMOTE_WRITE, 0,
user_defined_key, flags, &mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
goto err3;
}
memset(&av_attr, 0, sizeof av_attr);
av_attr.type = fi->domain_attr->av_type ?
fi->domain_attr->av_type : FI_AV_MAP;
av_attr.count = 1;
av_attr.name = NULL;
ret = fi_av_open(dom, &av_attr, &av, NULL);
if (ret) {
FT_PRINTERR("fi_av_open", ret);
goto err4;
}
ret = fi_endpoint(dom, fi, &ep, NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
goto err5;
}
return 0;
err5:
fi_close(&av->fid);
err4:
fi_close(&mr->fid);
err3:
fi_close(&rcntr->fid);
err2:
fi_close(&scntr->fid);
err1:
free(buf);
return ret;
}
void rdm_sr_setup_common_eps(void)
{
int ret = 0, i = 0, j = 0;
struct fi_av_attr attr;
size_t addrlen = 0;
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
assert(target);
source = malloc(BUF_SZ);
assert(source);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
cr_assert(!ret, "fi_cq_open");
ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep[i]);
cr_assert(!ret, "fi_ep_enable");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
cr_assert(!ret, "fi_ep_bind");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
cr_assert(!ret, "fi_ep_bind");
}
}
void rdm_api_setup_ep(void)
{
int ret, i, j;
struct fi_av_attr attr;
size_t addrlen = 0;
/* Get info about fabric services with the provided hints */
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
&fi[i]);
cr_assert(!ret, "fi_getinfo");
}
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
assert(target);
source = malloc(BUF_SZ);
assert(source);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (i = 0; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
cr_assert(!ret, "fi_cq_open");
ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep[i]);
cr_assert(!ret, "fi_ep_enable");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
cr_assert(!ret, "fi_ep_bind");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
cr_assert(!ret, "fi_ep_bind");
}
for (i = 0; i < NUMEPS; i++) {
ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i], source, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
cr_assert_eq(ret, 0);
mr_key[i] = fi_mr_key(rem_mr[i]);
}
}
void rdm_rma_setup(void)
{
int ret = 0;
struct fi_av_attr attr;
size_t addrlen = 0;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = 4;
hints->mode = ~0;
hints->fabric_attr->name = strdup("gni");
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
ret = fi_domain(fab, fi, &dom, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) &gni_domain_ops, NULL);
attr.type = FI_AV_MAP;
attr.count = 16;
ret = fi_av_open(dom, &attr, &av, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom, fi, &ep[0], NULL);
cr_assert(!ret, "fi_endpoint");
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
ret = fi_cq_open(dom, &cq_attr, &send_cq, 0);
cr_assert(!ret, "fi_cq_open");
/*
* imitate shmem, etc. use FI_WRITE for bind
* flag
*/
ret = fi_ep_bind(ep[0], &send_cq->fid, FI_TRANSMIT);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[0] = malloc(addrlen);
cr_assert(ep_name[0] != NULL);
ep_name[1] = malloc(addrlen);
cr_assert(ep_name[1] != NULL);
ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
cr_assert(ret == FI_SUCCESS);
ret = fi_endpoint(dom, fi, &ep[1], NULL);
cr_assert(!ret, "fi_endpoint");
cq_attr.format = FI_CQ_FORMAT_TAGGED;
ret = fi_cq_open(dom, &cq_attr, &recv_cq, 0);
cr_assert(!ret, "fi_cq_open");
ret = fi_ep_bind(ep[1], &recv_cq->fid, FI_RECV);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
cr_assert(ret == FI_SUCCESS);
ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
NULL);
cr_assert(ret == 1);
ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
NULL);
cr_assert(ret == 1);
ret = fi_ep_bind(ep[0], &av->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_ep_bind(ep[1], &av->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep[0]);
cr_assert(!ret, "fi_ep_enable");
ret = fi_enable(ep[1]);
cr_assert(!ret, "fi_ep_enable");
target = malloc(BUF_SZ);
assert(target);
source = malloc(BUF_SZ);
assert(source);
ret = fi_mr_reg(dom, target, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom, source, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source);
cr_assert_eq(ret, 0);
uc_source = malloc(BUF_SZ);
assert(uc_source);
mr_key = fi_mr_key(rem_mr);
ret = fi_cntr_open(dom, &cntr_attr, &write_cntr, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[0], &write_cntr->fid, FI_WRITE);
cr_assert(!ret, "fi_ep_bind");
ret = fi_cntr_open(dom, &cntr_attr, &read_cntr, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[0], &read_cntr->fid, FI_READ);
cr_assert(!ret, "fi_ep_bind");
writes = reads = write_errs = read_errs = 0;
}
int ft_alloc_active_res(struct fi_info *fi)
{
int ret;
ret = ft_alloc_msgs();
if (ret)
return ret;
if (cq_attr.format == FI_CQ_FORMAT_UNSPEC) {
if (fi->caps & FI_TAGGED)
cq_attr.format = FI_CQ_FORMAT_TAGGED;
else
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
}
if (opts.options & FT_OPT_TX_CQ) {
ft_cq_set_wait_attr();
cq_attr.size = fi->tx_attr->size;
ret = fi_cq_open(domain, &cq_attr, &txcq, &txcq);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
return ret;
}
if (opts.comp_method == FT_COMP_WAIT_FD) {
ret = fi_control(&txcq->fid, FI_GETWAIT, (void *) &tx_fd);
if (ret) {
FT_PRINTERR("fi_control(FI_GETWAIT)", ret);
return ret;
}
}
}
if (opts.options & FT_OPT_TX_CNTR) {
ft_cntr_set_wait_attr();
ret = fi_cntr_open(domain, &cntr_attr, &txcntr, &txcntr);
if (ret) {
FT_PRINTERR("fi_cntr_open", ret);
return ret;
}
}
if (opts.options & FT_OPT_RX_CQ) {
ft_cq_set_wait_attr();
cq_attr.size = fi->rx_attr->size;
ret = fi_cq_open(domain, &cq_attr, &rxcq, &rxcq);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
return ret;
}
if (opts.comp_method == FT_COMP_WAIT_FD) {
ret = fi_control(&rxcq->fid, FI_GETWAIT, (void *) &rx_fd);
if (ret) {
FT_PRINTERR("fi_control(FI_GETWAIT)", ret);
return ret;
}
}
}
if (opts.options & FT_OPT_RX_CNTR) {
ft_cntr_set_wait_attr();
ret = fi_cntr_open(domain, &cntr_attr, &rxcntr, &rxcntr);
if (ret) {
FT_PRINTERR("fi_cntr_open", ret);
return ret;
}
}
if (fi->ep_attr->type == FI_EP_RDM || fi->ep_attr->type == FI_EP_DGRAM) {
if (fi->domain_attr->av_type != FI_AV_UNSPEC)
av_attr.type = fi->domain_attr->av_type;
if (opts.av_name) {
av_attr.name = opts.av_name;
}
ret = fi_av_open(domain, &av_attr, &av, NULL);
if (ret) {
FT_PRINTERR("fi_av_open", ret);
return ret;
}
}
ret = fi_endpoint(domain, fi, &ep, NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
return ret;
}
return 0;
}
static inline void __api_cntr_setup(uint32_t version, int mr_mode)
{
int ret, i, j;
struct fi_av_attr attr = {0};
size_t addrlen = 0;
for (i = 0; i < NUMEPS; i++) {
hints[i] = fi_allocinfo();
cr_assert(hints[i], "fi_allocinfo");
hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
hints[i]->mode = mode_bits;
hints[i]->fabric_attr->prov_name = strdup("gni");
hints[i]->domain_attr->mr_mode = mr_mode;
}
/* Get info about fabric services with the provided hints */
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(version, NULL, 0, 0, hints[i],
&fi[i]);
cr_assert(!ret, "fi_getinfo");
}
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
/* 3x BUF_SZ for multi recv testing */
target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ * 3));
assert(target_base);
target = GNIT_ALIGN_BUFFER(char *, target_base);
source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
assert(source_base);
source = GNIT_ALIGN_BUFFER(char *, source_base);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (i = 0; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
}
for (i = 0; i < NUMEPS; i++) {
int target_requested_key =
USING_SCALABLE(fi[i]) ? (i * 2) : 0;
int source_requested_key =
USING_SCALABLE(fi[i]) ? (i * 2) + 1 : 0;
ret = fi_mr_reg(dom[i],
target,
3 * BUF_SZ,
FI_REMOTE_WRITE,
0,
target_requested_key,
0,
rem_mr + i,
&target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i],
source,
BUF_SZ,
FI_REMOTE_WRITE,
0,
source_requested_key,
0,
loc_mr + i,
&source);
cr_assert_eq(ret, 0);
if (USING_SCALABLE(fi[i])) {
MR_ENABLE(rem_mr[i], target, 3 * BUF_SZ);
MR_ENABLE(loc_mr[i], source, BUF_SZ);
}
mr_key[i] = fi_mr_key(rem_mr[i]);
}
}
int ft_alloc_ep_res(struct fi_info *fi)
{
int ret;
if (hints->caps & FI_RMA) {
ret = ft_set_rma_caps(fi, opts.rma_op);
if (ret)
return ret;
}
ret = ft_alloc_msgs();
if (ret)
return ret;
if (cq_attr.format == FI_CQ_FORMAT_UNSPEC) {
if (fi->caps & FI_TAGGED)
cq_attr.format = FI_CQ_FORMAT_TAGGED;
else
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
}
if (opts.options & FT_OPT_TX_CQ) {
ft_cq_set_wait_attr();
cq_attr.size = fi->tx_attr->size;
ret = fi_cq_open(domain, &cq_attr, &txcq, &txcq);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
return ret;
}
}
if (opts.options & FT_OPT_TX_CNTR) {
ft_cntr_set_wait_attr();
ret = fi_cntr_open(domain, &cntr_attr, &txcntr, &txcntr);
if (ret) {
FT_PRINTERR("fi_cntr_open", ret);
return ret;
}
}
if (opts.options & FT_OPT_RX_CQ) {
ft_cq_set_wait_attr();
cq_attr.size = fi->rx_attr->size;
ret = fi_cq_open(domain, &cq_attr, &rxcq, &rxcq);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
return ret;
}
}
if (opts.options & FT_OPT_RX_CNTR) {
ft_cntr_set_wait_attr();
ret = fi_cntr_open(domain, &cntr_attr, &rxcntr, &rxcntr);
if (ret) {
FT_PRINTERR("fi_cntr_open", ret);
return ret;
}
}
if (fi->ep_attr->type == FI_EP_RDM || fi->ep_attr->type == FI_EP_DGRAM) {
if (fi->domain_attr->av_type != FI_AV_UNSPEC)
av_attr.type = fi->domain_attr->av_type;
if (opts.av_name) {
av_attr.name = opts.av_name;
}
ret = fi_av_open(domain, &av_attr, &av, NULL);
if (ret) {
FT_PRINTERR("fi_av_open", ret);
return ret;
}
}
return 0;
}
void api_cntr_setup(void)
{
int ret, i, j;
struct fi_av_attr attr = {0};
size_t addrlen = 0;
for (i = 0; i < NUMEPS; i++) {
hints[i] = fi_allocinfo();
cr_assert(hints[i], "fi_allocinfo");
hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
hints[i]->mode = ~0;
hints[i]->fabric_attr->name = strdup("gni");
}
/* Get info about fabric services with the provided hints */
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
&fi[i]);
cr_assert(!ret, "fi_getinfo");
}
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
assert(target);
source = malloc(BUF_SZ);
assert(source);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (i = 0; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
}
for (i = 0; i < NUMEPS; i++) {
ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i], source, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
cr_assert_eq(ret, 0);
mr_key[i] = fi_mr_key(rem_mr[i]);
}
}
void sep_setup_common(int av_type)
{
int ret, i, j;
struct fi_av_attr av_attr = {0};
size_t addrlen = 0;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->ep_attr->type = FI_EP_RDM;
hints->caps = FI_ATOMIC | FI_RMA | FI_MSG | FI_NAMED_RX_CTX;
hints->mode = FI_LOCAL_MR;
hints->domain_attr->cq_data_size = NUMEPS * 2;
hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
hints->domain_attr->mr_mode = FI_MR_BASIC;
hints->fabric_attr->prov_name = strdup("gni");
hints->ep_attr->tx_ctx_cnt = ctx_cnt;
hints->ep_attr->rx_ctx_cnt = ctx_cnt;
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi[i]);
cr_assert(!ret, "fi_getinfo");
tx_cq[i] = calloc(ctx_cnt, sizeof(*tx_cq));
rx_cq[i] = calloc(ctx_cnt, sizeof(*rx_cq));
tx_ep[i] = calloc(ctx_cnt, sizeof(*tx_ep));
rx_ep[i] = calloc(ctx_cnt, sizeof(*rx_ep));
if (!tx_cq[i] || !tx_cq[i] ||
!tx_ep[i] || !rx_ep[i]) {
cr_assert(0, "calloc");
}
}
ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt);
ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt);
cr_assert(ctx_cnt, "ctx_cnt is 0");
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
rx_ctx_bits = 0;
while (ctx_cnt >> ++rx_ctx_bits);
av_attr.rx_ctx_bits = rx_ctx_bits;
av_attr.type = av_type;
av_attr.count = NUMEPS;
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = FI_WAIT_NONE;
rx_addr = calloc(ctx_cnt, sizeof(*rx_addr));
target = calloc(BUF_SZ, 1);
source = calloc(BUF_SZ, 1);
iov_src_buf = malloc(BUF_SZ * IOV_CNT);
iov_dest_buf = malloc(BUF_SZ * IOV_CNT);
src_iov = malloc(sizeof(struct iovec) * IOV_CNT);
dest_iov = malloc(sizeof(struct iovec) * IOV_CNT);
if (!rx_addr || !target || !source || !iov_src_buf || !iov_dest_buf ||
!src_iov || !dest_iov) {
cr_assert(0, "allocation");
}
for (i = 0; i < IOV_CNT; i++) {
src_iov[i].iov_base = malloc(BUF_SZ);
assert(src_iov[i].iov_base != NULL);
dest_iov[i].iov_base = malloc(BUF_SZ * 3);
assert(dest_iov[i].iov_base != NULL);
}
for (i = 0; i < NUMEPS; i++) {
fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt;
fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt;
ret = fi_domain(fab, fi[i], &dom[i], NULL);
cr_assert(!ret, "fi_domain");
ret = fi_scalable_ep(dom[i], fi[i], &sep[i], NULL);
cr_assert(!ret, "fi_scalable_ep");
ret = fi_av_open(dom[i], &av_attr, &av[i], NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_cntr_open(dom[i], &cntr_attr, &send_cntr[i], 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, &recv_cntr[i], 0);
cr_assert(!ret, "fi_cntr_open");
for (j = 0; j < ctx_cnt; j++) {
ret = fi_tx_context(sep[i], j, NULL, &tx_ep[i][j],
NULL);
cr_assert(!ret, "fi_tx_context");
ret = fi_cq_open(dom[i], &cq_attr, &tx_cq[i][j],
NULL);
cr_assert(!ret, "fi_cq_open");
ret = fi_rx_context(sep[i], j, NULL, &rx_ep[i][j],
NULL);
cr_assert(!ret, "fi_rx_context");
ret = fi_cq_open(dom[i], &cq_attr, &rx_cq[i][j],
NULL);
cr_assert(!ret, "fi_cq_open");
}
ret = fi_scalable_ep_bind(sep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_scalable_ep_bind");
for (j = 0; j < ctx_cnt; j++) {
ret = fi_ep_bind(tx_ep[i][j], &tx_cq[i][j]->fid,
FI_TRANSMIT);
cr_assert(!ret, "fi_ep_bind");
ret = fi_ep_bind(tx_ep[i][j], &send_cntr[i]->fid,
FI_SEND | FI_WRITE);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(tx_ep[i][j]);
cr_assert(!ret, "fi_enable");
ret = fi_ep_bind(rx_ep[i][j], &rx_cq[i][j]->fid,
FI_RECV);
cr_assert(!ret, "fi_ep_bind");
ret = fi_ep_bind(rx_ep[i][j], &recv_cntr[i]->fid,
FI_RECV | FI_READ);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(rx_ep[i][j]);
cr_assert(!ret, "fi_enable");
}
}
for (i = 0; i < NUMEPS; i++) {
ret = fi_enable(sep[i]);
cr_assert(!ret, "fi_enable");
ret = fi_getname(&sep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&sep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
ret = fi_mr_reg(dom[i], target, BUF_SZ, FI_REMOTE_WRITE,
0, 0, 0, &rem_mr[i], &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i], source, BUF_SZ, FI_REMOTE_WRITE,
0, 0, 0, &loc_mr[i], &source);
cr_assert_eq(ret, 0);
mr_key[i] = fi_mr_key(rem_mr[i]);
ret = fi_mr_reg(dom[i], iov_dest_buf, IOV_CNT * BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, iov_dest_buf_mr + i,
&iov_dest_buf);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i], iov_src_buf, IOV_CNT * BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, iov_src_buf_mr + i,
&iov_src_buf);
cr_assert_eq(ret, 0);
}
for (i = 0; i < NUMEPS; i++) {
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
}
for (i = 0; i < ctx_cnt; i++) {
rx_addr[i] = fi_rx_addr(gni_addr[1], i, rx_ctx_bits);
}
}
static inline int allocate_recv_cntr_mr(void)
{
int ret = 0;
/* ------------------------------------*/
/* POST enable resources for to EP */
/* ------------------------------------*/
/* since this is AFTER enable and RMA you must create memory regions for incoming reads/writes
* and outgoing non-blocking Puts, specifying entire VA range */
#ifndef ENABLE_HARD_POLLING
{
struct fi_cntr_attr cntr_attr = {0};
// Create counter for incoming writes
cntr_attr.events = FI_CNTR_EVENTS_COMP;
cntr_attr.flags = 0;
ret = fi_cntr_open(shmem_transport_ofi_domainfd, &cntr_attr,
&shmem_transport_ofi_target_cntrfd, NULL);
if(ret!=0){
OFI_ERRMSG("target cntr_open failed\n");
return ret;
}
}
#endif
#if defined(ENABLE_MR_SCALABLE) && defined(ENABLE_REMOTE_VIRTUAL_ADDRESSING)
ret = fi_mr_reg(shmem_transport_ofi_domainfd, 0, UINT64_MAX,
FI_REMOTE_READ | FI_REMOTE_WRITE, 0, 0ULL, 0,
&shmem_transport_ofi_target_mrfd, NULL);
if(ret!=0){
OFI_ERRMSG("mr_reg failed\n");
return ret;
}
// Bind counter with target memory region for incoming messages
#ifndef ENABLE_HARD_POLLING
ret = fi_mr_bind(shmem_transport_ofi_target_mrfd,
&shmem_transport_ofi_target_cntrfd->fid,
FI_REMOTE_WRITE | FI_REMOTE_READ);
if(ret!=0){
OFI_ERRMSG("mr_bind failed\n");
return ret;
}
ret = fi_ep_bind(shmem_transport_ofi_cntr_epfd,
&shmem_transport_ofi_target_cntrfd->fid, FI_REMOTE_WRITE | FI_REMOTE_READ);
if(ret!=0){
OFI_ERRMSG("ep_bind cntr_epfd2put_cntr failed\n");
return ret;
}
#endif /* ndef ENABLE_HARD_POLLING */
#else
/* Register separate data and heap segments using keys 0 and 1,
* respectively. In MR_BASIC_MODE, the keys are ignored and selected by
* the provider. */
ret = fi_mr_reg(shmem_transport_ofi_domainfd, shmem_internal_heap_base,
shmem_internal_heap_length,
FI_REMOTE_READ | FI_REMOTE_WRITE, 0, 1ULL, 0,
&shmem_transport_ofi_target_heap_mrfd, NULL);
if (ret != 0) {
OFI_ERRMSG("mr_reg heap failed\n");
return ret;
}
ret = fi_mr_reg(shmem_transport_ofi_domainfd, shmem_internal_data_base,
shmem_internal_data_length,
FI_REMOTE_READ | FI_REMOTE_WRITE, 0, 0ULL, 0,
&shmem_transport_ofi_target_data_mrfd, NULL);
if (ret != 0) {
OFI_ERRMSG("mr_reg data segment failed\n");
return ret;
}
/* Bind counter with target memory region for incoming messages */
#ifndef ENABLE_HARD_POLLING
ret = fi_mr_bind(shmem_transport_ofi_target_heap_mrfd,
&shmem_transport_ofi_target_cntrfd->fid,
FI_REMOTE_WRITE | FI_REMOTE_READ);
if (ret != 0) {
OFI_ERRMSG("mr_bind heap failed\n");
return ret;
}
ret = fi_mr_bind(shmem_transport_ofi_target_data_mrfd,
&shmem_transport_ofi_target_cntrfd->fid,
FI_REMOTE_WRITE | FI_REMOTE_READ);
if (ret != 0) {
OFI_ERRMSG("mr_bind data segment failed\n");
return ret;
}
ret = fi_ep_bind(shmem_transport_ofi_cntr_epfd,
&shmem_transport_ofi_target_cntrfd->fid, FI_REMOTE_WRITE | FI_REMOTE_READ);
if(ret!=0){
OFI_ERRMSG("ep_bind cntr_epfd2put_cntr failed\n");
return ret;
}
#endif /* ndef ENABLE_HARD_POLLING */
#endif
return ret;
}
