Test(domain, cache_flush_op)
{
int i, ret;
const int num_doms = 11;
struct fid_domain *doms[num_doms];
struct fi_gni_ops_domain *gni_domain_ops;
struct fid_mr *mr;
char *buf = calloc(1024, sizeof(char));
cr_assert(buf);
memset(doms, 0, num_doms*sizeof(struct fid_domain *));
for (i = 0; i < num_doms; i++) {
ret = fi_domain(fabric, fi, &doms[i], NULL);
cr_assert(ret == FI_SUCCESS, "fi_domain");
ret = fi_open_ops(&doms[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) &gni_domain_ops, NULL);
cr_assert(ret == FI_SUCCESS, "fi_open_ops");
ret = fi_mr_reg(doms[i], buf, 1024, FI_READ, 0, 0, 0, &mr, NULL);
cr_assert(ret == FI_SUCCESS, "fi_reg_mr");
ret = fi_close(&mr->fid);
cr_assert(ret == FI_SUCCESS, "fi_close mr");
ret = gni_domain_ops->flush_cache(&doms[i]->fid);
cr_assert(ret == FI_SUCCESS, "flush cache");
ret = fi_close(&doms[i]->fid);
cr_assert(ret == FI_SUCCESS, "fi_close domain");
}
free(buf);
}
Test(endpoint, getsetopt_gni_ep)
{
int ret;
int val;
struct fid_ep *ep = NULL;
struct fi_gni_ops_ep *ep_ops;
struct gnix_fid_ep *ep_priv = NULL;
ret = fi_endpoint(dom, fi, &ep, NULL);
cr_assert(!ret, "fi_endpoint");
ep_priv = (struct gnix_fid_ep *) ep;
ret = fi_open_ops(&ep->fid, "ep ops 1", 0, (void **) &ep_ops, NULL);
cr_assert(!ret, "fi_open_ops endpoint");
ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
cr_assert(!ret, "ep_ops get_val");
cr_assert_eq(val, 0);
cr_assert_eq(ep_priv->use_tag_hlist, 0);
val = 1; // set the hash implementation
ret = ep_ops->set_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
cr_assert(!ret, "ep_ops set_val");
cr_assert_eq(ep_priv->use_tag_hlist, 1);
cr_assert_eq(ep_priv->unexp_recv_queue.attr.type, GNIX_TAG_HLIST);
cr_assert_eq(ep_priv->posted_recv_queue.attr.type, GNIX_TAG_HLIST);
cr_assert_eq(ep_priv->tagged_unexp_recv_queue.attr.type, GNIX_TAG_HLIST);
cr_assert_eq(ep_priv->tagged_posted_recv_queue.attr.type, GNIX_TAG_HLIST);
val = 0; // reset the value
ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
cr_assert(!ret, "ep_ops get_val");
cr_assert_eq(val, 1);
cr_assert_eq(ep_priv->use_tag_hlist, 1);
ret = fi_close(&ep->fid);
cr_assert(!ret, "fi_close endpoint");
}
Test(domain, invalid_open_ops)
{
int ret;
struct fid_domain *dom;
struct fi_gni_ops_domain *gni_domain_ops;
uint32_t val = 0;
ret = fi_domain(fabric, fi, &dom, NULL);
cr_assert(ret == FI_SUCCESS, "fi_domain");
ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) &gni_domain_ops, NULL);
cr_assert(ret == FI_SUCCESS, "fi_open_ops");
ret = gni_domain_ops->get_val(&dom->fid, GNI_NUM_DOM_OPS, &val);
cr_assert(ret == -FI_EINVAL, "get_val");
ret = gni_domain_ops->set_val(&dom->fid, GNI_NUM_DOM_OPS, &val);
cr_assert(ret == -FI_EINVAL, "set_val");
ret = fi_close(&dom->fid);
cr_assert(ret == FI_SUCCESS, "fi_close domain");
}
void rdm_sr_setup_common_eps(void)
{
int ret = 0, i = 0, j = 0;
struct fi_av_attr attr;
size_t addrlen = 0;
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
assert(target);
source = malloc(BUF_SZ);
assert(source);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
cr_assert(!ret, "fi_cq_open");
ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep[i]);
cr_assert(!ret, "fi_ep_enable");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
cr_assert(!ret, "fi_ep_bind");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
cr_assert(!ret, "fi_ep_bind");
}
}
void rdm_api_setup_ep(void)
{
int ret, i, j;
struct fi_av_attr attr;
size_t addrlen = 0;
/* Get info about fabric services with the provided hints */
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
&fi[i]);
cr_assert(!ret, "fi_getinfo");
}
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
assert(target);
source = malloc(BUF_SZ);
assert(source);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (i = 0; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
cr_assert(!ret, "fi_cq_open");
ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep[i]);
cr_assert(!ret, "fi_ep_enable");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
cr_assert(!ret, "fi_ep_bind");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
cr_assert(!ret, "fi_ep_bind");
}
for (i = 0; i < NUMEPS; i++) {
ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i], source, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
cr_assert_eq(ret, 0);
mr_key[i] = fi_mr_key(rem_mr[i]);
}
}
static void setup(void)
{
int i, j;
int ret = 0;
struct fi_av_attr attr;
size_t addrlen = 0;
struct fi_gni_ops_domain *gni_domain_ops;
uint32_t rx_cq_size;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = 4;
hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
hints->mode = ~0;
hints->fabric_attr->name = strdup("gni");
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
attr.type = FI_AV_TABLE;
attr.count = NUM_EPS;
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
for (i = 0; i < NUM_EPS; i++) {
ret = fi_domain(fab, fi, &dom[i], NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0,
(void **) &gni_domain_ops, NULL);
cr_assert(ret == FI_SUCCESS, "fi_open_ops");
rx_cq_size = min_rx_cq_size;
ret = gni_domain_ops->set_val(&dom[i]->fid, GNI_RX_CQ_SIZE,
&rx_cq_size);
cr_assert(ret == FI_SUCCESS, "set_val");
ret = fi_av_open(dom[i], &attr, &av[i], NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi, &ep[i], NULL);
cr_assert(!ret, "fi_endpoint");
cr_assert(ep[i]);
ret = fi_cq_open(dom[i], &cq_attr, &msg_cq[i], 0);
cr_assert(!ret, "fi_cq_open");
ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
cr_assert(!ret, "fi_ep_bind");
}
ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
cr_assert_eq(ret, -FI_ETOOSMALL);
cr_assert(addrlen > 0);
for (i = 0; i < NUM_EPS; i++) {
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
for (j = 0; j < NUM_EPS; j++) {
ret = fi_av_insert(av[j], ep_name[i],
1, &gni_addr[i], 0, NULL);
cr_assert(ret == 1);
}
}
for (i = 0; i < NUM_EPS; i++) {
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep[i]);
cr_assert(!ret, "fi_ep_enable");
ret = fi_mr_reg(dom[i], target, NUM_EPS*sizeof(int),
FI_RECV, 0, 0, 0, &rem_mr[i], &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i], source, NUM_EPS*sizeof(int),
FI_SEND, 0, 0, 0, &loc_mr[i], &source);
cr_assert_eq(ret, 0);
mr_key[i] = fi_mr_key(rem_mr[i]);
}
}
void rdm_rma_setup(void)
{
int ret = 0;
struct fi_av_attr attr;
size_t addrlen = 0;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->domain_attr->cq_data_size = 4;
hints->mode = ~0;
hints->fabric_attr->name = strdup("gni");
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
ret = fi_domain(fab, fi, &dom, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) &gni_domain_ops, NULL);
attr.type = FI_AV_MAP;
attr.count = 16;
ret = fi_av_open(dom, &attr, &av, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom, fi, &ep[0], NULL);
cr_assert(!ret, "fi_endpoint");
cq_attr.format = FI_CQ_FORMAT_TAGGED;
cq_attr.size = 1024;
cq_attr.wait_obj = 0;
ret = fi_cq_open(dom, &cq_attr, &send_cq, 0);
cr_assert(!ret, "fi_cq_open");
/*
* imitate shmem, etc. use FI_WRITE for bind
* flag
*/
ret = fi_ep_bind(ep[0], &send_cq->fid, FI_TRANSMIT);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[0] = malloc(addrlen);
cr_assert(ep_name[0] != NULL);
ep_name[1] = malloc(addrlen);
cr_assert(ep_name[1] != NULL);
ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
cr_assert(ret == FI_SUCCESS);
ret = fi_endpoint(dom, fi, &ep[1], NULL);
cr_assert(!ret, "fi_endpoint");
cq_attr.format = FI_CQ_FORMAT_TAGGED;
ret = fi_cq_open(dom, &cq_attr, &recv_cq, 0);
cr_assert(!ret, "fi_cq_open");
ret = fi_ep_bind(ep[1], &recv_cq->fid, FI_RECV);
cr_assert(!ret, "fi_ep_bind");
ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
cr_assert(ret == FI_SUCCESS);
ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
NULL);
cr_assert(ret == 1);
ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
NULL);
cr_assert(ret == 1);
ret = fi_ep_bind(ep[0], &av->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_ep_bind(ep[1], &av->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep[0]);
cr_assert(!ret, "fi_ep_enable");
ret = fi_enable(ep[1]);
cr_assert(!ret, "fi_ep_enable");
target = malloc(BUF_SZ);
assert(target);
source = malloc(BUF_SZ);
assert(source);
ret = fi_mr_reg(dom, target, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom, source, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source);
cr_assert_eq(ret, 0);
uc_source = malloc(BUF_SZ);
assert(uc_source);
mr_key = fi_mr_key(rem_mr);
ret = fi_cntr_open(dom, &cntr_attr, &write_cntr, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[0], &write_cntr->fid, FI_WRITE);
cr_assert(!ret, "fi_ep_bind");
ret = fi_cntr_open(dom, &cntr_attr, &read_cntr, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[0], &read_cntr->fid, FI_READ);
cr_assert(!ret, "fi_ep_bind");
writes = reads = write_errs = read_errs = 0;
}
Test(domain, open_ops)
{
int i, ret;
const int num_doms = 11;
struct fid_domain *doms[num_doms];
struct fi_gni_ops_domain *gni_domain_ops;
enum dom_ops_val op;
uint32_t val;
char *other_reg_type = "none";
char *string_val;
bool xpmem_toggle = false, xpmem_check;
memset(doms, 0, num_doms*sizeof(struct fid_domain *));
for (i = 0; i < num_doms; i++) {
ret = fi_domain(fabric, fi, &doms[i], NULL);
cr_assert(ret == FI_SUCCESS, "fi_domain");
ret = fi_open_ops(&doms[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) &gni_domain_ops, NULL);
cr_assert(ret == FI_SUCCESS, "fi_open_ops");
for (op = 0; op < GNI_NUM_DOM_OPS; op++) {
val = i*op+op;
switch (op) {
case GNI_MR_CACHE:
ret = gni_domain_ops->set_val(&doms[i]->fid, op,
&other_reg_type);
break;
case GNI_XPMEM_ENABLE:
ret = gni_domain_ops->set_val(&doms[i]->fid, op,
&xpmem_toggle);
break;
default:
ret = gni_domain_ops->set_val(&doms[i]->fid, op, &val);
break;
}
cr_assert(ret == FI_SUCCESS, "set_val");
switch (op) {
case GNI_MR_CACHE:
ret = gni_domain_ops->get_val(&doms[i]->fid, op, &string_val);
break;
case GNI_XPMEM_ENABLE:
ret = gni_domain_ops->get_val(&doms[i]->fid, op,
&xpmem_check);
break;
default:
ret = gni_domain_ops->get_val(&doms[i]->fid, op, &val);
break;
}
cr_assert(ret == FI_SUCCESS, "get_val");
switch (op) {
case GNI_MR_CACHE:
cr_assert_eq(strncmp(other_reg_type, string_val,
strlen(other_reg_type)), 0, "Incorrect op value");
break;
case GNI_XPMEM_ENABLE:
cr_assert(xpmem_toggle == xpmem_check,
"Incorrect op value");
default:
cr_assert(val == i*op+op, "Incorrect op value");
break;
}
}
ret = fi_close(&doms[i]->fid);
cr_assert(ret == FI_SUCCESS, "fi_close domain");
}
}
static inline void __api_cntr_setup(uint32_t version, int mr_mode)
{
int ret, i, j;
struct fi_av_attr attr = {0};
size_t addrlen = 0;
for (i = 0; i < NUMEPS; i++) {
hints[i] = fi_allocinfo();
cr_assert(hints[i], "fi_allocinfo");
hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
hints[i]->mode = mode_bits;
hints[i]->fabric_attr->prov_name = strdup("gni");
hints[i]->domain_attr->mr_mode = mr_mode;
}
/* Get info about fabric services with the provided hints */
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(version, NULL, 0, 0, hints[i],
&fi[i]);
cr_assert(!ret, "fi_getinfo");
}
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
/* 3x BUF_SZ for multi recv testing */
target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ * 3));
assert(target_base);
target = GNIT_ALIGN_BUFFER(char *, target_base);
source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
assert(source_base);
source = GNIT_ALIGN_BUFFER(char *, source_base);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (i = 0; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
}
for (i = 0; i < NUMEPS; i++) {
int target_requested_key =
USING_SCALABLE(fi[i]) ? (i * 2) : 0;
int source_requested_key =
USING_SCALABLE(fi[i]) ? (i * 2) + 1 : 0;
ret = fi_mr_reg(dom[i],
target,
3 * BUF_SZ,
FI_REMOTE_WRITE,
0,
target_requested_key,
0,
rem_mr + i,
&target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i],
source,
BUF_SZ,
FI_REMOTE_WRITE,
0,
source_requested_key,
0,
loc_mr + i,
&source);
cr_assert_eq(ret, 0);
if (USING_SCALABLE(fi[i])) {
MR_ENABLE(rem_mr[i], target, 3 * BUF_SZ);
MR_ENABLE(loc_mr[i], source, BUF_SZ);
}
mr_key[i] = fi_mr_key(rem_mr[i]);
}
}
void api_cntr_setup(void)
{
int ret, i, j;
struct fi_av_attr attr = {0};
size_t addrlen = 0;
for (i = 0; i < NUMEPS; i++) {
hints[i] = fi_allocinfo();
cr_assert(hints[i], "fi_allocinfo");
hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
hints[i]->mode = ~0;
hints[i]->fabric_attr->name = strdup("gni");
}
/* Get info about fabric services with the provided hints */
for (i = 0; i < NUMEPS; i++) {
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
&fi[i]);
cr_assert(!ret, "fi_getinfo");
}
attr.type = FI_AV_MAP;
attr.count = NUMEPS;
target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
assert(target);
source = malloc(BUF_SZ);
assert(source);
uc_target = malloc(BUF_SZ);
assert(uc_target);
uc_source = malloc(BUF_SZ);
assert(uc_source);
ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
for (i = 0; i < NUMEPS; i++) {
ret = fi_domain(fab, fi[i], dom + i, NULL);
cr_assert(!ret, "fi_domain");
ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
0, (void **) (gni_domain_ops + i), NULL);
ret = fi_av_open(dom[i], &attr, av + i, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
cr_assert(!ret, "fi_endpoint");
ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
cr_assert(addrlen > 0);
ep_name[i] = malloc(addrlen);
cr_assert(ep_name[i] != NULL);
ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
cr_assert(ret == FI_SUCCESS);
}
for (i = 0; i < NUMEPS; i++) {
/* Insert all gni addresses into each av */
for (j = 0; j < NUMEPS; j++) {
ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
0, NULL);
cr_assert(ret == 1);
}
ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
cr_assert(!ret, "fi_ep_bind");
}
for (i = 0; i < NUMEPS; i++) {
ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
cr_assert_eq(ret, 0);
ret = fi_mr_reg(dom[i], source, BUF_SZ,
FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
cr_assert_eq(ret, 0);
mr_key[i] = fi_mr_key(rem_mr[i]);
}
}
