Test(domain, cache_flush_op) { int i, ret; const int num_doms = 11; struct fid_domain *doms[num_doms]; struct fi_gni_ops_domain *gni_domain_ops; struct fid_mr *mr; char *buf = calloc(1024, sizeof(char)); cr_assert(buf); memset(doms, 0, num_doms*sizeof(struct fid_domain *)); for (i = 0; i < num_doms; i++) { ret = fi_domain(fabric, fi, &doms[i], NULL); cr_assert(ret == FI_SUCCESS, "fi_domain"); ret = fi_open_ops(&doms[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) &gni_domain_ops, NULL); cr_assert(ret == FI_SUCCESS, "fi_open_ops"); ret = fi_mr_reg(doms[i], buf, 1024, FI_READ, 0, 0, 0, &mr, NULL); cr_assert(ret == FI_SUCCESS, "fi_reg_mr"); ret = fi_close(&mr->fid); cr_assert(ret == FI_SUCCESS, "fi_close mr"); ret = gni_domain_ops->flush_cache(&doms[i]->fid); cr_assert(ret == FI_SUCCESS, "flush cache"); ret = fi_close(&doms[i]->fid); cr_assert(ret == FI_SUCCESS, "fi_close domain"); } free(buf); }
Test(endpoint, getsetopt_gni_ep) { int ret; int val; struct fid_ep *ep = NULL; struct fi_gni_ops_ep *ep_ops; struct gnix_fid_ep *ep_priv = NULL; ret = fi_endpoint(dom, fi, &ep, NULL); cr_assert(!ret, "fi_endpoint"); ep_priv = (struct gnix_fid_ep *) ep; ret = fi_open_ops(&ep->fid, "ep ops 1", 0, (void **) &ep_ops, NULL); cr_assert(!ret, "fi_open_ops endpoint"); ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val); cr_assert(!ret, "ep_ops get_val"); cr_assert_eq(val, 0); cr_assert_eq(ep_priv->use_tag_hlist, 0); val = 1; // set the hash implementation ret = ep_ops->set_val(&ep->fid, GNI_HASH_TAG_IMPL, &val); cr_assert(!ret, "ep_ops set_val"); cr_assert_eq(ep_priv->use_tag_hlist, 1); cr_assert_eq(ep_priv->unexp_recv_queue.attr.type, GNIX_TAG_HLIST); cr_assert_eq(ep_priv->posted_recv_queue.attr.type, GNIX_TAG_HLIST); cr_assert_eq(ep_priv->tagged_unexp_recv_queue.attr.type, GNIX_TAG_HLIST); cr_assert_eq(ep_priv->tagged_posted_recv_queue.attr.type, GNIX_TAG_HLIST); val = 0; // reset the value ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val); cr_assert(!ret, "ep_ops get_val"); cr_assert_eq(val, 1); cr_assert_eq(ep_priv->use_tag_hlist, 1); ret = fi_close(&ep->fid); cr_assert(!ret, "fi_close endpoint"); }
Test(domain, invalid_open_ops) { int ret; struct fid_domain *dom; struct fi_gni_ops_domain *gni_domain_ops; uint32_t val = 0; ret = fi_domain(fabric, fi, &dom, NULL); cr_assert(ret == FI_SUCCESS, "fi_domain"); ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) &gni_domain_ops, NULL); cr_assert(ret == FI_SUCCESS, "fi_open_ops"); ret = gni_domain_ops->get_val(&dom->fid, GNI_NUM_DOM_OPS, &val); cr_assert(ret == -FI_EINVAL, "get_val"); ret = gni_domain_ops->set_val(&dom->fid, GNI_NUM_DOM_OPS, &val); cr_assert(ret == -FI_EINVAL, "set_val"); ret = fi_close(&dom->fid); cr_assert(ret == FI_SUCCESS, "fi_close domain"); }
void rdm_sr_setup_common_eps(void) { int ret = 0, i = 0, j = 0; struct fi_av_attr attr; size_t addrlen = 0; attr.type = FI_AV_MAP; attr.count = NUMEPS; cq_attr.format = FI_CQ_FORMAT_TAGGED; cq_attr.size = 1024; cq_attr.wait_obj = 0; target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */ assert(target); source = malloc(BUF_SZ); assert(source); uc_target = malloc(BUF_SZ); assert(uc_target); uc_source = malloc(BUF_SZ); assert(uc_source); ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); for (; i < NUMEPS; i++) { ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) (gni_domain_ops + i), NULL); ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[i]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); } for (i = 0; i < NUMEPS; i++) { /* Insert all gni addresses into each av */ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1); } ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable"); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind"); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind"); } }
void rdm_api_setup_ep(void) { int ret, i, j; struct fi_av_attr attr; size_t addrlen = 0; /* Get info about fabric services with the provided hints */ for (i = 0; i < NUMEPS; i++) { ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i], &fi[i]); cr_assert(!ret, "fi_getinfo"); } attr.type = FI_AV_MAP; attr.count = NUMEPS; cq_attr.format = FI_CQ_FORMAT_TAGGED; cq_attr.size = 1024; cq_attr.wait_obj = 0; target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */ assert(target); source = malloc(BUF_SZ); assert(source); uc_target = malloc(BUF_SZ); assert(uc_target); uc_source = malloc(BUF_SZ); assert(uc_source); ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); for (i = 0; i < NUMEPS; i++) { ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) (gni_domain_ops + i), NULL); ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[i]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); } for (i = 0; i < NUMEPS; i++) { /* Insert all gni addresses into each av */ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1); } ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable"); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND); cr_assert(!ret, "fi_ep_bind"); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind"); } for (i = 0; i < NUMEPS; i++) { ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom[i], source, BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source); cr_assert_eq(ret, 0); mr_key[i] = fi_mr_key(rem_mr[i]); } }
static void setup(void) { int i, j; int ret = 0; struct fi_av_attr attr; size_t addrlen = 0; struct fi_gni_ops_domain *gni_domain_ops; uint32_t rx_cq_size; hints = fi_allocinfo(); cr_assert(hints, "fi_allocinfo"); hints->domain_attr->cq_data_size = 4; hints->domain_attr->data_progress = FI_PROGRESS_MANUAL; hints->mode = ~0; hints->fabric_attr->name = strdup("gni"); ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi); cr_assert(!ret, "fi_getinfo"); ret = fi_fabric(fi->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); attr.type = FI_AV_TABLE; attr.count = NUM_EPS; cq_attr.format = FI_CQ_FORMAT_CONTEXT; cq_attr.size = 1024; cq_attr.wait_obj = 0; for (i = 0; i < NUM_EPS; i++) { ret = fi_domain(fab, fi, &dom[i], NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) &gni_domain_ops, NULL); cr_assert(ret == FI_SUCCESS, "fi_open_ops"); rx_cq_size = min_rx_cq_size; ret = gni_domain_ops->set_val(&dom[i]->fid, GNI_RX_CQ_SIZE, &rx_cq_size); cr_assert(ret == FI_SUCCESS, "set_val"); ret = fi_av_open(dom[i], &attr, &av[i], NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi, &ep[i], NULL); cr_assert(!ret, "fi_endpoint"); cr_assert(ep[i]); ret = fi_cq_open(dom[i], &cq_attr, &msg_cq[i], 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV); cr_assert(!ret, "fi_ep_bind"); } ret = fi_getname(&ep[0]->fid, NULL, &addrlen); cr_assert_eq(ret, -FI_ETOOSMALL); cr_assert(addrlen > 0); for (i = 0; i < NUM_EPS; i++) { ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); for (j = 0; j < NUM_EPS; j++) { ret = fi_av_insert(av[j], ep_name[i], 1, &gni_addr[i], 0, NULL); cr_assert(ret == 1); } } for (i = 0; i < NUM_EPS; i++) { ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[i]); cr_assert(!ret, "fi_ep_enable"); ret = fi_mr_reg(dom[i], target, NUM_EPS*sizeof(int), FI_RECV, 0, 0, 0, &rem_mr[i], &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom[i], source, NUM_EPS*sizeof(int), FI_SEND, 0, 0, 0, &loc_mr[i], &source); cr_assert_eq(ret, 0); mr_key[i] = fi_mr_key(rem_mr[i]); } }
void rdm_rma_setup(void) { int ret = 0; struct fi_av_attr attr; size_t addrlen = 0; hints = fi_allocinfo(); cr_assert(hints, "fi_allocinfo"); hints->domain_attr->cq_data_size = 4; hints->mode = ~0; hints->fabric_attr->name = strdup("gni"); ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi); cr_assert(!ret, "fi_getinfo"); ret = fi_fabric(fi->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); ret = fi_domain(fab, fi, &dom, NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) &gni_domain_ops, NULL); attr.type = FI_AV_MAP; attr.count = 16; ret = fi_av_open(dom, &attr, &av, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom, fi, &ep[0], NULL); cr_assert(!ret, "fi_endpoint"); cq_attr.format = FI_CQ_FORMAT_TAGGED; cq_attr.size = 1024; cq_attr.wait_obj = 0; ret = fi_cq_open(dom, &cq_attr, &send_cq, 0); cr_assert(!ret, "fi_cq_open"); /* * imitate shmem, etc. use FI_WRITE for bind * flag */ ret = fi_ep_bind(ep[0], &send_cq->fid, FI_TRANSMIT); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[0]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[0] = malloc(addrlen); cr_assert(ep_name[0] != NULL); ep_name[1] = malloc(addrlen); cr_assert(ep_name[1] != NULL); ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen); cr_assert(ret == FI_SUCCESS); ret = fi_endpoint(dom, fi, &ep[1], NULL); cr_assert(!ret, "fi_endpoint"); cq_attr.format = FI_CQ_FORMAT_TAGGED; ret = fi_cq_open(dom, &cq_attr, &recv_cq, 0); cr_assert(!ret, "fi_cq_open"); ret = fi_ep_bind(ep[1], &recv_cq->fid, FI_RECV); cr_assert(!ret, "fi_ep_bind"); ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen); cr_assert(ret == FI_SUCCESS); ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0, NULL); cr_assert(ret == 1); ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0, NULL); cr_assert(ret == 1); ret = fi_ep_bind(ep[0], &av->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_ep_bind(ep[1], &av->fid, 0); cr_assert(!ret, "fi_ep_bind"); ret = fi_enable(ep[0]); cr_assert(!ret, "fi_ep_enable"); ret = fi_enable(ep[1]); cr_assert(!ret, "fi_ep_enable"); target = malloc(BUF_SZ); assert(target); source = malloc(BUF_SZ); assert(source); ret = fi_mr_reg(dom, target, BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom, source, BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source); cr_assert_eq(ret, 0); uc_source = malloc(BUF_SZ); assert(uc_source); mr_key = fi_mr_key(rem_mr); ret = fi_cntr_open(dom, &cntr_attr, &write_cntr, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[0], &write_cntr->fid, FI_WRITE); cr_assert(!ret, "fi_ep_bind"); ret = fi_cntr_open(dom, &cntr_attr, &read_cntr, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_ep_bind(ep[0], &read_cntr->fid, FI_READ); cr_assert(!ret, "fi_ep_bind"); writes = reads = write_errs = read_errs = 0; }
Test(domain, open_ops) { int i, ret; const int num_doms = 11; struct fid_domain *doms[num_doms]; struct fi_gni_ops_domain *gni_domain_ops; enum dom_ops_val op; uint32_t val; char *other_reg_type = "none"; char *string_val; bool xpmem_toggle = false, xpmem_check; memset(doms, 0, num_doms*sizeof(struct fid_domain *)); for (i = 0; i < num_doms; i++) { ret = fi_domain(fabric, fi, &doms[i], NULL); cr_assert(ret == FI_SUCCESS, "fi_domain"); ret = fi_open_ops(&doms[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) &gni_domain_ops, NULL); cr_assert(ret == FI_SUCCESS, "fi_open_ops"); for (op = 0; op < GNI_NUM_DOM_OPS; op++) { val = i*op+op; switch (op) { case GNI_MR_CACHE: ret = gni_domain_ops->set_val(&doms[i]->fid, op, &other_reg_type); break; case GNI_XPMEM_ENABLE: ret = gni_domain_ops->set_val(&doms[i]->fid, op, &xpmem_toggle); break; default: ret = gni_domain_ops->set_val(&doms[i]->fid, op, &val); break; } cr_assert(ret == FI_SUCCESS, "set_val"); switch (op) { case GNI_MR_CACHE: ret = gni_domain_ops->get_val(&doms[i]->fid, op, &string_val); break; case GNI_XPMEM_ENABLE: ret = gni_domain_ops->get_val(&doms[i]->fid, op, &xpmem_check); break; default: ret = gni_domain_ops->get_val(&doms[i]->fid, op, &val); break; } cr_assert(ret == FI_SUCCESS, "get_val"); switch (op) { case GNI_MR_CACHE: cr_assert_eq(strncmp(other_reg_type, string_val, strlen(other_reg_type)), 0, "Incorrect op value"); break; case GNI_XPMEM_ENABLE: cr_assert(xpmem_toggle == xpmem_check, "Incorrect op value"); default: cr_assert(val == i*op+op, "Incorrect op value"); break; } } ret = fi_close(&doms[i]->fid); cr_assert(ret == FI_SUCCESS, "fi_close domain"); } }
static inline void __api_cntr_setup(uint32_t version, int mr_mode) { int ret, i, j; struct fi_av_attr attr = {0}; size_t addrlen = 0; for (i = 0; i < NUMEPS; i++) { hints[i] = fi_allocinfo(); cr_assert(hints[i], "fi_allocinfo"); hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO; hints[i]->mode = mode_bits; hints[i]->fabric_attr->prov_name = strdup("gni"); hints[i]->domain_attr->mr_mode = mr_mode; } /* Get info about fabric services with the provided hints */ for (i = 0; i < NUMEPS; i++) { ret = fi_getinfo(version, NULL, 0, 0, hints[i], &fi[i]); cr_assert(!ret, "fi_getinfo"); } attr.type = FI_AV_MAP; attr.count = NUMEPS; /* 3x BUF_SZ for multi recv testing */ target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ * 3)); assert(target_base); target = GNIT_ALIGN_BUFFER(char *, target_base); source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ)); assert(source_base); source = GNIT_ALIGN_BUFFER(char *, source_base); uc_target = malloc(BUF_SZ); assert(uc_target); uc_source = malloc(BUF_SZ); assert(uc_source); ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); for (i = 0; i < NUMEPS; i++) { ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) (gni_domain_ops + i), NULL); ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_getname(&ep[i]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); } for (i = 0; i < NUMEPS; i++) { /* Insert all gni addresses into each av */ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1); } ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); } for (i = 0; i < NUMEPS; i++) { int target_requested_key = USING_SCALABLE(fi[i]) ? (i * 2) : 0; int source_requested_key = USING_SCALABLE(fi[i]) ? (i * 2) + 1 : 0; ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ, FI_REMOTE_WRITE, 0, target_requested_key, 0, rem_mr + i, &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom[i], source, BUF_SZ, FI_REMOTE_WRITE, 0, source_requested_key, 0, loc_mr + i, &source); cr_assert_eq(ret, 0); if (USING_SCALABLE(fi[i])) { MR_ENABLE(rem_mr[i], target, 3 * BUF_SZ); MR_ENABLE(loc_mr[i], source, BUF_SZ); } mr_key[i] = fi_mr_key(rem_mr[i]); } }
void api_cntr_setup(void) { int ret, i, j; struct fi_av_attr attr = {0}; size_t addrlen = 0; for (i = 0; i < NUMEPS; i++) { hints[i] = fi_allocinfo(); cr_assert(hints[i], "fi_allocinfo"); hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO; hints[i]->mode = ~0; hints[i]->fabric_attr->name = strdup("gni"); } /* Get info about fabric services with the provided hints */ for (i = 0; i < NUMEPS; i++) { ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i], &fi[i]); cr_assert(!ret, "fi_getinfo"); } attr.type = FI_AV_MAP; attr.count = NUMEPS; target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */ assert(target); source = malloc(BUF_SZ); assert(source); uc_target = malloc(BUF_SZ); assert(uc_target); uc_source = malloc(BUF_SZ); assert(uc_source); ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL); cr_assert(!ret, "fi_fabric"); for (i = 0; i < NUMEPS; i++) { ret = fi_domain(fab, fi[i], dom + i, NULL); cr_assert(!ret, "fi_domain"); ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1, 0, (void **) (gni_domain_ops + i), NULL); ret = fi_av_open(dom[i], &attr, av + i, NULL); cr_assert(!ret, "fi_av_open"); ret = fi_endpoint(dom[i], fi[i], ep + i, NULL); cr_assert(!ret, "fi_endpoint"); ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0); cr_assert(!ret, "fi_cntr_open"); ret = fi_getname(&ep[i]->fid, NULL, &addrlen); cr_assert(addrlen > 0); ep_name[i] = malloc(addrlen); cr_assert(ep_name[i] != NULL); ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen); cr_assert(ret == FI_SUCCESS); } for (i = 0; i < NUMEPS; i++) { /* Insert all gni addresses into each av */ for (j = 0; j < NUMEPS; j++) { ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j], 0, NULL); cr_assert(ret == 1); } ret = fi_ep_bind(ep[i], &av[i]->fid, 0); cr_assert(!ret, "fi_ep_bind"); } for (i = 0; i < NUMEPS; i++) { ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target); cr_assert_eq(ret, 0); ret = fi_mr_reg(dom[i], source, BUF_SZ, FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source); cr_assert_eq(ret, 0); mr_key[i] = fi_mr_key(rem_mr[i]); } }