Ejemplo n.º 1
0
static int client_connect(void)
{
	struct fi_eq_cm_entry entry;
	uint32_t event;
	struct fi_info *fi;
	ssize_t rd;
	int ret;

	ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
	if (ret)
		return ret;

	ret = fi_getinfo(FT_FIVERSION, opts.dst_addr, opts.dst_port, 0, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		goto err0;
	}

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto err1;
	}

 	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		FT_PRINTERR("fi_domain", ret);
		goto err2;
	}

	ret = fi_endpoint(dom, fi, &ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		goto err3;
	}

	ret = alloc_ep_res(fi);
	if (ret)
		goto err3;

	ret = bind_ep_res();
	if (ret)
		goto err5;

	ret = fi_connect(ep, fi->dest_addr, NULL, 0);
	if (ret) {
		FT_PRINTERR("fi_connect", ret);
		goto err5;
	}

 	rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
	if (rd != sizeof entry) {
		FT_PRINTERR("fi_eq_sread", rd);
		return (int) rd;
	}

 	if (event != FI_CONNECTED || entry.fid != &ep->fid) {
 		fprintf(stderr, "Unexpected CM event %d fid %p (ep %p)\n",
 			event, entry.fid, ep);
 		ret = -FI_EOTHER;
 		goto err1;
 	}

	fi_freeinfo(fi);
	return 0;

err5:
	free_ep_res();
err3:
	fi_close(&dom->fid);
err2:
	fi_close(&fab->fid);
err1:
	fi_freeinfo(fi);
err0:
	return ret;
}
Ejemplo n.º 2
0
static int init_fabric(void)
{
	uint64_t flags = 0;
	char *node, *service;
	int ret;

	ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
	if (ret)
		return ret;
	
	ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	/* Check the number of EPs supported by the provider */
	if (ep_cnt > fi->domain_attr->ep_cnt) {
		ep_cnt = fi->domain_attr->ep_cnt;
		fprintf(stderr, "Provider can support only %d of EPs\n", ep_cnt);
	}

	/* Get remote address */
	if (opts.dst_addr) {
		addrlen = fi->dest_addrlen;
		remote_addr = malloc(addrlen * ep_cnt);
		memcpy(remote_addr, fi->dest_addr, addrlen);
	}

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto err0;
	}

	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		FT_PRINTERR("fi_domain", ret);
		goto err1;
	}

	fi->ep_attr->tx_ctx_cnt = FI_SHARED_CONTEXT;
	fi->ep_attr->rx_ctx_cnt = FI_SHARED_CONTEXT;

	ret = alloc_ep_res(fi);
	if (ret)
		goto err3;

	ret = bind_ep_res();
	if (ret)
		goto err4;

	return 0;

err4:
	free_ep_res();
err3:
	fi_close(&dom->fid);
err1:
	fi_close(&fab->fid);
err0:
	return ret;
}
Ejemplo n.º 3
0
static int init_fabric(void)
{
	struct fi_info *fi;
	uint64_t flags = 0;
	char *node, *service;
	int ret;

	if (opts.dst_addr) {
		ret = ft_getsrcaddr(opts.src_addr, opts.src_port, hints);
		if (ret)
			return ret;
		node = opts.dst_addr;
		service = opts.dst_port;
	} else {
		node = opts.src_addr;
		service = opts.src_port;
		flags = FI_SOURCE;
	}

	ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	/* We use provider MR attributes and direct address (no offsets) 
	 * for RMA calls */
	if (!(fi->mode & FI_PROV_MR_ATTR))
		fi->mode |= FI_PROV_MR_ATTR;

	/* Get remote address */
	if (opts.dst_addr) {
		addrlen = fi->dest_addrlen;
		remote_addr = malloc(addrlen);
		memcpy(remote_addr, fi->dest_addr, addrlen);
	}

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto err0;
	}

	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		FT_PRINTERR("fi_domain", ret);
		goto err1;
	}

	ret = alloc_ep_res(fi);
	if (ret)
		goto err3;

	ret = bind_ep_res();
	if (ret)
		goto err4;

	return 0;

err4:
	free_ep_res();
err3:
	fi_close(&dom->fid);
err1:
	fi_close(&fab->fid);
err0:
	return ret;
}
Ejemplo n.º 4
0
void rdm_sr_setup(void)
{
	int ret = 0;
	struct fi_av_attr attr;
	size_t addrlen = 0;

	hints = fi_allocinfo();
	cr_assert(hints, "fi_allocinfo");

	hints->domain_attr->cq_data_size = 4;
	hints->mode = ~0;

	hints->fabric_attr->name = strdup("gni");

	ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	attr.type = FI_AV_MAP;
	attr.count = 16;

	ret = fi_av_open(dom, &attr, &av, NULL);
	cr_assert(!ret, "fi_av_open");

	ret = fi_endpoint(dom, fi, &ep[0], NULL);
	cr_assert(!ret, "fi_endpoint");

	cq_attr.format = FI_CQ_FORMAT_CONTEXT;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	ret = fi_cq_open(dom, &cq_attr, &msg_cq[0], 0);
	cr_assert(!ret, "fi_cq_open");

	ret = fi_cq_open(dom, &cq_attr, &msg_cq[1], 0);
	cr_assert(!ret, "fi_cq_open");

	ret = fi_ep_bind(ep[0], &msg_cq[0]->fid, FI_SEND | FI_RECV);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
	cr_assert(addrlen > 0);

	ep_name[0] = malloc(addrlen);
	cr_assert(ep_name[0] != NULL);

	ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_endpoint(dom, fi, &ep[1], NULL);
	cr_assert(!ret, "fi_endpoint");

	ret = fi_ep_bind(ep[1], &msg_cq[1]->fid, FI_SEND | FI_RECV);
	cr_assert(!ret, "fi_ep_bind");

	ep_name[1] = malloc(addrlen);
	cr_assert(ep_name[1] != NULL);

	ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
				NULL);
	cr_assert(ret == 1);

	ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
				NULL);
	cr_assert(ret == 1);

	ret = fi_ep_bind(ep[0], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_ep_bind(ep[1], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_enable(ep[0]);
	cr_assert(!ret, "fi_ep_enable");

	ret = fi_enable(ep[1]);
	cr_assert(!ret, "fi_ep_enable");

	target = malloc(BUF_SZ);
	assert(target);

	source = malloc(BUF_SZ);
	assert(source);

	ret = fi_mr_reg(dom, target, BUF_SZ,
			FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target);
	cr_assert_eq(ret, 0);

	ret = fi_mr_reg(dom, source, BUF_SZ,
			FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source);
	cr_assert_eq(ret, 0);

	mr_key = fi_mr_key(rem_mr);
}
Ejemplo n.º 5
0
int main(int argc, char **argv)
{
	char *node, *service;
	uint64_t flags = 0;
	struct fi_eq_attr eq_attr;
	int op, ret;

	opts = INIT_OPTS;

	hints = fi_allocinfo();
	if (!hints)
		exit(1);

	hints->caps = FI_MSG;
	hints->ep_attr->type = FI_EP_MSG;
	hints->mode = FI_LOCAL_MR;
	hints->tx_attr->mode = hints->mode;
	hints->rx_attr->mode = hints->mode;

	hints->ep_attr->max_msg_size = 100;
	hints->tx_attr->size = 10;
	hints->rx_attr->size = 10;

	while ((op = getopt(argc, argv, "c:C:S:h" ADDR_OPTS INFO_OPTS)) != -1) {
		switch (op) {
		case 'c':
			connections = atoi(optarg);
			break;
		case 'C':
			hints->tx_attr->size = atoi(optarg);
			hints->rx_attr->size = hints->tx_attr->size;
			break;
		case 'S':
			hints->ep_attr->max_msg_size = atoi(optarg);
			break;
		default:
			ft_parse_addr_opts(op, optarg, &opts);
			ft_parseinfo(op, optarg, hints);
			break;
		case '?':
		case 'h':
			usage(argv[0]);
		}
	}

	if (optind < argc)
		opts.dst_addr = argv[optind];
	
	connects_left = connections;

	ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
	if (ret)
		return ret;
	
	ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &info);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		goto exit0;
	}

	printf("using provider: %s\n", info->fabric_attr->prov_name);
	ret = fi_fabric(info->fabric_attr, &fabric, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto exit1;
	}

	memset(&eq_attr, 0, sizeof eq_attr);
	eq_attr.wait_obj = FI_WAIT_UNSPEC;
	ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
	if (ret) {
		FT_PRINTERR("fi_eq_open", ret);
		goto exit2;
	}

	if (alloc_nodes())
		goto exit3;

	ret = opts.dst_addr ? run_client() : run_server();

	printf("test complete\n");
	destroy_nodes();


exit3:
	fi_close(&eq->fid);
exit2:
	fi_close(&fabric->fid);
exit1:
	fi_freeinfo(info);
exit0:
	fi_freeinfo(hints);
	return -ret;
}
Ejemplo n.º 6
0
void rdm_api_setup_ep(void)
{
	int ret, i, j;
	struct fi_av_attr attr;
	size_t addrlen = 0;

	/* Get info about fabric services with the provided hints */
	for (i = 0; i < NUMEPS; i++) {
		ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
				 &fi[i]);
		cr_assert(!ret, "fi_getinfo");
	}

	attr.type = FI_AV_MAP;
	attr.count = NUMEPS;

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
	assert(target);

	source = malloc(BUF_SZ);
	assert(source);

	uc_target = malloc(BUF_SZ);
	assert(uc_target);

	uc_source = malloc(BUF_SZ);
	assert(uc_source);

	ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	for (i = 0; i < NUMEPS; i++) {
		ret = fi_domain(fab, fi[i], dom + i, NULL);
		cr_assert(!ret, "fi_domain");

		ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
				  0, (void **) (gni_domain_ops + i), NULL);

		ret = fi_av_open(dom[i], &attr, av + i, NULL);
		cr_assert(!ret, "fi_av_open");

		ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
		cr_assert(!ret, "fi_endpoint");

		ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
		cr_assert(!ret, "fi_cq_open");

		ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
		cr_assert(addrlen > 0);

		ep_name[i] = malloc(addrlen);
		cr_assert(ep_name[i] != NULL);

		ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
		cr_assert(ret == FI_SUCCESS);
	}

	for (i = 0; i < NUMEPS; i++) {
		/* Insert all gni addresses into each av */
		for (j = 0; j < NUMEPS; j++) {
			ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
					   0, NULL);
			cr_assert(ret == 1);
		}

		ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_enable(ep[i]);
		cr_assert(!ret, "fi_ep_enable");

		ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
		cr_assert(!ret, "fi_ep_bind");
	}

	for (i = 0; i < NUMEPS; i++) {
		ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
				FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
		cr_assert_eq(ret, 0);

		ret = fi_mr_reg(dom[i], source, BUF_SZ,
				FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
		cr_assert_eq(ret, 0);

		mr_key[i] = fi_mr_key(rem_mr[i]);
	}
}
Ejemplo n.º 7
0
void api_cntr_setup(void)
{
    int ret, i, j;
    struct fi_av_attr attr = {0};
    size_t addrlen = 0;

    for (i = 0; i < NUMEPS; i++) {
        hints[i] = fi_allocinfo();
        cr_assert(hints[i], "fi_allocinfo");

        hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
        hints[i]->mode = ~0;
        hints[i]->fabric_attr->name = strdup("gni");
    }

    /* Get info about fabric services with the provided hints */
    for (i = 0; i < NUMEPS; i++) {
        ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints[i],
                         &fi[i]);
        cr_assert(!ret, "fi_getinfo");
    }

    attr.type = FI_AV_MAP;
    attr.count = NUMEPS;

    target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
    assert(target);

    source = malloc(BUF_SZ);
    assert(source);

    uc_target = malloc(BUF_SZ);
    assert(uc_target);

    uc_source = malloc(BUF_SZ);
    assert(uc_source);

    ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
    cr_assert(!ret, "fi_fabric");

    for (i = 0; i < NUMEPS; i++) {
        ret = fi_domain(fab, fi[i], dom + i, NULL);
        cr_assert(!ret, "fi_domain");

        ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
                          0, (void **) (gni_domain_ops + i), NULL);

        ret = fi_av_open(dom[i], &attr, av + i, NULL);
        cr_assert(!ret, "fi_av_open");

        ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
        cr_assert(!ret, "fi_endpoint");

        ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
        cr_assert(!ret, "fi_cntr_open");

        ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
        cr_assert(!ret, "fi_cntr_open");

        ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
        cr_assert(!ret, "fi_cntr_open");

        ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
        cr_assert(!ret, "fi_cntr_open");

        ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
        cr_assert(addrlen > 0);

        ep_name[i] = malloc(addrlen);
        cr_assert(ep_name[i] != NULL);

        ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
        cr_assert(ret == FI_SUCCESS);
    }

    for (i = 0; i < NUMEPS; i++) {
        /* Insert all gni addresses into each av */
        for (j = 0; j < NUMEPS; j++) {
            ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
                               0, NULL);
            cr_assert(ret == 1);
        }

        ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
        cr_assert(!ret, "fi_ep_bind");

    }

    for (i = 0; i < NUMEPS; i++) {
        ret = fi_mr_reg(dom[i], target, 3 * BUF_SZ,
                        FI_REMOTE_WRITE, 0, 0, 0, rem_mr + i, &target);
        cr_assert_eq(ret, 0);

        ret = fi_mr_reg(dom[i], source, BUF_SZ,
                        FI_REMOTE_WRITE, 0, 0, 0, loc_mr + i, &source);
        cr_assert_eq(ret, 0);

        mr_key[i] = fi_mr_key(rem_mr[i]);
    }
}
Ejemplo n.º 8
0
static inline void __api_cntr_setup(uint32_t version, int mr_mode)
{
	int ret, i, j;
	struct fi_av_attr attr = {0};
	size_t addrlen = 0;

	for (i = 0; i < NUMEPS; i++) {
		hints[i] = fi_allocinfo();
		cr_assert(hints[i], "fi_allocinfo");

		hints[i]->domain_attr->data_progress = FI_PROGRESS_AUTO;
		hints[i]->mode = mode_bits;
		hints[i]->fabric_attr->prov_name = strdup("gni");
		hints[i]->domain_attr->mr_mode = mr_mode;
	}

	/* Get info about fabric services with the provided hints */
	for (i = 0; i < NUMEPS; i++) {
		ret = fi_getinfo(version, NULL, 0, 0, hints[i],
				 &fi[i]);
		cr_assert(!ret, "fi_getinfo");
	}

	attr.type = FI_AV_MAP;
	attr.count = NUMEPS;

	/* 3x BUF_SZ for multi recv testing */
	target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ * 3));
	assert(target_base);
	target = GNIT_ALIGN_BUFFER(char *, target_base);

	source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
	assert(source_base);
	source = GNIT_ALIGN_BUFFER(char *, source_base);

	uc_target = malloc(BUF_SZ);
	assert(uc_target);

	uc_source = malloc(BUF_SZ);
	assert(uc_source);

	ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	for (i = 0; i < NUMEPS; i++) {
		ret = fi_domain(fab, fi[i], dom + i, NULL);
		cr_assert(!ret, "fi_domain");

		ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
				  0, (void **) (gni_domain_ops + i), NULL);

		ret = fi_av_open(dom[i], &attr, av + i, NULL);
		cr_assert(!ret, "fi_av_open");

		ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
		cr_assert(!ret, "fi_endpoint");

		ret = fi_cntr_open(dom[i], &cntr_attr, write_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_cntr_open(dom[i], &cntr_attr, read_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
		cr_assert(addrlen > 0);

		ep_name[i] = malloc(addrlen);
		cr_assert(ep_name[i] != NULL);

		ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
		cr_assert(ret == FI_SUCCESS);
	}

	for (i = 0; i < NUMEPS; i++) {
		/* Insert all gni addresses into each av */
		for (j = 0; j < NUMEPS; j++) {
			ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
					   0, NULL);
			cr_assert(ret == 1);
		}

		ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
		cr_assert(!ret, "fi_ep_bind");

	}

	 for (i = 0; i < NUMEPS; i++) {
		int target_requested_key =
			USING_SCALABLE(fi[i]) ? (i * 2) : 0;
		int source_requested_key =
			USING_SCALABLE(fi[i]) ? (i * 2) + 1 : 0;

		ret = fi_mr_reg(dom[i],
				  target,
				  3 * BUF_SZ,
				  FI_REMOTE_WRITE,
				  0,
				  target_requested_key,
				  0,
				  rem_mr + i,
				  &target);
		cr_assert_eq(ret, 0);

		ret = fi_mr_reg(dom[i],
				  source,
				  BUF_SZ,
				  FI_REMOTE_WRITE,
				  0,
				  source_requested_key,
				  0,
				  loc_mr + i,
				  &source);
		cr_assert_eq(ret, 0);

		if (USING_SCALABLE(fi[i])) {
			MR_ENABLE(rem_mr[i], target, 3 * BUF_SZ);
			MR_ENABLE(loc_mr[i], source, BUF_SZ);
		}
		mr_key[i] = fi_mr_key(rem_mr[i]);
	}
}
Ejemplo n.º 9
0
int main(int argc, char **argv)
{
	int i;
	int op, ret, num_domains = 1;

	hints = fi_allocinfo();
	if (hints == NULL)
		exit(EXIT_FAILURE);

	while ((op = getopt(argc, argv, "f:p:n:")) != -1) {
		switch (op) {
		case 'a':
			fabric_hints.name = strdup(optarg);
			break;
		case 'n':
			num_domains = atoi(optarg);
			break;
		case 'f':
			fabric_hints.prov_name = strdup(optarg);
			break;
		default:
			printf("usage: %s\n", argv[0]);
			printf("\t[-a fabric_name]\n");
			printf("\t[-f provider_name]\n");
			printf("\t[-n num domains to open]\n");
			exit(EXIT_FAILURE);
		}
	}

	hints->fabric_attr = &fabric_hints;
	hints->mode = ~0;

	ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
	if (ret != 0) {
		printf("fi_getinfo %s\n", fi_strerror(-ret));
		goto err;
	}

	ret = fi_fabric(fi->fabric_attr, &fabric, NULL);
	if (ret != 0) {
		printf("fi_fabric %s\n", fi_strerror(-ret));
		goto err;
	}

	domain_vec = calloc(num_domains,sizeof (struct fid_domain *));
	if (domain_vec == NULL) {
		perror("malloc");
		goto err;
	}

	for (i = 0; i < num_domains; i++) {
		ret = fi_domain(fabric, fi, &domain_vec[i], NULL);
		if (ret != FI_SUCCESS) {
			printf("fi_domain num %d %s\n", i, fi_strerror(-ret));
			goto err;
		}
	}

	for (i = 0; i < num_domains; i++) {
		ret = fi_close(&domain_vec[i]->fid);
		if (ret != FI_SUCCESS) {
			printf("Error %d closing domain num %d: %s\n", ret,
				i, fi_strerror(-ret));
			goto err;
		}
		domain_vec[i] = NULL;
	}
	free(domain_vec);
	domain_vec = NULL;

	ret = fi_close(&fabric->fid);
	if (ret != FI_SUCCESS) {
		printf("Error %d closing fabric: %s\n", ret, fi_strerror(-ret));
		exit(EXIT_FAILURE);
	}

	return ret;
err:
	if (domain_vec != NULL) {
		for (i=0;i<num_domains;i++) {
			if (domain_vec[i] != NULL) {
				ret = fi_close(&domain_vec[i]->fid);
				if (ret != FI_SUCCESS) {
					printf("Error in cleanup %d closing domain num %d: %s\n",
					       ret, i, fi_strerror(-ret));
				}
				domain_vec[i] = NULL;
			}
		}
		free(domain_vec);
		domain_vec = NULL;
	}

	if (fabric != NULL) {
		ret = fi_close(&fabric->fid);
		if (ret != FI_SUCCESS) {
			printf("Error in cleanup %d closing fabric: %s\n", ret,
			       fi_strerror(-ret));
		}
	}
	return -ret;
}
Ejemplo n.º 10
0
static mca_mtl_base_module_t*
ompi_mtl_ofi_component_init(bool enable_progress_threads,
                            bool enable_mpi_threads)
{
    int ret, fi_version;
    struct fi_info *hints;
    struct fi_info *providers = NULL, *prov = NULL;
    struct fi_cq_attr cq_attr = {0};
    struct fi_av_attr av_attr = {0};
    char ep_name[FI_NAME_MAX] = {0};
    size_t namelen;

    /**
     * Hints to filter providers
     * See man fi_getinfo for a list of all filters
     * mode:  Select capabilities MTL is prepared to support.
     *        In this case, MTL will pass in context into communication calls
     * ep_type:  reliable datagram operation
     * caps:     Capabilities required from the provider.
     *           Tag matching is specified to implement MPI semantics.
     * msg_order: Guarantee that messages with same tag are ordered.
     */
    hints = fi_allocinfo();
    if (!hints) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: Could not allocate fi_info\n",
                            __FILE__, __LINE__);
        goto error;
    }
    hints->mode               = FI_CONTEXT;
    hints->ep_attr->type      = FI_EP_RDM;      /* Reliable datagram         */
    hints->caps               = FI_TAGGED;      /* Tag matching interface    */
    hints->tx_attr->msg_order = FI_ORDER_SAS;
    hints->rx_attr->msg_order = FI_ORDER_SAS;

    hints->domain_attr->threading        = FI_THREAD_UNSPEC;
    hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;

    /**
     * FI_VERSION provides binary backward and forward compatibility support
     * Specify the version of OFI is coded to, the provider will select struct
     * layouts that are compatible with this version.
     */
    fi_version = FI_VERSION(1, 0);

    /**
     * fi_getinfo:  returns information about fabric  services for reaching a
     * remote node or service.  this does not necessarily allocate resources.
     * Pass NULL for name/service because we want a list of providers supported.
     */
    ret = fi_getinfo(fi_version,    /* OFI version requested                    */
                     NULL,          /* Optional name or fabric to resolve       */
                     NULL,          /* Optional service name or port to request */
                     0ULL,          /* Optional flag                            */
                     hints,        /* In: Hints to filter providers            */
                     &providers);   /* Out: List of matching providers          */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_getinfo failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Select a provider from the list returned by fi_getinfo().
     */
    prov = select_ofi_provider(providers);
    if (!prov) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: select_ofi_provider: no provider found\n",
                            __FILE__, __LINE__);
        goto error;
    }


    /**
     * Open fabric
     * The getinfo struct returns a fabric attribute struct that can be used to
     * instantiate the virtual or physical network. This opens a "fabric
     * provider". See man fi_fabric for details.
     */
    ret = fi_fabric(prov->fabric_attr,    /* In:  Fabric attributes             */
                    &ompi_mtl_ofi.fabric, /* Out: Fabric handle                 */
                    NULL);                /* Optional context for fabric events */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_fabric failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Create the access domain, which is the physical or virtual network or
     * hardware port/collection of ports.  Returns a domain object that can be
     * used to create endpoints.  See man fi_domain for details.
     */
    ret = fi_domain(ompi_mtl_ofi.fabric,  /* In:  Fabric object                 */
                    prov,                 /* In:  Provider                      */
                    &ompi_mtl_ofi.domain, /* Out: Domain oject                  */
                    NULL);                /* Optional context for domain events */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_domain failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Create a transport level communication endpoint.  To use the endpoint,
     * it must be bound to completion counters or event queues and enabled,
     * and the resources consumed by it, such as address vectors, counters,
     * completion queues, etc.
     * see man fi_endpoint for more details.
     */
    ret = fi_endpoint(ompi_mtl_ofi.domain, /* In:  Domain object   */
                      prov,                /* In:  Provider        */
                      &ompi_mtl_ofi.ep,    /* Out: Endpoint object */
                      NULL);               /* Optional context     */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_endpoint failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Save the maximum inject size.
     */
    ompi_mtl_ofi.max_inject_size = prov->tx_attr->inject_size;

    /**
     * Create the objects that will be bound to the endpoint.
     * The objects include:
     *     - completion queue for events
     *     - address vector of other endpoint addresses
     *     - dynamic memory-spanning memory region
     */
    cq_attr.format = FI_CQ_FORMAT_TAGGED;
    ret = fi_cq_open(ompi_mtl_ofi.domain, &cq_attr, &ompi_mtl_ofi.cq, NULL);
    if (ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_cq_open failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * The remote fi_addr will be stored in the ofi_endpoint struct.
     * So, we use the AV in "map" mode.
     */
    av_attr.type = FI_AV_MAP;
    ret = fi_av_open(ompi_mtl_ofi.domain, &av_attr, &ompi_mtl_ofi.av, NULL);
    if (ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_av_open failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Bind the CQ and AV to the endpoint object.
     */
    ret = fi_ep_bind(ompi_mtl_ofi.ep,
                     (fid_t)ompi_mtl_ofi.cq,
                     FI_SEND | FI_RECV);
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_bind CQ-EP failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    ret = fi_ep_bind(ompi_mtl_ofi.ep,
                     (fid_t)ompi_mtl_ofi.av,
                     0);
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_bind AV-EP failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Enable the endpoint for communication
     * This commits the bind operations.
     */
    ret = fi_enable(ompi_mtl_ofi.ep);
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_enable failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Free providers info since it's not needed anymore.
     */
    fi_freeinfo(hints);
    hints = NULL;
    fi_freeinfo(providers);
    providers = NULL;

    /**
     * Get our address and publish it with modex.
     */
    namelen = sizeof(ep_name);
    ret = fi_getname((fid_t)ompi_mtl_ofi.ep, &ep_name[0], &namelen);
    if (ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_getname failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    OFI_COMPAT_MODEX_SEND(ret,
                          &mca_mtl_ofi_component.super.mtl_version,
                          &ep_name,
                          namelen);
    if (OMPI_SUCCESS != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: modex_send failed: %d\n",
                            __FILE__, __LINE__, ret);
        goto error;
    }

    ompi_mtl_ofi.epnamelen = namelen;

    /**
     * Set the ANY_SRC address.
     */
    ompi_mtl_ofi.any_addr = FI_ADDR_UNSPEC;

    /**
     * Activate progress callback.
     */
    ret = opal_progress_register(ompi_mtl_ofi_progress_no_inline);
    if (OMPI_SUCCESS != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: opal_progress_register failed: %d\n",
                            __FILE__, __LINE__, ret);
        goto error;
    }

    return &ompi_mtl_ofi.base;

error:
    if (providers) {
        (void) fi_freeinfo(providers);
    }
    if (hints) {
        (void) fi_freeinfo(hints);
    }
    if (ompi_mtl_ofi.av) {
        (void) fi_close((fid_t)ompi_mtl_ofi.av);
    }
    if (ompi_mtl_ofi.cq) {
        (void) fi_close((fid_t)ompi_mtl_ofi.cq);
    }
    if (ompi_mtl_ofi.ep) {
        (void) fi_close((fid_t)ompi_mtl_ofi.ep);
    }
    if (ompi_mtl_ofi.domain) {
        (void) fi_close((fid_t)ompi_mtl_ofi.domain);
    }
    if (ompi_mtl_ofi.fabric) {
        (void) fi_close((fid_t)ompi_mtl_ofi.fabric);
    }
    return NULL;
}
Ejemplo n.º 11
0
void sep_setup_common(int av_type)
{
	int ret, i, j;
	struct fi_av_attr av_attr = {0};
	size_t addrlen = 0;

	hints = fi_allocinfo();
	cr_assert(hints, "fi_allocinfo");
	hints->ep_attr->type = FI_EP_RDM;
	hints->caps = FI_ATOMIC | FI_RMA | FI_MSG | FI_NAMED_RX_CTX;
	hints->mode = FI_LOCAL_MR;
	hints->domain_attr->cq_data_size = NUMEPS * 2;
	hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
	hints->domain_attr->mr_mode = FI_MR_BASIC;
	hints->fabric_attr->prov_name = strdup("gni");
	hints->ep_attr->tx_ctx_cnt = ctx_cnt;
	hints->ep_attr->rx_ctx_cnt = ctx_cnt;

	for (i = 0; i < NUMEPS; i++) {
		ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi[i]);
		cr_assert(!ret, "fi_getinfo");

		tx_cq[i] = calloc(ctx_cnt, sizeof(*tx_cq));
		rx_cq[i] = calloc(ctx_cnt, sizeof(*rx_cq));
		tx_ep[i] = calloc(ctx_cnt, sizeof(*tx_ep));
		rx_ep[i] = calloc(ctx_cnt, sizeof(*rx_ep));
		if (!tx_cq[i] || !tx_cq[i] ||
		    !tx_ep[i] || !rx_ep[i]) {
			cr_assert(0, "calloc");
		}
	}

	ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt);
	ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt);
	cr_assert(ctx_cnt, "ctx_cnt is 0");

	ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	rx_ctx_bits = 0;
	while (ctx_cnt >> ++rx_ctx_bits);
	av_attr.rx_ctx_bits = rx_ctx_bits;
	av_attr.type = av_type;
	av_attr.count = NUMEPS;

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = FI_WAIT_NONE;

	rx_addr = calloc(ctx_cnt, sizeof(*rx_addr));
	target = calloc(BUF_SZ, 1);
	source = calloc(BUF_SZ, 1);
	iov_src_buf = malloc(BUF_SZ * IOV_CNT);
	iov_dest_buf = malloc(BUF_SZ * IOV_CNT);
	src_iov = malloc(sizeof(struct iovec) * IOV_CNT);
	dest_iov = malloc(sizeof(struct iovec) * IOV_CNT);

	if (!rx_addr || !target || !source || !iov_src_buf || !iov_dest_buf ||
	    !src_iov || !dest_iov) {
		cr_assert(0, "allocation");
	}

	for (i = 0; i < IOV_CNT; i++) {
		src_iov[i].iov_base = malloc(BUF_SZ);
		assert(src_iov[i].iov_base != NULL);

		dest_iov[i].iov_base = malloc(BUF_SZ * 3);
		assert(dest_iov[i].iov_base != NULL);
	}

	for (i = 0; i < NUMEPS; i++) {
		fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt;
		fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt;

		ret = fi_domain(fab, fi[i], &dom[i], NULL);
		cr_assert(!ret, "fi_domain");

		ret = fi_scalable_ep(dom[i], fi[i], &sep[i], NULL);
		cr_assert(!ret, "fi_scalable_ep");

		ret = fi_av_open(dom[i], &av_attr, &av[i], NULL);
		cr_assert(!ret, "fi_av_open");

		ret = fi_cntr_open(dom[i], &cntr_attr, &send_cntr[i], 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_cntr_open(dom[i], &cntr_attr, &recv_cntr[i], 0);
		cr_assert(!ret, "fi_cntr_open");

		for (j = 0; j < ctx_cnt; j++) {
			ret = fi_tx_context(sep[i], j, NULL, &tx_ep[i][j],
					    NULL);
			cr_assert(!ret, "fi_tx_context");

			ret = fi_cq_open(dom[i], &cq_attr, &tx_cq[i][j],
					 NULL);
			cr_assert(!ret, "fi_cq_open");

			ret = fi_rx_context(sep[i], j, NULL, &rx_ep[i][j],
					    NULL);
			cr_assert(!ret, "fi_rx_context");

			ret = fi_cq_open(dom[i], &cq_attr, &rx_cq[i][j],
					 NULL);
			cr_assert(!ret, "fi_cq_open");
		}

		ret = fi_scalable_ep_bind(sep[i], &av[i]->fid, 0);
		cr_assert(!ret, "fi_scalable_ep_bind");

		for (j = 0; j < ctx_cnt; j++) {
			ret = fi_ep_bind(tx_ep[i][j], &tx_cq[i][j]->fid,
					 FI_TRANSMIT);
			cr_assert(!ret, "fi_ep_bind");

			ret = fi_ep_bind(tx_ep[i][j], &send_cntr[i]->fid,
					 FI_SEND | FI_WRITE);
			cr_assert(!ret, "fi_ep_bind");

			ret = fi_enable(tx_ep[i][j]);
			cr_assert(!ret, "fi_enable");

			ret = fi_ep_bind(rx_ep[i][j], &rx_cq[i][j]->fid,
					 FI_RECV);
			cr_assert(!ret, "fi_ep_bind");

			ret = fi_ep_bind(rx_ep[i][j], &recv_cntr[i]->fid,
					 FI_RECV | FI_READ);
			cr_assert(!ret, "fi_ep_bind");

			ret = fi_enable(rx_ep[i][j]);
			cr_assert(!ret, "fi_enable");

		}
	}

	for (i = 0; i < NUMEPS; i++) {
		ret = fi_enable(sep[i]);
		cr_assert(!ret, "fi_enable");

		ret = fi_getname(&sep[i]->fid, NULL, &addrlen);
		cr_assert(addrlen > 0);

		ep_name[i] = malloc(addrlen);
		cr_assert(ep_name[i] != NULL);

		ret = fi_getname(&sep[i]->fid, ep_name[i], &addrlen);
		cr_assert(ret == FI_SUCCESS);

		ret = fi_mr_reg(dom[i], target, BUF_SZ, FI_REMOTE_WRITE,
				0, 0, 0, &rem_mr[i], &target);
		cr_assert_eq(ret, 0);

		ret = fi_mr_reg(dom[i], source, BUF_SZ, FI_REMOTE_WRITE,
				0, 0, 0, &loc_mr[i], &source);
		cr_assert_eq(ret, 0);

		mr_key[i] = fi_mr_key(rem_mr[i]);

		ret = fi_mr_reg(dom[i], iov_dest_buf, IOV_CNT * BUF_SZ,
				FI_REMOTE_WRITE, 0, 0, 0, iov_dest_buf_mr + i,
				&iov_dest_buf);
		cr_assert_eq(ret, 0);

		ret = fi_mr_reg(dom[i], iov_src_buf, IOV_CNT * BUF_SZ,
				FI_REMOTE_WRITE, 0, 0, 0, iov_src_buf_mr + i,
				&iov_src_buf);
		cr_assert_eq(ret, 0);

	}

	for (i = 0; i < NUMEPS; i++) {
		for (j = 0; j < NUMEPS; j++) {
			ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
					   0, NULL);
			cr_assert(ret == 1);
		}
	}

	for (i = 0; i < ctx_cnt; i++) {
		rx_addr[i] = fi_rx_addr(gni_addr[1], i, rx_ctx_bits);
	}
}
Ejemplo n.º 12
0
static int init_fabric(void)
{
	struct fi_info *fi;
	char *node;
	uint64_t flags = 0;
	int ret;

	if (src_addr) {
		ret = getaddr(src_addr, NULL, 
				(struct sockaddr **) &hints.src_addr,
				(socklen_t *) &hints.src_addrlen);
		if (ret) {
			fprintf(stderr, "source address error %s\n", 
					gai_strerror(ret));
			return ret;
		}
	}

	if (dst_addr) {
		node = dst_addr;
	} else {
		node = src_addr;
		flags = FI_SOURCE;
	}

	ret = fi_getinfo(FI_VERSION(1, 0), node, port, flags, &hints, &fi);
	if (ret) {
		FI_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	// we use provider MR attributes and direct address (no offsets) 
	// for RMA calls
	if (!(fi->mode & FI_PROV_MR_ATTR))
		fi->mode |= FI_PROV_MR_ATTR;

	// get remote address
	if (dst_addr) {
		addrlen = fi->dest_addrlen;
		remote_addr = malloc(addrlen);
		memcpy(remote_addr, fi->dest_addr, addrlen);
	}

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FI_PRINTERR("fi_fabric", ret);
		goto err0;
	}

	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		FI_PRINTERR("fi_domain", ret);
		goto err1;
	}

	ret = fi_endpoint(dom, fi, &ep, NULL);
	if (ret) {
		FI_PRINTERR("fi_endpoint", ret);
		goto err2;
	}

	if (dst_addr == NULL) {
		printf("EP opened on fabric %s\n", fi->fabric_attr->name);
	}

	ret = alloc_ep_res(fi);
	if (ret)
		goto err3;
	
	ret = bind_ep_res();
	if (ret)
		goto err4;
	
	return 0;

err4:
	free_ep_res();
err3:
	fi_close(&ep->fid);
err2:
	fi_close(&dom->fid);
err1:
	fi_close(&fab->fid);
err0:
	fi_freeinfo(fi);
	
	return ret;
}
Ejemplo n.º 13
0
static void vc_setup_common(void)
{
	int ret = 0;
	struct fi_av_attr attr;
	size_t addrlen = 0;
	struct gnix_fid_av *gnix_av;

	hints->fabric_attr->name = strdup("gni");

	ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	attr.type = FI_AV_MAP;
	attr.count = 16;

	ret = fi_av_open(dom, &attr, &av, NULL);
	cr_assert(!ret, "fi_av_open");

	gnix_av = container_of(av, struct gnix_fid_av, av_fid);

	ret = fi_endpoint(dom, fi, &ep[0], NULL);
	cr_assert(!ret, "fi_endpoint");

	ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
	cr_assert(addrlen > 0);

	ep_name[0] = malloc(addrlen);
	cr_assert(ep_name[0] != NULL);

	ep_name[1] = malloc(addrlen);
	cr_assert(ep_name[1] != NULL);

	ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_endpoint(dom, fi, &ep[1], NULL);
	cr_assert(!ret, "fi_endpoint");

	ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
				NULL);
	cr_assert(ret == 1);

	ret = _gnix_av_lookup(gnix_av, gni_addr[0], &gnix_addr[0]);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
				NULL);
	cr_assert(ret == 1);

	ret = _gnix_av_lookup(gnix_av, gni_addr[1], &gnix_addr[1]);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_ep_bind(ep[0], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	ret = fi_cq_open(dom, &cq_attr, &cq, 0);
	cr_assert(!ret, "fi_cq_open");

	ret = fi_ep_bind(ep[0], &cq->fid, FI_SEND | FI_RECV);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_enable(ep[0]);
	cr_assert(!ret, "fi_enable");

	ret = fi_ep_bind(ep[1], &cq->fid, FI_SEND | FI_RECV);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_ep_bind(ep[1], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_enable(ep[1]);
	cr_assert(!ret, "fi_ep_enable");
}
Ejemplo n.º 14
0
static void libfabric_init() {
  int i;
  struct fi_info *info = NULL;
  struct fi_info *hints = fi_allocinfo();
  struct fi_av_attr av_attr = {0};
  struct fi_cq_attr cq_attr = {0};
  int max_tx_ctx, max_rx_ctx;
  int comm_concurrency;
  int rx_ctx_cnt;
  int rx_ctx_bits = 0;

  hints->mode = ~0;

  hints->caps = FI_RMA
             | FI_ATOMIC
             | FI_SOURCE /* do we want this? */
             | FI_READ
             | FI_WRITE
             | FI_REMOTE_READ
             | FI_REMOTE_WRITE
             | FI_MULTI_RECV
             | FI_FENCE;

  hints->addr_format = FI_FORMAT_UNSPEC;

#if defined(CHPL_COMM_SUBSTRATE_SOCKETS)
  //
  // fi_freeinfo(hints) will free() hints->fabric_attr->prov_name; this
  // is documented, though poorly.  So, get that space from malloc().
  //
  {
    const char s[] = "sockets";
    char* sDup = sys_malloc(sizeof(s));
    strcpy(sDup, s);
    hints->fabric_attr->prov_name = sDup;
  }
#elif defined(CHPL_COMM_SUBSTRATE_GNI)
#error "Substrate GNI not supported"
#else
#error "Substrate type not supported"
#endif

  /* connectionless reliable */
  hints->ep_attr->type = FI_EP_RDM;

  hints->domain_attr->threading = FI_THREAD_UNSPEC;
  hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
  hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
  hints->domain_attr->av_type = FI_AV_TABLE;
  hints->domain_attr->mr_mode = FI_MR_SCALABLE;
  hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
  // hints->domain_attr->cq_data_size

  hints->tx_attr->op_flags = FI_COMPLETION;
  hints->rx_attr->op_flags = FI_COMPLETION;

  OFICHKERR(fi_getinfo(FI_VERSION(1,0), NULL, NULL, 0, hints, &info));

  if (info == NULL) {
    chpl_internal_error("No fabrics detected.");
  } else {
#ifdef PRINT_FI_GETINFO
    struct fi_info *cur;
    for (cur = info; cur; cur = cur->next) {
      printf("---\n");
      printf("%s", fi_tostr(cur, FI_TYPE_INFO));
    }
    printf("\n");
#endif
  }

  ofi.num_am_ctx = 1; // Would we ever want more?

  max_tx_ctx = info->domain_attr->max_ep_tx_ctx;
  max_rx_ctx = info->domain_attr->max_ep_rx_ctx;
  comm_concurrency = get_comm_concurrency();

  ofi.num_tx_ctx = comm_concurrency+ofi.num_am_ctx > max_tx_ctx ?
    max_tx_ctx-ofi.num_am_ctx : comm_concurrency;
  ofi.num_rx_ctx = comm_concurrency+ofi.num_am_ctx > max_rx_ctx ?
    max_rx_ctx-ofi.num_am_ctx : comm_concurrency;

  info->ep_attr->tx_ctx_cnt = ofi.num_tx_ctx + ofi.num_am_ctx;
  info->ep_attr->rx_ctx_cnt = ofi.num_rx_ctx + ofi.num_am_ctx;

  OFICHKERR(fi_fabric(info->fabric_attr, &ofi.fabric, NULL));
  OFICHKERR(fi_domain(ofi.fabric, info, &ofi.domain, NULL));

  rx_ctx_cnt = ofi.num_rx_ctx + ofi.num_am_ctx;
  while (rx_ctx_cnt >> ++rx_ctx_bits);
  av_attr.rx_ctx_bits = rx_ctx_bits;
  av_attr.type = FI_AV_TABLE;
  av_attr.count = chpl_numNodes;
  OFICHKERR(fi_av_open(ofi.domain, &av_attr, &ofi.av, NULL));

  OFICHKERR(fi_scalable_ep(ofi.domain, info, &ofi.ep, NULL));
  OFICHKERR(fi_scalable_ep_bind(ofi.ep, &ofi.av->fid, 0));

  /* set up tx and rx contexts */
  cq_attr.format = FI_CQ_FORMAT_CONTEXT;
  cq_attr.size = 1024; /* ??? */
  cq_attr.wait_obj = FI_WAIT_UNSPEC;
  ofi.tx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_tx_ctx,
                                                   sizeof(ofi.tx_ep[0]),
                                                   CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                   0, 0);
  ofi.tx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_tx_ctx,
                                                   sizeof(ofi.tx_cq[0]),
                                                   CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                   0, 0);
  for (i = 0; i < ofi.num_tx_ctx; i++) {
    OFICHKERR(fi_tx_context(ofi.ep, i, NULL, &ofi.tx_ep[i], NULL));
    OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.tx_cq[i], NULL));
    OFICHKERR(fi_ep_bind(ofi.tx_ep[i], &ofi.tx_cq[i]->fid, FI_TRANSMIT));
    OFICHKERR(fi_enable(ofi.tx_ep[i]));
  }

  ofi.rx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_rx_ctx,
                                                   sizeof(ofi.rx_ep[0]),
                                                   CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                   0, 0);
  ofi.rx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_rx_ctx,
                                                    sizeof(ofi.rx_cq[0]),
                                                   CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                   0, 0);
  for (i = 0; i < ofi.num_rx_ctx; i++) {
    OFICHKERR(fi_rx_context(ofi.ep, i, NULL, &ofi.rx_ep[i], NULL));
    OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.rx_cq[i], NULL));
    OFICHKERR(fi_ep_bind(ofi.rx_ep[i], &ofi.rx_cq[i]->fid, FI_RECV));
    OFICHKERR(fi_enable(ofi.rx_ep[i]));
  }

  ofi.am_tx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_am_ctx,
                                                       sizeof(ofi.am_tx_ep[0]),
                                                       CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                       0, 0);
  ofi.am_tx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_am_ctx,
                                                      sizeof(ofi.am_tx_cq[0]),
                                                      CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                      0, 0);

  /* set up AM contexts */
  for (i = 0; i < ofi.num_am_ctx; i++) {
    OFICHKERR(fi_tx_context(ofi.ep, i+ofi.num_tx_ctx, NULL, &ofi.am_tx_ep[i], NULL));
    OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.am_tx_cq[i], NULL));
    OFICHKERR(fi_ep_bind(ofi.am_tx_ep[i], &ofi.am_tx_cq[i]->fid, FI_TRANSMIT));
    OFICHKERR(fi_enable(ofi.am_tx_ep[i]));
  }

  ofi.am_rx_ep = (struct fid_ep **) chpl_mem_allocMany(ofi.num_am_ctx,
                                                       sizeof(ofi.am_rx_ep[0]),
                                                       CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                       0, 0);
  ofi.am_rx_cq = (struct fid_cq **) chpl_mem_allocMany(ofi.num_am_ctx,
                                                      sizeof(ofi.am_rx_cq[0]),
                                                      CHPL_RT_MD_COMM_PER_LOC_INFO,
                                                      0, 0);
  for (i = 0; i < ofi.num_am_ctx; i++) {
    OFICHKERR(fi_rx_context(ofi.ep, i+ofi.num_rx_ctx, NULL, &ofi.am_rx_ep[i], NULL));
    OFICHKERR(fi_cq_open(ofi.domain, &cq_attr, &ofi.am_rx_cq[i], NULL));
    OFICHKERR(fi_ep_bind(ofi.am_rx_ep[i], &ofi.am_rx_cq[i]->fid, FI_RECV));
    OFICHKERR(fi_enable(ofi.am_rx_ep[i]));
  }

  OFICHKERR(fi_enable(ofi.ep));

  libfabric_init_addrvec(rx_ctx_cnt, rx_ctx_bits);

  OFICHKERR(fi_mr_reg(ofi.domain, 0, SIZE_MAX,
                      FI_READ | FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE |
                      FI_SEND | FI_RECV, 0,
                      (uint64_t) chpl_nodeID, 0, &ofi.mr, NULL));

  fi_freeinfo(info);  /* No error returned */
  fi_freeinfo(hints); /* No error returned */

  chpl_msg(2, "%d: completed libfabric initialization\n", chpl_nodeID);
}
Ejemplo n.º 15
0
void rdm_rma_setup(void)
{
	int ret = 0;
	struct fi_av_attr attr;
	size_t addrlen = 0;

	hints = fi_allocinfo();
	cr_assert(hints, "fi_allocinfo");

	hints->domain_attr->cq_data_size = 4;
	hints->mode = ~0;

	hints->fabric_attr->name = strdup("gni");

	ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	ret = fi_open_ops(&dom->fid, FI_GNI_DOMAIN_OPS_1,
			  0, (void **) &gni_domain_ops, NULL);

	attr.type = FI_AV_MAP;
	attr.count = 16;

	ret = fi_av_open(dom, &attr, &av, NULL);
	cr_assert(!ret, "fi_av_open");

	ret = fi_endpoint(dom, fi, &ep[0], NULL);
	cr_assert(!ret, "fi_endpoint");

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	ret = fi_cq_open(dom, &cq_attr, &send_cq, 0);
	cr_assert(!ret, "fi_cq_open");

	/*
	 * imitate shmem, etc. use FI_WRITE for bind
	 * flag
	 */
	ret = fi_ep_bind(ep[0], &send_cq->fid, FI_TRANSMIT);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
	cr_assert(addrlen > 0);

	ep_name[0] = malloc(addrlen);
	cr_assert(ep_name[0] != NULL);

	ep_name[1] = malloc(addrlen);
	cr_assert(ep_name[1] != NULL);

	ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_endpoint(dom, fi, &ep[1], NULL);
	cr_assert(!ret, "fi_endpoint");

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	ret = fi_cq_open(dom, &cq_attr, &recv_cq, 0);
	cr_assert(!ret, "fi_cq_open");

	ret = fi_ep_bind(ep[1], &recv_cq->fid, FI_RECV);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
				NULL);
	cr_assert(ret == 1);

	ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
				NULL);
	cr_assert(ret == 1);

	ret = fi_ep_bind(ep[0], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_ep_bind(ep[1], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_enable(ep[0]);
	cr_assert(!ret, "fi_ep_enable");

	ret = fi_enable(ep[1]);
	cr_assert(!ret, "fi_ep_enable");

	target = malloc(BUF_SZ);
	assert(target);

	source = malloc(BUF_SZ);
	assert(source);

	ret = fi_mr_reg(dom, target, BUF_SZ,
			FI_REMOTE_WRITE, 0, 0, 0, &rem_mr, &target);
	cr_assert_eq(ret, 0);

	ret = fi_mr_reg(dom, source, BUF_SZ,
			FI_REMOTE_WRITE, 0, 0, 0, &loc_mr, &source);
	cr_assert_eq(ret, 0);

	uc_source = malloc(BUF_SZ);
	assert(uc_source);

	mr_key = fi_mr_key(rem_mr);

	ret = fi_cntr_open(dom, &cntr_attr, &write_cntr, 0);
	cr_assert(!ret, "fi_cntr_open");

	ret = fi_ep_bind(ep[0], &write_cntr->fid, FI_WRITE);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_cntr_open(dom, &cntr_attr, &read_cntr, 0);
	cr_assert(!ret, "fi_cntr_open");

	ret = fi_ep_bind(ep[0], &read_cntr->fid, FI_READ);
	cr_assert(!ret, "fi_ep_bind");

	writes = reads = write_errs = read_errs = 0;
}
Ejemplo n.º 16
0
static int init_fabric(void)
{
	char *node, *service;
	uint64_t flags = 0;
	int ret;
	
	ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
	if (ret)
		return ret;
	
	/* Get fabric info */
	ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}
	
	/* Get remote address of the server */
	if (opts.dst_addr) {
		addrlen = fi->dest_addrlen;
		remote_addr = malloc(addrlen);
		memcpy(remote_addr, fi->dest_addr, addrlen);
	}

	/* Open fabric */
	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto err1;
	}

	/* Open domain */
	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		FT_PRINTERR("fi_domain", ret);
		goto err2;
	}
	
	ret = alloc_ep_res(fi);
	if (ret)
		goto err4;

	ret = bind_ep_res();
	if (ret)
		goto err5;
	
	if (opts.dst_addr) {
		/* Insert address to the AV and get the fabric address back */ 	
		ret = fi_av_insert(av, remote_addr, 1, &remote_fi_addr, 0, 
				&fi_ctx_av);
		if (ret != 1) {
			FT_PRINTERR("fi_av_insert", ret);
			return ret;
		}
	}

	return 0;

err5:
	free_ep_res();
err4:
	fi_close(&dom->fid);
err2:
	fi_close(&fab->fid);
err1:
	return ret;
}
Ejemplo n.º 17
0
void rdm_sr_setup_common_eps(void)
{
	int ret = 0, i = 0, j = 0;
	struct fi_av_attr attr;
	size_t addrlen = 0;

	memset(&attr, 0, sizeof(attr));
	attr.type = FI_AV_MAP;
	attr.count = NUMEPS;

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	target = malloc(BUF_SZ * 3); /* 3x BUF_SZ for multi recv testing */
	assert(target);

	dest_iov = malloc(sizeof(struct iovec) * IOV_CNT);
	assert(dest_iov);
	d_iov = malloc(sizeof(struct iovec) * IOV_CNT);
	assert(d_iov);

	source = malloc(BUF_SZ);
	assert(source);

	src_iov = malloc(sizeof(struct iovec) * IOV_CNT);
	assert(src_iov);
	s_iov = malloc(sizeof(struct iovec) * IOV_CNT);
	assert(s_iov);

	for (i = 0; i < IOV_CNT; i++) {
		src_iov[i].iov_base = malloc(BUF_SZ);
		assert(src_iov[i].iov_base != NULL);

		dest_iov[i].iov_base = malloc(BUF_SZ * 3);
		assert(dest_iov[i].iov_base != NULL);
	}

	iov_src_buf = malloc(BUF_SZ * IOV_CNT);
	assert(iov_src_buf);

	iov_dest_buf = malloc(BUF_SZ * IOV_CNT);
	assert(iov_dest_buf);

	uc_target = malloc(BUF_SZ);
	assert(uc_target);

	uc_source = malloc(BUF_SZ);
	assert(uc_source);

	ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	for (i = 0; i < NUMEPS; i++) {
		ret = fi_domain(fab, fi[i], dom + i, NULL);
		cr_assert(!ret, "fi_domain");

		ret = fi_open_ops(&dom[i]->fid, FI_GNI_DOMAIN_OPS_1,
				  0, (void **) (gni_domain_ops + i), NULL);

		ret = fi_av_open(dom[i], &attr, av + i, NULL);
		cr_assert(!ret, "fi_av_open");

		ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
		cr_assert(!ret, "fi_endpoint");

		ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
		cr_assert(!ret, "fi_cq_open");

		ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
		cr_assert(addrlen > 0);

		ep_name[i] = malloc(addrlen);
		cr_assert(ep_name[i] != NULL);

		ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
		cr_assert(ret == FI_SUCCESS);
	}

	for (i = 0; i < NUMEPS; i++) {
		/* Insert all gni addresses into each av */
		for (j = 0; j < NUMEPS; j++) {
			ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
					   0, NULL);
			cr_assert(ret == 1);
		}

		ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_enable(ep[i]);
		cr_assert(!ret, "fi_ep_enable");

		ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_ep_bind(ep[i], &send_cntr[i]->fid, FI_SEND);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open");

		ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid, FI_RECV);
		cr_assert(!ret, "fi_ep_bind");
	}
}
Ejemplo n.º 18
0
static int client_connect(void)
{
	struct fi_eq_cm_entry entry;
	uint32_t event;
	struct fi_info *fi;
	ssize_t rd;
	int ret;

	if (src_addr) {
		ret = getaddr(src_addr, NULL, (struct sockaddr **) &hints.src_addr,
			      (socklen_t *) &hints.src_addrlen);
		if (ret)
			printf("source address error %s\n", gai_strerror(ret));
	}

	ret = fi_getinfo(FI_VERSION(1, 0), dst_addr, port, 0, &hints, &fi);
	if (ret) {
		printf("fi_getinfo %s\n", strerror(-ret));
		goto err0;
	}

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		printf("fi_fabric %s\n", fi_strerror(-ret));
		goto err1;
	}

	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		printf("fi_domain %s %s\n", fi_strerror(-ret),
			fi->domain_attr->name);
		goto err2;
	}

	ret = fi_endpoint(dom, fi, &ep, NULL);
	if (ret) {
		printf("fi_endpoint %s\n", fi_strerror(-ret));
		goto err3;
	}

	ret = alloc_ep_res(fi);
	if (ret)
		goto err4;

	ret = bind_ep_res();
	if (ret)
		goto err5;

	ret = fi_connect(ep, fi->dest_addr, NULL, 0);
	if (ret) {
		printf("fi_connect %s\n", fi_strerror(-ret));
		goto err5;
	}

	rd = fi_eq_sread(cmeq, &event, &entry, sizeof entry, -1, 0);
	if (rd != sizeof entry) {
		printf("fi_eq_sread %zd %s\n", rd, fi_strerror((int) -rd));
		return (int) rd;
	}

	if (event != FI_COMPLETE || entry.fid != &ep->fid) {
		printf("Unexpected CM event %d fid %p (ep %p)\n",
			event, entry.fid, ep);
		ret = -FI_EOTHER;
		goto err1;
	}

	if (hints.src_addr)
		free(hints.src_addr);
	fi_freeinfo(fi);
	return 0;

err5:
	free_ep_res();
err4:
	fi_close(&ep->fid);
err3:
	fi_close(&dom->fid);
err2:
	fi_close(&fab->fid);
err1:
	fi_freeinfo(fi);
err0:
	if (hints.src_addr)
		free(hints.src_addr);
	return ret;
}
Ejemplo n.º 19
0
int main(int argc, char **argv)
{
	int op, ret, ndoms = 3, neps = 3, i, j;

	hints = fi_allocinfo();
	if (!hints) {
		exit(1);
	}

	while ((op = getopt(argc, argv, "f:p:d:D:n:d:e:h")) != -1) {
		switch (op) {
		case 'f':
			hints->fabric_attr->name = strdup(optarg);
			break;
		case 'p':
			hints->fabric_attr->prov_name = strdup(optarg);
			break;
		case 'd':
			ndoms = atoi(optarg);
			break;
		case 'e':
			neps = atoi(optarg);
			break;
		case 'h':
		default:
			printf("usage: %s\n", argv[0]);
			printf("\t[-f fabric_name]\n");
			printf("\t[-p provider_name]\n");
			printf("\t[-d ndomains]\n");
			printf("\t[-e neps]\n");
			exit(1);
		}
	}

	hints->mode = ~0;

	ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
	if (ret != 0) {
		printf("fi_getinfo %s\n", fi_strerror(-ret));
		exit(1);
	}

	ret = fi_fabric(fi->fabric_attr, &fabric, NULL);
	if (ret != 0) {
		printf("fi_fabric %s\n", fi_strerror(-ret));
		exit(1);
	}

	domains = (struct fid_domain **)malloc(ndoms * sizeof(struct fid_domain *));
	assert(domains);
	eps = (struct fid_ep **)malloc(neps * ndoms * sizeof(struct fid_ep *));
	assert(eps);

	for (i = 0; i < ndoms; i++) {
		ret = fi_domain(fabric, fi, &domains[i], NULL);
		if (ret != 0) {
			printf("fi_domain %s\n", fi_strerror(-ret));
			exit(1);
		}

		for (j = 0; j < neps; j++) {
			int idx = (i * neps) + j;
			ret = fi_endpoint(domains[i], fi, &eps[idx], NULL);
			if (ret != 0) {
				printf("[%d:%d] ]fi_endpoint %s\n", i, j, fi_strerror(-ret));
				exit(1);
			}
		}
	}

	for (i = 0; i < ndoms; i++) {
		for (j = 0; j < neps; j++) {
			int idx = (i * neps) + j;
			ret = fi_close(&eps[idx]->fid);
			if (ret != 0) {
				printf("Error %d closing ep: %s\n", ret, fi_strerror(-ret));
				exit(1);
			}
		}

		ret = fi_close(&domains[i]->fid);
		if (ret != 0) {
			printf("Error %d closing domain: %s\n", ret, fi_strerror(-ret));
			exit(1);
		}
	}

	free(eps);
	free(domains);

	ret = fi_close(&fabric->fid);
	if (ret != 0) {
		printf("Error %d closing fabric: %s\n", ret, fi_strerror(-ret));
		exit(1);
	}

	fi_freeinfo(fi);
	fi_freeinfo(hints);

	return ret;
}
Ejemplo n.º 20
0
static int setup_handle(void)
{
	static char buf[BUFSIZ];
	struct addrinfo *ai, aihints;
	const char *bound_addr_str;
	int ret;

	memset(&aihints, 0, sizeof aihints);
	aihints.ai_flags = AI_PASSIVE;
	ret = getaddrinfo(opts.src_addr, opts.src_port, &aihints, &ai);
	if (ret == EAI_SYSTEM) {
		FT_ERR("getaddrinfo for %s:%s: %s",
			opts.src_addr, opts.src_port, strerror(errno));
		return -ret;
	} else if (ret) {
		FT_ERR("getaddrinfo: %s", gai_strerror(ret));
		return -FI_ENODATA;
	}

	switch (ai->ai_family) {
	case AF_INET:
		hints->addr_format = FI_SOCKADDR_IN;
		break;
	case AF_INET6:
		hints->addr_format = FI_SOCKADDR_IN6;
		break;
	}

	/* Get fabric info */
	ret = fi_getinfo(FT_FIVERSION, opts.src_addr, NULL, FI_SOURCE, hints, &fi_pep);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		goto out;
	}
	free(fi_pep->src_addr);
	fi_pep->src_addr = NULL;
	fi_pep->src_addrlen = 0;

	ret = fi_fabric(fi_pep->fabric_attr, &fabric, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto out;
	}

	ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
	if (ret) {
		FT_PRINTERR("fi_eq_open", ret);
		goto out;
	}

	/* Open a passive endpoint */
	ret = fi_passive_ep(fabric, fi_pep, &pep, NULL);
	if (ret) {
		FT_PRINTERR("fi_passive_ep", ret);
		goto out;
	}

	ret = fi_setname(&pep->fid, ai->ai_addr, ai->ai_addrlen);
	if (ret) {
		FT_PRINTERR("fi_setname", ret);
		goto out;
	}

	ret = fi_getname(&pep->fid, &bound_addr, &bound_addr_len);
	if (ret) {
		FT_PRINTERR("fi_getname", ret);
		goto out;
	}

	/* Verify port number */
	switch (ai->ai_family) {
	case AF_INET:
		if (bound_addr.sin.sin_port == 0) {
			FT_ERR("port number is 0 after fi_setname()");
			ret = -FI_EINVAL;
			goto out;
		}
		break;
	case AF_INET6:
		if (bound_addr.sin6.sin6_port == 0) {
			FT_ERR("port number is 0 after fi_setname()");
			ret = -FI_EINVAL;
			goto out;
		}
		break;
	}

	bound_addr_str = sockaddrstr(&bound_addr, bound_addr_len, buf, BUFSIZ);
	if (!bound_addr_str) {
		FT_ERR("Unable to get bound_addr as string!");
		ret = -FI_EINVAL;
		goto out;
	}
	printf("bound_addr: \"%s\"\n", bound_addr_str);

	hints->handle = &pep->fid;
out:
	freeaddrinfo(ai);
	return ret;
}
Ejemplo n.º 21
0
static int init_fabric(void)
{
	uint64_t flags = 0;
	char *node, *service;
	int ret;

	ret = ft_read_addr_opts(&node, &service, hints, &flags, &opts);
	if (ret)
		return ret;

	ret = fi_getinfo(FT_FIVERSION, node, service, flags, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	/* Check the optimal number of TX and RX contexts supported by the provider */
	ctx_cnt = MIN(ctx_cnt, fi->domain_attr->tx_ctx_cnt);
	ctx_cnt = MIN(ctx_cnt, fi->domain_attr->rx_ctx_cnt);
	if (!ctx_cnt) {
		fprintf(stderr, "Provider doesn't support contexts\n");
		return 1;
	}

	/* We use provider MR attributes and direct address (no offsets) for RMA calls */
	if (!(fi->mode & FI_PROV_MR_ATTR))
		fi->mode |= FI_PROV_MR_ATTR;

	/* Get remote address */
	if (opts.dst_addr) {
		addrlen = fi->dest_addrlen;
		remote_addr = malloc(addrlen);
		memcpy(remote_addr, fi->dest_addr, addrlen);
	}

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto err0;
	}

	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		FT_PRINTERR("fi_domain", ret);
		goto err1;
	}

	/* Set the required number of TX and RX context counts */
	fi->ep_attr->tx_ctx_cnt = ctx_cnt;
	fi->ep_attr->rx_ctx_cnt = ctx_cnt;

	ret = fi_scalable_ep(dom, fi, &sep, NULL);
	if (ret) {
		FT_PRINTERR("fi_scalable_ep", ret);
		goto err2;
	}

	ret = alloc_ep_res(sep);
	if (ret)
		goto err3;

	ret = bind_ep_res();
	if (ret)
		goto err4;

	return 0;

err4:
	free_ep_res();
err3:
	fi_close(&sep->fid);
err2:
	fi_close(&dom->fid);
err1:
	fi_close(&fab->fid);
err0:
	return ret;
}