Exemple #1
0
static int alloc_ep(void)
{
	int i, ret;

	ep_array = calloc(ep_cnt, sizeof(*ep_array));
	if (!ep_array)
		return -FI_ENOMEM;

	ret = fi_endpoint(domain, fi, &ep_array[0], NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		return ret;
	}

	for (i = 1; i < ep_cnt; i++) {
		if (hints->ep_attr->type == FI_EP_MSG)
			ret = fi_endpoint(domain, fi, &ep_array[i], NULL);
		else
			ret = fi_endpoint(domain, fi_dup, &ep_array[i], NULL);
		if (ret) {
			FT_PRINTERR("fi_endpoint", ret);
			return ret;
		}
	}

	return 0;
}
Exemple #2
0
void dg_setup_prog_manual(void)
{
	int ret = 0;

	hints = fi_allocinfo();
	cr_assert(hints, "fi_allocinfo");

	hints->domain_attr->cq_data_size = 4;
	hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
	hints->mode = ~0;

	hints->fabric_attr->name = strdup("gni");

	ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	ret = fi_endpoint(dom, fi, &ep, NULL);
	cr_assert(!ret, "fi_endpoint");
}
Exemple #3
0
static int setup_av_ep(struct fid_ep **ep, fi_addr_t *remote_addr)
{
	int ret;
	hints->src_addr = NULL;

	fi_freeinfo(fi);

	ret = fi_getinfo(FT_FIVERSION, NULL, NULL, 0, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	ret = fi_endpoint(domain, fi, ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		return ret;
	}

	ret = ft_setup_ep(*ep, eq, av, txcq, rxcq, txcntr, rxcntr, false);
	if (ret)
		return ret;

	ret = ft_init_av_addr(av, *ep, remote_addr);
	if (ret)
		return ret;

	return 0;
}
Exemple #4
0
void dg_setup(void)
{
	int ret = 0;
	char my_hostname[HOST_NAME_MAX];

	hints = fi_allocinfo();
	cr_assert(hints, "fi_allocinfo");

	hints->domain_attr->cq_data_size = 4;
	hints->mode = ~0;

	hints->fabric_attr->name = strdup("gni");

	ret = gethostname(my_hostname, sizeof(my_hostname));
	cr_assert(!ret, "gethostname");

	ret = fi_getinfo(FI_VERSION(1, 0), my_hostname, my_cdm_id, FI_SOURCE,
			 hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	ret = fi_endpoint(dom, fi, &ep, NULL);
	cr_assert(!ret, "fi_endpoint");
}
Exemple #5
0
Test(endpoint, open_close)
{
	int i, ret;
	const int num_eps = 61;
	struct fid_ep *eps[num_eps];

	memset(eps, 0, num_eps*sizeof(struct fid_ep *));

	for (i = 0; i < num_eps; i++) {
		ret = fi_endpoint(dom, fi, &eps[i], NULL);
		cr_assert(!ret, "fi_endpoint");
		struct gnix_fid_ep *ep = container_of(eps[i],
						      struct gnix_fid_ep,
						      ep_fid);
		cr_assert(ep, "endpoint not allcoated");

		/* Check fields (fill in as implemented) */
		cr_assert(ep->nic, "NIC not allocated");
		cr_assert(!_gnix_fl_empty(&ep->fr_freelist),
			  "gnix_fab_req freelist empty");
	}

	for (i = num_eps-1; i >= 0; i--) {
		ret = fi_close(&eps[i]->fid);
		cr_assert(!ret, "fi_close endpoint");
	}

}
Exemple #6
0
static int setup_server_ep(struct fid_ep **ep)
{
	int ret;

	ret = ft_retrieve_conn_req(eq, &fi);
	if (ret)
		goto failed_accept;

	ret = fi_endpoint(domain, fi, ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		goto failed_accept;
	}

	ret = ft_setup_ep(*ep, eq, av, txcq, rxcq, txcntr, rxcntr, false);
	if (ret)
		goto failed_accept;

	ret = ft_accept_connection(*ep, eq);
	if (ret)
		goto failed_accept;

	return 0;

failed_accept:
	fi_reject(pep, fi->handle, NULL, 0);
	return ret;
}
Exemple #7
0
Test(endpoint, sizeleft)
{
	int ret;
	size_t sz;
	struct fid_ep *ep = NULL;

	ret = fi_endpoint(dom, fi, &ep, NULL);
	cr_assert(ret == FI_SUCCESS, "fi_endpoint");

	/* Test in disabled state. */
	sz = fi_rx_size_left(ep);
	cr_assert(sz == -FI_EOPBADSTATE, "fi_rx_size_left");

	sz = fi_tx_size_left(ep);
	cr_assert(sz == -FI_EOPBADSTATE, "fi_tx_size_left");

	ret = fi_enable(ep);
	cr_assert(ret == FI_SUCCESS, "fi_enable");

	/* Test default values. */
	sz = fi_rx_size_left(ep);
	cr_assert(sz == GNIX_RX_SIZE_DEFAULT, "fi_rx_size_left");

	sz = fi_tx_size_left(ep);
	cr_assert(sz == GNIX_TX_SIZE_DEFAULT, "fi_tx_size_left");

	ret = fi_close(&ep->fid);
	cr_assert(!ret, "fi_close endpoint");
}
Exemple #8
0
static inline void cntr_setup_eps(void)
{
	int i, ret;
	struct fi_av_attr attr;

	hints = fi_allocinfo();
	cr_assert(hints, "fi_allocinfo");

	hints->domain_attr->cq_data_size = 4;
	hints->mode = ~0;

	hints->fabric_attr->name = strdup("gni");

	ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	attr.type = FI_AV_MAP;
	attr.count = 16;

	ret = fi_av_open(dom, &attr, &av, NULL);
	cr_assert(!ret, "fi_av_open");

	for (i = 0; i < NUM_EPS; i++) {
		ret = fi_endpoint(dom, fi, &ep[i], NULL);
		cr_assert(!ret, "fi_endpoint");
	}
}
Exemple #9
0
int cm_client_start_connect(void)
{
	int ret;
	struct sockaddr_in loc_sa;

	cm_local_ip(&loc_sa);

	cli_hints = fi_allocinfo();
	cli_hints->fabric_attr->name = strdup("gni");
	cli_hints->caps = GNIX_EP_PRIMARY_CAPS;
	cli_hints->ep_attr->type = FI_EP_MSG;
	cli_hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;

	ret = fi_getinfo(fi_version(), inet_ntoa(loc_sa.sin_addr),
			 DEF_PORT, 0, cli_hints, &cli_fi);
	cr_assert(!ret);

	ret = fi_fabric(cli_fi->fabric_attr, &cli_fab, NULL);
	cr_assert(!ret);

	ret = fi_eq_open(cli_fab, &eq_attr, &cli_eq, NULL);
	cr_assert(!ret);

	ret = fi_domain(cli_fab, cli_fi, &cli_dom, NULL);
	cr_assert(!ret);

	ret = fi_endpoint(cli_dom, cli_fi, &cli_ep, NULL);
	cr_assert(!ret, "fi_endpoint");

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	ret = fi_cq_open(cli_dom, &cq_attr, &cli_cq, &cli_cq);
	cr_assert(!ret);

	ret = fi_ep_bind(cli_ep, &cli_eq->fid, 0);
	cr_assert(!ret);

	ret = fi_ep_bind(cli_ep, &cli_cq->fid, FI_SEND | FI_RECV);
	cr_assert(!ret);

	ret = fi_enable(cli_ep);
	cr_assert(!ret);

	ret = fi_connect(cli_ep, cli_fi->dest_addr, cli_cm_in_data,
			 GNIX_CM_DATA_MAX_SIZE+1);
	cr_assert(ret == -FI_EINVAL);

	ret = fi_connect(cli_ep, cli_fi->dest_addr, cli_cm_in_data,
			 strlen(cli_cm_in_data));
	cr_assert(!ret);

	dbg_printf("Client connect complete.\n");

	return 0;
}
Exemple #10
0
static int init_fabric(void)
{
	int ret;
	uint64_t flags = 0;

	/* Get fabric info */
	ret = fi_getinfo(CT_FIVERSION, NULL, NULL, flags, hints, &fi);
	if (ret) {
		ct_print_fi_error("fi_getinfo", ret);
		return ret;
	}

	/* Open fabric */
	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		ct_print_fi_error("fi_fabric", ret);
		goto err1;
	}

	/* Open domain */
	ret = fi_domain(fab, fi, &dom, NULL);
	if (ret) {
		ct_print_fi_error("fi_domain", ret);
		goto err2;
	}

	/* Open endpoint */
	ret = fi_endpoint(dom, fi, &ep, NULL);
	if (ret) {
		ct_print_fi_error("fi_endpoint", ret);
		goto err3;
	}

	/* Allocate endpoint resources */
	ret = alloc_ep_res();
	if (ret)
		goto err4;

	/* Bind EQs and AVs with endpoint */
	ret = bind_ep_res();
	if (ret)
		goto err5;

	return 0;

err5:
	free_ep_res();
err4:
	fi_close(&ep->fid);
err3:
	fi_close(&dom->fid);
err2:
	fi_close(&fab->fid);
err1:
	return ret;
}
static int alloc_ep_res(struct fi_info *fi)
{
	struct fi_rx_attr rx_attr;
	struct fi_tx_attr tx_attr;
	int i, ret = 0;

	ret = ft_alloc_bufs();
	if (ret)
		return ret;

	remote_fi_addr = malloc(sizeof(*remote_fi_addr) * ep_cnt);
	if (!remote_fi_addr) {
		perror("malloc");
		return -FI_ENOMEM;
	}

	av_attr.count = ep_cnt;

	ret = ft_alloc_active_res(fi);
	if (ret)
		return ret;

	/* TODO: avoid allocating EP when EP array is used. */
	FT_CLOSE_FID(ep);

	memset(&tx_attr, 0, sizeof tx_attr);
	memset(&rx_attr, 0, sizeof rx_attr);

	ret = fi_stx_context(domain, &tx_attr, &stx_ctx, NULL);
	if (ret) {
		FT_PRINTERR("fi_stx_context", ret);
		return ret;
	}

	ret = fi_srx_context(domain, &rx_attr, &srx_ctx, NULL);
	if (ret) {
		FT_PRINTERR("fi_srx_context", ret);
		return ret;
	}

	ep_array = calloc(ep_cnt, sizeof(*ep_array));
	if (!ep_array) {
		perror("malloc");
		return ret;
	}
	for (i = 0; i < ep_cnt; i++) {
		ret = fi_endpoint(domain, fi, &ep_array[i], NULL);
		if (ret) {
			FT_PRINTERR("fi_endpoint", ret);
			return ret;
		}
	}

	return 0;
}
Exemple #12
0
static inline int allocate_endpoints(struct fabric_info *info)
{

  int ret = 0;

    /* ------------------------------------*/
    /* 		Allocate Endpoints	   */
    /* ------------------------------------*/

    /* this endpoint is used to get completion events and
     * used to expose memory to incoming reads/writes */
    info->p_info->ep_attr->tx_ctx_cnt = FI_SHARED_CONTEXT;
    info->p_info->tx_attr->op_flags = FI_DELIVERY_COMPLETE;
    ret = fi_endpoint(shmem_transport_ofi_domainfd,
                      info->p_info, &shmem_transport_ofi_epfd, NULL);
    if(ret!=0){
	OFI_ERRMSG("epfd creation failed\n");
	return ret;
    }

    /* In hard polling builds, ask for remote read/write counter updates to
     * support shmem_wait that blocks on an update arriving from the network */
#ifndef ENABLE_HARD_POLLING
    info->p_info->caps |= FI_REMOTE_WRITE | FI_REMOTE_READ;
#endif
    /* Remove FI_CONTEXT flag from this endpoint */
    info->p_info->mode = 0;
    info->p_info->tx_attr->mode = 0;
    info->p_info->rx_attr->mode = 0;
    ret = fi_endpoint(shmem_transport_ofi_domainfd,
                      info->p_info, &shmem_transport_ofi_cntr_epfd, NULL);
    if(ret!=0){
	OFI_ERRMSG("cntr_epfd creation failed\n");
	return ret;
    }

    return ret;

}
Exemple #13
0
int cm_server_accept(void)
{
	uint32_t event;
	ssize_t rd;
	int ret;
	struct fi_eq_cm_entry *entry;
	void *eqe_buf[EQE_SIZE] = {0};

	rd = fi_eq_sread(srv_eq, &event, &eqe_buf, EQE_SIZE, -1, 0);
	cr_assert(rd == (sizeof(*entry) + strlen(cli_cm_in_data)));
	cr_assert(event == FI_CONNREQ);

	entry = (struct fi_eq_cm_entry *)eqe_buf;
	cr_assert(!memcmp(cli_cm_in_data, entry->data,
			  strlen(cli_cm_in_data)));

	ret = fi_domain(srv_fab, entry->info, &srv_dom, NULL);
	cr_assert(!ret);

	ret = fi_endpoint(srv_dom, entry->info, &srv_ep, NULL);
	cr_assert(!ret, "fi_endpoint");

	fi_freeinfo(entry->info);

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	ret = fi_cq_open(srv_dom, &cq_attr, &srv_cq, &srv_cq);
	cr_assert(!ret);

	ret = fi_ep_bind(srv_ep, &srv_eq->fid, 0);
	cr_assert(!ret);

	ret = fi_ep_bind(srv_ep, &srv_cq->fid, FI_SEND | FI_RECV);
	cr_assert(!ret);

	ret = fi_enable(srv_ep);
	cr_assert(!ret);

	ret = fi_accept(srv_ep, srv_cm_in_data, GNIX_CM_DATA_MAX_SIZE+1);
	cr_assert(ret == -FI_EINVAL);

	ret = fi_accept(srv_ep, srv_cm_in_data, strlen(srv_cm_in_data));
	cr_assert(!ret);

	dbg_printf("Server accept complete.\n");

	return 0;
}
Exemple #14
0
int ft_alloc_active_res(struct fi_info *fi)
{
	int ret;

	ret = ft_alloc_ep_res(fi);
	if (ret)
		return ret;

	ret = fi_endpoint(domain, fi, &ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		return ret;
	}

	return 0;
}
Exemple #15
0
/*
 * rpmemd_fip_init_ep -- initialize active endpoint
 */
static int
rpmemd_fip_init_ep(struct rpmemd_fip *fip, struct fi_info *info)
{
	int ret;

	/* create an endpoint from fabric interface info */
	ret = fi_endpoint(fip->domain, info, &fip->ep, NULL);
	if (ret) {
		RPMEMD_FI_ERR(ret, "allocating endpoint");
		goto err_endpoint;
	}

	/* bind event queue to the endpoint */
	ret = fi_ep_bind(fip->ep, &fip->eq->fid, 0);
	if (ret) {
		RPMEMD_FI_ERR(ret, "binding event queue to endpoint");
		goto err_bind_eq;
	}

	/*
	 * Bind completion queue to the endpoint.
	 * Use a single completion queue for outbound and inbound work
	 * requests. Use selective completion implies adding FI_COMPLETE
	 * flag to each WR which needs a completion.
	 */
	ret = fi_ep_bind(fip->ep, &fip->cq->fid,
			FI_RECV | FI_TRANSMIT | FI_SELECTIVE_COMPLETION);
	if (ret) {
		RPMEMD_FI_ERR(ret, "binding completion queue to endpoint");
		goto err_bind_cq;
	}

	/* enable the endpoint */
	ret = fi_enable(fip->ep);
	if (ret) {
		RPMEMD_FI_ERR(ret, "enabling endpoint");
		goto err_enable;
	}

	return 0;
err_enable:
err_bind_cq:
err_bind_eq:
	RPMEMD_FI_CLOSE(fip->ep, "closing endpoint");
err_endpoint:
	return -1;
}
Exemple #16
0
Test(endpoint_1_0, no_auth_key_support)
{
	int ret;
	struct fid_ep *ep;
	void *old_auth_key = fi->ep_attr->auth_key;
	int old_auth_key_size = fi->ep_attr->auth_key_size;

	fi->ep_attr->auth_key = (void *) 0xdeadbeef;
	fi->ep_attr->auth_key_size = 47;

	ret = fi_endpoint(dom, fi, &ep, NULL);
	cr_assert(ret == -FI_EINVAL, "fi_endpoint, ret=%d expected=%d",
		ret, -FI_EINVAL);

	fi->ep_attr->auth_key = old_auth_key;
	fi->ep_attr->auth_key_size = old_auth_key_size;
}
Exemple #17
0
void vc_lookup_setup(int av_type, int av_size)
{
	int ret = 0;
	struct fi_av_attr attr;

	hints = fi_allocinfo();

	hints->mode = mode_bits;
	hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
	hints->fabric_attr->prov_name = strdup("gni");

	/* Create endpoint */
	ret = fi_getinfo(fi_version(), NULL, 0, 0, hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	attr.type = av_type;
	attr.count = av_size;

	ret = fi_av_open(dom, &attr, &av, NULL);
	cr_assert(!ret, "fi_av_open");

	ret = fi_endpoint(dom, fi, &ep, NULL);
	cr_assert(!ret, "fi_endpoint");

	gnix_ep = container_of(ep, struct gnix_fid_ep, ep_fid);

	ret = fi_getname(&ep->fid, NULL, &ep_name_len);

	ret = fi_getname(&ep->fid, &ep_name, &ep_name_len);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_ep_bind(ep, &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_enable(ep);
	cr_assert(!ret, "fi_ep_enable");

	fi_freeinfo(hints);
}
Exemple #18
0
void Connection::on_connect_request(struct fi_eq_cm_entry* event,
                                    struct fid_domain* pd,
                                    struct fid_cq* cq) {
  int err = fi_endpoint(pd, event->info, &ep_, this);
  if (err) {
    L_(fatal) << "fi_endpoint failed: " << err << "=" << fi_strerror(-err);
    throw LibfabricException("fi_endpoint failed");
  }

#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
  err = fi_ep_bind(ep_, (::fid_t)eq_, 0);
  if (err) {
    L_(fatal) << "fi_ep_bind failed to eq: " << err << "=" << fi_strerror(-err);
    throw LibfabricException("fi_ep_bind failed to eq");
  }
  err = fi_ep_bind(ep_, (fid_t)cq, FI_SEND | FI_RECV | FI_SELECTIVE_COMPLETION);
  if (err) {
    L_(fatal) << "fi_ep_bind failed to cq: " << err << "=" << fi_strerror(-err);
    throw LibfabricException("fi_ep_bind failed to cq");
  }
#pragma GCC diagnostic pop

  // setup(pd);
  setup_mr(pd);

  auto private_data = get_private_data();
  assert(private_data->size() <= 255);

  err = fi_enable(ep_);
  if (err) {
    L_(fatal) << "fi_enable failed: " << err << "=" << fi_strerror(-err);
    throw LibfabricException("fi_enable failed");
  }
  // accept_connect_request();
  err = fi_accept(ep_, private_data->data(), private_data->size());
  if (err) {
    L_(fatal) << "fi_accept failed: " << err << "=" << fi_strerror(-err);
    throw LibfabricException("fi_accept failed");
  }

  // setup(pd);
  setup();
}
Exemple #19
0
static int
rx_size_left_err(void)
{
	int ret;
	int testret;
	struct fid_ep *ep;
	struct fi_info *myfi;

	testret = FAIL;
	ep = NULL;

	myfi = fi_dupinfo(fi);

	/* datapath operation, not expected to be caught by libfabric */
#if 0
	ret = fi_rx_size_left(NULL);
	if (ret != -FI_EINVAL) {
		goto fail;
	}
#endif

	ret = fi_endpoint(domain, myfi, &ep, NULL);
	if (ret != 0) {
		printf("fi_endpoint %s\n", fi_strerror(-ret));
		goto fail;
	}

	/* ep starts in a non-enabled state, may fail, should not SEGV */
	fi_rx_size_left(ep);

	testret = PASS;
fail:
	if (ep != NULL) {
		ret = fi_close(&ep->fid);
		if (ret != 0)
			printf("fi_close %s\n", fi_strerror(-ret));
		ep = NULL;
	}
	if (myfi != NULL) {
		fi_freeinfo(myfi);
	}
	return testret;
}
Exemple #20
0
static int setup_client_ep(struct fid_ep **ep)
{
	int ret;

	ret = fi_endpoint(domain, fi, ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		return ret;
	}

	ret = ft_setup_ep(*ep, eq, av, txcq, rxcq, txcntr, rxcntr, false);
	if (ret)
		return ret;

	ret = ft_connect_ep(*ep, eq, fi->dest_addr);
	if (ret)
		return ret;

	return 0;
}
Exemple #21
0
Test(endpoint, getsetopt_gni_ep)
{
	int ret;
	int val;
	struct fid_ep *ep = NULL;
	struct fi_gni_ops_ep *ep_ops;
	struct gnix_fid_ep *ep_priv = NULL;

	ret = fi_endpoint(dom, fi, &ep, NULL);
	cr_assert(!ret, "fi_endpoint");

	ep_priv = (struct gnix_fid_ep *) ep;

	ret = fi_open_ops(&ep->fid, "ep ops 1", 0, (void **) &ep_ops, NULL);
	cr_assert(!ret, "fi_open_ops endpoint");

	ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
	cr_assert(!ret, "ep_ops get_val");
	cr_assert_eq(val, 0);
	cr_assert_eq(ep_priv->use_tag_hlist, 0);

	val = 1; // set the hash implementation
	ret = ep_ops->set_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
	cr_assert(!ret, "ep_ops set_val");
	cr_assert_eq(ep_priv->use_tag_hlist, 1);
	cr_assert_eq(ep_priv->unexp_recv_queue.attr.type, GNIX_TAG_HLIST);
	cr_assert_eq(ep_priv->posted_recv_queue.attr.type, GNIX_TAG_HLIST);
	cr_assert_eq(ep_priv->tagged_unexp_recv_queue.attr.type, GNIX_TAG_HLIST);
	cr_assert_eq(ep_priv->tagged_posted_recv_queue.attr.type, GNIX_TAG_HLIST);


	val = 0; // reset the value
	ret = ep_ops->get_val(&ep->fid, GNI_HASH_TAG_IMPL, &val);
	cr_assert(!ret, "ep_ops get_val");
	cr_assert_eq(val, 1);
	cr_assert_eq(ep_priv->use_tag_hlist, 1);

	ret = fi_close(&ep->fid);
	cr_assert(!ret, "fi_close endpoint");
}
Exemple #22
0
static int client_open_new_ep()
{
	size_t opt_size;
	int ret;

	FT_CLOSE_FID(ep);

	ret = fi_endpoint(domain, fi, &ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		return ret;
	}

	ret = ft_enable_ep_recv();
	if (ret)
		return ret;

	/* Get the maximum cm_size for this domain + endpoint combination */
	opt_size = sizeof(opt_size);
	return fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_CM_DATA_SIZE,
		&cm_data_size, &opt_size);
}
Exemple #23
0
static void fas_ep_setup(void)
{
	int ret, i, j;
	size_t addrlen = 0;

	fas_setup_common(fi_version());
	ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt);
	ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt);

	for (i = 0; i < NUMEPS; i++) {
		fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt;
		fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt;

		ret = fi_domain(fab, fi[i], dom + i, NULL);
		cr_assert(!ret, "fi_domain returned: %s", fi_strerror(-ret));

		ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));

		ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));

		switch (ep_type) {
		case EP:
			ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
			cr_assert(!ret, "fi_endpoint returned: %s",
				  fi_strerror(-ret));
			break;
		case SEP:
			ret = fi_scalable_ep(dom[i], fi[i], ep + i,
					     NULL);
			cr_assert(!ret, "fi_endpoint returned: %s",
				  fi_strerror(-ret));
			break;
		case PEP:
			ret = fi_passive_ep(fab, fi[i], pep + i,
					    NULL);
			cr_assert(!ret, "fi_endpoint returned: %s",
				  fi_strerror(-ret));
			ret = fi_getname(get_fid[ep_type](i), NULL,
					 &addrlen);
			if (use_str_fmt) {
				cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN,
					  "fi_getname returned: %s",
					  fi_strerror(-ret));
			} else {
				cr_assert(addrlen ==
					  sizeof(struct gnix_ep_name),
					  "fi_getname returned: %s",
					  fi_strerror(-ret));
			}
			ep_name_len[i] = addrlen;
			continue;
		default:
			cr_assert_fail("Unknown endpoint type.");
		}

		ret = fi_av_open(dom[i], &attr, av + i, NULL);
		cr_assert(!ret, "fi_av_open returned: %s", fi_strerror(-ret));

		switch (ep_type) {
		case EP:
		case PEP:
			ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i,
					 0);
			cr_assert(!ret, "fi_cq_open returned: %s",
				  fi_strerror(-ret));

			ret = fi_ep_bind(ep[i], &msg_cq[i]->fid,
					 FI_SEND | FI_RECV);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));
			break;
		case SEP:
			dbg_printf(BLUE
					   "ctx_cnt = %d\n"
					   COLOR_RESET,
				   ctx_cnt);

			for (j = 0; j < ctx_cnt; j++) {
				ret = fi_tx_context(ep[i], j, NULL,
						    &tx_ep[i][j], NULL);
				cr_assert(!ret,
					  "fi_tx_context  returned: %s",
					  fi_strerror(-ret));

				ret = fi_cq_open(dom[i], &cq_attr,
						 &tx_cq[i][j],
						 NULL);
				cr_assert(!ret,
					  "fi_cq_open  returned: %s",
					  fi_strerror(-ret));

				ret = fi_rx_context(ep[i], j, NULL,
						    &rx_ep[i][j], NULL);
				cr_assert(!ret,
					  "fi_rx_context  returned: %s",
					  fi_strerror(-ret));

				ret = fi_cq_open(dom[i], &cq_attr,
						 &rx_cq[i][j],
						 NULL);
				cr_assert(!ret,
					  "fi_cq_open  returned: %s",
					  fi_strerror(-ret));
			}
			break;
		default:
			cr_assert_fail("Unknown endpoint type.");
		}

		ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen);
		if (use_str_fmt) {
			cr_assert(addrlen > sizeof(struct gnix_ep_name),
				  "fi_getname returned: %s",
				  fi_strerror(-ret));
		} else {
			cr_assert(addrlen == sizeof(struct gnix_ep_name),
				  "fi_getname returned: %s",
				  fi_strerror(-ret));
		}

		ep_name[i] = malloc(addrlen);
		ep_name_len[i] = addrlen;

		dbg_printf(BLUE
				   "ep_name_len[%d] = %lu\n"
				   COLOR_RESET, i,
			   ep_name_len[i]);
		cr_assert(ep_name[i] != NULL, "malloc returned: %s",
			  strerror(errno));

		ret = fi_getname(get_fid[ep_type](i), ep_name[i], &addrlen);
		cr_assert(ret == FI_SUCCESS, "fi_getname returned: %s",
			  fi_strerror(-ret));
	}

	/* Just testing setname / getname for passive endpoints */
	if (ep_type == PEP)
		return;

	for (i = 0; i < NUMEPS; i++) {
		/*Insert all gni addresses into each av*/
		for (j = 0; j < NUMEPS; j++) {
			ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
					   0, NULL);
			cr_assert(ret == 1, "fi_av_insert returned: %s",
				  fi_strerror(-ret));
		}

		switch (ep_type) {
		case EP:
			ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));

			ret = fi_ep_bind(ep[i], &send_cntr[i]->fid,
					 FI_SEND);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));

			ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid,
					 FI_RECV);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));
			break;
		case SEP:
			ret = fi_scalable_ep_bind(ep[i], &av[i]->fid,
						  0);
			cr_assert(!ret,
				  "fi_scalable_ep_bind returned: %s",
				  fi_strerror(-ret));
			dbg_printf(BLUE
					   "ctx_cnt = %d\n"
					   COLOR_RESET,
				   ctx_cnt);
			for (j = 0; j < ctx_cnt; j++) {
				ret = fi_ep_bind(tx_ep[i][j],
						 &tx_cq[i][j]->fid,
						 FI_TRANSMIT);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_ep_bind(tx_ep[i][j],
						 &send_cntr[i]->fid,
						 FI_SEND);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_enable(tx_ep[i][j]);
				cr_assert(!ret,
					  "fi_enable  returned: %s",
					  fi_strerror(-ret));

				ret = fi_ep_bind(rx_ep[i][j],
						 &rx_cq[i][j]->fid,
						 FI_RECV);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_ep_bind(rx_ep[i][j],
						 &recv_cntr[i]->fid,
						 FI_RECV);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_enable(rx_ep[i][j]);
				cr_assert(!ret,
					  "fi_enable  returned: %s",
					  fi_strerror(-ret));

			}
			break;
		case PEP:
			break;
		default:
			cr_assert_fail("Unknown endpoint type.");
		}

		ret = fi_enable(ep[i]);
		cr_assert(!ret, "fi_ep_enable returned: %s", fi_strerror(-ret));

		if (ep_type != SEP) {
			ret = fi_enable(ep[i]);
			cr_assert_eq(ret, -FI_EOPBADSTATE,
				     "fi_enable returned: %s",
				     fi_strerror(-ret));
		}
	}
}
Exemple #24
0
void cancel_setup(void)
{
	int ret = 0;
	struct fi_av_attr attr;
	size_t addrlen = 0;
	int rem_requested_key, loc_requested_key;

	hints = fi_allocinfo();
	cr_assert(hints, "fi_allocinfo");

	hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
	hints->domain_attr->cq_data_size = 4;
	hints->mode = mode_bits;

	hints->fabric_attr->prov_name = strdup("gni");

	ret = fi_getinfo(fi_version(), NULL, 0, 0, hints, &fi);
	cr_assert(!ret, "fi_getinfo");

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	ret = fi_domain(fab, fi, &dom, NULL);
	cr_assert(!ret, "fi_domain");

	memset(&attr, 0, sizeof(attr));
	attr.type = FI_AV_MAP;
	attr.count = 16;

	ret = fi_av_open(dom, &attr, &av, NULL);
	cr_assert(!ret, "fi_av_open");

	ret = fi_endpoint(dom, fi, &ep[0], NULL);
	cr_assert(!ret, "fi_endpoint");

	cq_attr.format = FI_CQ_FORMAT_CONTEXT;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	ret = fi_cq_open(dom, &cq_attr, &msg_cq[0], 0);
	cr_assert(!ret, "fi_cq_open");

	ret = fi_cq_open(dom, &cq_attr, &msg_cq[1], 0);
	cr_assert(!ret, "fi_cq_open");

	ret = fi_ep_bind(ep[0], &msg_cq[0]->fid, FI_SEND | FI_RECV);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_getname(&ep[0]->fid, NULL, &addrlen);
	cr_assert(addrlen > 0);

	ep_name[0] = malloc(addrlen);
	cr_assert(ep_name[0] != NULL);

	ret = fi_getname(&ep[0]->fid, ep_name[0], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_endpoint(dom, fi, &ep[1], NULL);
	cr_assert(!ret, "fi_endpoint");

	ret = fi_ep_bind(ep[1], &msg_cq[1]->fid, FI_SEND | FI_RECV);
	cr_assert(!ret, "fi_ep_bind");

	ep_name[1] = malloc(addrlen);
	cr_assert(ep_name[1] != NULL);

	ret = fi_getname(&ep[1]->fid, ep_name[1], &addrlen);
	cr_assert(ret == FI_SUCCESS);

	ret = fi_av_insert(av, ep_name[0], 1, &gni_addr[0], 0,
				NULL);
	cr_assert(ret == 1);

	ret = fi_av_insert(av, ep_name[1], 1, &gni_addr[1], 0,
				NULL);
	cr_assert(ret == 1);

	ret = fi_ep_bind(ep[0], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_ep_bind(ep[1], &av->fid, 0);
	cr_assert(!ret, "fi_ep_bind");

	ret = fi_enable(ep[0]);
	cr_assert(!ret, "fi_ep_enable");

	ret = fi_enable(ep[1]);
	cr_assert(!ret, "fi_ep_enable");

	target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
	assert(target_base);
	target = GNIT_ALIGN_BUFFER(char *, target_base);

	source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
	assert(source_base);
	source = GNIT_ALIGN_BUFFER(char *, source_base);

	rem_requested_key = USING_SCALABLE(fi) ? 1 : 0;
	loc_requested_key = USING_SCALABLE(fi) ? 2 : 0;

	ret = fi_mr_reg(dom,
			  target,
			  BUF_SZ,
			  FI_REMOTE_WRITE,
			  0,
			  rem_requested_key,
			  0,
			  &rem_mr,
			  &target);
	cr_assert_eq(ret, 0);

	ret = fi_mr_reg(dom,
			  source,
			  BUF_SZ,
			  FI_REMOTE_WRITE,
			  0,
			  loc_requested_key,
			  0,
			  &loc_mr,
			  &source);
	cr_assert_eq(ret, 0);

	if (USING_SCALABLE(fi)) {
		MR_ENABLE(rem_mr, target, BUF_SZ);
		MR_ENABLE(loc_mr, source, BUF_SZ);
	}

	mr_key = fi_mr_key(rem_mr);
}
static mca_mtl_base_module_t*
ompi_mtl_ofi_component_init(bool enable_progress_threads,
                            bool enable_mpi_threads)
{
    int ret, fi_version;
    struct fi_info *hints;
    struct fi_info *providers = NULL, *prov = NULL;
    struct fi_cq_attr cq_attr = {0};
    struct fi_av_attr av_attr = {0};
    char ep_name[FI_NAME_MAX] = {0};
    size_t namelen;

    /**
     * Hints to filter providers
     * See man fi_getinfo for a list of all filters
     * mode:  Select capabilities MTL is prepared to support.
     *        In this case, MTL will pass in context into communication calls
     * ep_type:  reliable datagram operation
     * caps:     Capabilities required from the provider.
     *           Tag matching is specified to implement MPI semantics.
     * msg_order: Guarantee that messages with same tag are ordered.
     */
    hints = fi_allocinfo();
    if (!hints) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: Could not allocate fi_info\n",
                            __FILE__, __LINE__);
        goto error;
    }
    hints->mode               = FI_CONTEXT;
    hints->ep_attr->type      = FI_EP_RDM;      /* Reliable datagram         */
    hints->caps               = FI_TAGGED;      /* Tag matching interface    */
    hints->tx_attr->msg_order = FI_ORDER_SAS;
    hints->rx_attr->msg_order = FI_ORDER_SAS;

    hints->domain_attr->threading        = FI_THREAD_UNSPEC;

    if (MTL_OFI_PROG_AUTO == control_progress) {
        hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
    } else {
        hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
    }

    if (MTL_OFI_PROG_MANUAL == data_progress) {
        hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
    } else {
        hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
    }

    if (MTL_OFI_AV_TABLE == av_type) {
        hints->domain_attr->av_type          = FI_AV_TABLE;
    } else {
        hints->domain_attr->av_type          = FI_AV_MAP;
    }

    hints->domain_attr->resource_mgmt    = FI_RM_ENABLED;

    /**
     * FI_VERSION provides binary backward and forward compatibility support
     * Specify the version of OFI is coded to, the provider will select struct
     * layouts that are compatible with this version.
     */
    fi_version = FI_VERSION(1, 0);

    /**
     * fi_getinfo:  returns information about fabric  services for reaching a
     * remote node or service.  this does not necessarily allocate resources.
     * Pass NULL for name/service because we want a list of providers supported.
     */
    ret = fi_getinfo(fi_version,    /* OFI version requested                    */
                     NULL,          /* Optional name or fabric to resolve       */
                     NULL,          /* Optional service name or port to request */
                     0ULL,          /* Optional flag                            */
                     hints,        /* In: Hints to filter providers            */
                     &providers);   /* Out: List of matching providers          */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_getinfo failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Select a provider from the list returned by fi_getinfo().
     */
    prov = select_ofi_provider(providers);
    if (!prov) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: select_ofi_provider: no provider found\n",
                            __FILE__, __LINE__);
        goto error;
    }


    /**
     * Open fabric
     * The getinfo struct returns a fabric attribute struct that can be used to
     * instantiate the virtual or physical network. This opens a "fabric
     * provider". See man fi_fabric for details.
     */
    ret = fi_fabric(prov->fabric_attr,    /* In:  Fabric attributes             */
                    &ompi_mtl_ofi.fabric, /* Out: Fabric handle                 */
                    NULL);                /* Optional context for fabric events */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_fabric failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Create the access domain, which is the physical or virtual network or
     * hardware port/collection of ports.  Returns a domain object that can be
     * used to create endpoints.  See man fi_domain for details.
     */
    ret = fi_domain(ompi_mtl_ofi.fabric,  /* In:  Fabric object                 */
                    prov,                 /* In:  Provider                      */
                    &ompi_mtl_ofi.domain, /* Out: Domain oject                  */
                    NULL);                /* Optional context for domain events */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_domain failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Create a transport level communication endpoint.  To use the endpoint,
     * it must be bound to completion counters or event queues and enabled,
     * and the resources consumed by it, such as address vectors, counters,
     * completion queues, etc.
     * see man fi_endpoint for more details.
     */
    ret = fi_endpoint(ompi_mtl_ofi.domain, /* In:  Domain object   */
                      prov,                /* In:  Provider        */
                      &ompi_mtl_ofi.ep,    /* Out: Endpoint object */
                      NULL);               /* Optional context     */
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_endpoint failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Save the maximum inject size.
     */
    ompi_mtl_ofi.max_inject_size = prov->tx_attr->inject_size;

    /**
     * Create the objects that will be bound to the endpoint.
     * The objects include:
     *     - completion queue for events
     *     - address vector of other endpoint addresses
     *     - dynamic memory-spanning memory region
     */
    cq_attr.format = FI_CQ_FORMAT_TAGGED;
    ret = fi_cq_open(ompi_mtl_ofi.domain, &cq_attr, &ompi_mtl_ofi.cq, NULL);
    if (ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_cq_open failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * The remote fi_addr will be stored in the ofi_endpoint struct.
     */

    av_attr.type = (MTL_OFI_AV_TABLE == av_type) ? FI_AV_TABLE: FI_AV_MAP;

    ret = fi_av_open(ompi_mtl_ofi.domain, &av_attr, &ompi_mtl_ofi.av, NULL);
    if (ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_av_open failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Bind the CQ and AV to the endpoint object.
     */
    ret = fi_ep_bind(ompi_mtl_ofi.ep,
                     (fid_t)ompi_mtl_ofi.cq,
                     FI_SEND | FI_RECV);
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_bind CQ-EP failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    ret = fi_ep_bind(ompi_mtl_ofi.ep,
                     (fid_t)ompi_mtl_ofi.av,
                     0);
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_bind AV-EP failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Enable the endpoint for communication
     * This commits the bind operations.
     */
    ret = fi_enable(ompi_mtl_ofi.ep);
    if (0 != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_enable failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    /**
     * Free providers info since it's not needed anymore.
     */
    fi_freeinfo(hints);
    hints = NULL;
    fi_freeinfo(providers);
    providers = NULL;

    /**
     * Get our address and publish it with modex.
     */
    namelen = sizeof(ep_name);
    ret = fi_getname((fid_t)ompi_mtl_ofi.ep, &ep_name[0], &namelen);
    if (ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: fi_getname failed: %s\n",
                            __FILE__, __LINE__, fi_strerror(-ret));
        goto error;
    }

    OFI_COMPAT_MODEX_SEND(ret,
                          &mca_mtl_ofi_component.super.mtl_version,
                          &ep_name,
                          namelen);
    if (OMPI_SUCCESS != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: modex_send failed: %d\n",
                            __FILE__, __LINE__, ret);
        goto error;
    }

    ompi_mtl_ofi.epnamelen = namelen;

    /**
     * Set the ANY_SRC address.
     */
    ompi_mtl_ofi.any_addr = FI_ADDR_UNSPEC;

    /**
     * Activate progress callback.
     */
    ret = opal_progress_register(ompi_mtl_ofi_progress_no_inline);
    if (OMPI_SUCCESS != ret) {
        opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
                            "%s:%d: opal_progress_register failed: %d\n",
                            __FILE__, __LINE__, ret);
        goto error;
    }

    return &ompi_mtl_ofi.base;

error:
    if (providers) {
        (void) fi_freeinfo(providers);
    }
    if (hints) {
        (void) fi_freeinfo(hints);
    }
    if (ompi_mtl_ofi.av) {
        (void) fi_close((fid_t)ompi_mtl_ofi.av);
    }
    if (ompi_mtl_ofi.cq) {
        (void) fi_close((fid_t)ompi_mtl_ofi.cq);
    }
    if (ompi_mtl_ofi.ep) {
        (void) fi_close((fid_t)ompi_mtl_ofi.ep);
    }
    if (ompi_mtl_ofi.domain) {
        (void) fi_close((fid_t)ompi_mtl_ofi.domain);
    }
    if (ompi_mtl_ofi.fabric) {
        (void) fi_close((fid_t)ompi_mtl_ofi.fabric);
    }
    return NULL;
}
Exemple #26
0
Test(endpoint, getsetopt)
{
	int ret;
	struct fid_ep *ep = NULL;
	uint64_t val;
	size_t len;

	ret = fi_endpoint(dom, fi, &ep, NULL);
	cr_assert(!ret, "fi_endpoint");

	/* Test bad params. */
	ret = fi_getopt(&ep->fid, !FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			(void *)&val, &len);
	cr_assert(ret == -FI_ENOPROTOOPT, "fi_getopt");

	ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_CM_DATA_SIZE+1,
			(void *)&val, &len);
	cr_assert(ret == -FI_ENOPROTOOPT, "fi_getopt");

	ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			NULL, &len);
	cr_assert(ret == -FI_EINVAL, "fi_getopt");

	ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			(void *)&val, NULL);
	cr_assert(ret == -FI_EINVAL, "fi_getopt");

	ret = fi_setopt(&ep->fid, !FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			(void *)&val, sizeof(size_t));
	cr_assert(ret == -FI_ENOPROTOOPT, "fi_setopt");

	ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, !FI_OPT_MIN_MULTI_RECV,
			(void *)&val, sizeof(size_t));
	cr_assert(ret == -FI_ENOPROTOOPT, "fi_setopt");

	ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			NULL, sizeof(size_t));
	cr_assert(ret == -FI_EINVAL, "fi_setopt");

	ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			(void *)&val, sizeof(size_t) - 1);
	cr_assert(ret == -FI_EINVAL, "fi_setopt");

	/* Test update. */
	ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			(void *)&val, &len);
	cr_assert(!ret, "fi_getopt");
	cr_assert(val == GNIX_OPT_MIN_MULTI_RECV_DEFAULT, "fi_getopt");
	cr_assert(len == sizeof(size_t), "fi_getopt");

	ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_CM_DATA_SIZE,
			(void *)&val, &len);
	cr_assert(!ret, "fi_getopt");
	cr_assert(val == GNIX_CM_DATA_MAX_SIZE, "fi_getopt");
	cr_assert(len == sizeof(size_t), "fi_getopt");

	val = 128;
	ret = fi_setopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			(void *)&val, sizeof(size_t));
	cr_assert(!ret, "fi_setopt");

	ret = fi_getopt(&ep->fid, FI_OPT_ENDPOINT, FI_OPT_MIN_MULTI_RECV,
			(void *)&val, &len);
	cr_assert(!ret, "fi_getopt");
	cr_assert(val == 128, "fi_getopt");
	cr_assert(len == sizeof(size_t), "fi_getopt");

	ret = fi_close(&ep->fid);
	cr_assert(!ret, "fi_close endpoint");
}
Exemple #27
0
static int alloc_ep_res(struct fi_info *fi)
{
	struct fi_cq_attr cq_attr;
	struct fi_av_attr av_attr;
	int ret;

	buffer_size = opts.user_options & FT_OPT_SIZE ?
			opts.transfer_size : test_size[TEST_CNT - 1].size;
	buf = malloc(MAX(buffer_size, sizeof(uint64_t)));
	if (!buf) {
		perror("malloc");
		return -1;
	}

	result = malloc(MAX(buffer_size, sizeof(uint64_t)));
	if (!result) {
		perror("malloc");
		return -1;
	}
	
	compare = malloc(MAX(buffer_size, sizeof(uint64_t)));
	if (!compare) {
		perror("malloc");
		return -1;
	}
	
	memset(&cq_attr, 0, sizeof cq_attr);
	cq_attr.format = FI_CQ_FORMAT_CONTEXT;
	cq_attr.wait_obj = FI_WAIT_NONE;
	cq_attr.size = 128;
	ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
	if (ret) {
		FT_PRINTERR("fi_cq_open", ret);
		goto err1;
	}

	ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
	if (ret) {
		FT_PRINTERR("fi_cq_open", ret);
		goto err2;
	}
	
	// registers local data buffer buff that specifies 
	// the first operand of the atomic operation
	ret = fi_mr_reg(dom, buf, MAX(buffer_size, sizeof(uint64_t)), 
		FI_REMOTE_READ | FI_REMOTE_WRITE, 0,
		get_mr_key(), 0, &mr, NULL);
	if (ret) {
		FT_PRINTERR("fi_mr_reg", ret);
		goto err3;
	}

	// registers local data buffer that stores initial value of 
	// the remote buffer
	ret = fi_mr_reg(dom, result, MAX(buffer_size, sizeof(uint64_t)), 
		FI_REMOTE_READ | FI_REMOTE_WRITE, 0,
		get_mr_key(), 0, &mr_result, NULL);
	if (ret) {
		FT_PRINTERR("fi_mr_reg", -ret);
		goto err4;
	}
	
	// registers local data buffer that contains comparison data
	ret = fi_mr_reg(dom, compare, MAX(buffer_size, sizeof(uint64_t)), 
		FI_REMOTE_READ | FI_REMOTE_WRITE, 0,
		get_mr_key(), 0, &mr_compare, NULL);
	if (ret) {
		FT_PRINTERR("fi_mr_reg", ret);
		goto err5;
	}

	memset(&av_attr, 0, sizeof av_attr);
	av_attr.type = fi->domain_attr->av_type ?
			fi->domain_attr->av_type : FI_AV_MAP;
	av_attr.count = 1;
	av_attr.name = NULL;

	ret = fi_av_open(dom, &av_attr, &av, NULL);
	if (ret) {
		FT_PRINTERR("fi_av_open", ret);
		goto err6;
	}
	
	ret = fi_endpoint(dom, fi, &ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		goto err7;
	}

	return 0;

err7:
	fi_close(&av->fid);
err6:
	fi_close(&mr_compare->fid);
err5:
	fi_close(&mr_result->fid);
err4:
	fi_close(&mr->fid);
err3:
	fi_close(&rcq->fid);
err2:
	fi_close(&scq->fid);
err1:
	free(buf);
	free(result);
	free(compare);
	
	return ret;
}
Exemple #28
0
/*
 * rpmem_fip_init_cq -- (internal) initialize completion queue(s)
 */
static int
rpmem_fip_init_cq(struct rpmem_fip *fip)
{
	int ret;

	struct fi_cq_attr cq_attr = {
		.size = fip->cq_size,
		.flags = 0,
		.format = FI_CQ_FORMAT_MSG,
		.wait_obj = FI_WAIT_UNSPEC,
		.signaling_vector = 0,
		.wait_cond = FI_CQ_COND_NONE,
		.wait_set = NULL,
	};

	ret = fi_cq_open(fip->domain, &cq_attr, &fip->cq, NULL);
	if (ret) {
		RPMEM_FI_ERR(ret, "opening completion queue");
		goto err_cq_open;
	}

	return 0;
err_cq_open:
	return -1;
}

/*
 * rpmem_fip_fini_cq -- (internal) deinitialize completion queue(s)
 */
static int
rpmem_fip_fini_cq(struct rpmem_fip *fip)
{
	return RPMEM_FI_CLOSE(fip->cq, "closing completion queue");
}

/*
 * rpmem_fip_init_ep -- (internal) initialize endpoint
 */
static int
rpmem_fip_init_ep(struct rpmem_fip *fip)
{
	int ret;

	/* create an endpoint */
	ret = fi_endpoint(fip->domain, fip->fi, &fip->ep, NULL);
	if (ret) {
		RPMEM_FI_ERR(ret, "allocating endpoint");
		goto err_endpoint;
	}

	/*
	 * Bind an event queue to an endpoint to get
	 * connection-related events for the endpoint.
	 */
	ret = fi_ep_bind(fip->ep, &fip->eq->fid, 0);
	if (ret) {
		RPMEM_FI_ERR(ret, "binding event queue to endpoint");
		goto err_ep_bind_eq;
	}

	/*
	 * Bind a completion queue to an endpoint to get completion
	 * events of specified inbound/outbound operations.
	 *
	 * FI_SELECTIVE_COMPLETION means all inbound/outbound operations
	 * must explicitly specify if the completion event should be
	 * generated or not using FI_COMPLETION flag.
	 *
	 * The completion events received are highly related to the
	 * persistency method used and are configured in lanes
	 * initialization specified for persistency method utilized.
	 */
	ret = fi_ep_bind(fip->ep, &fip->cq->fid,
			FI_RECV | FI_TRANSMIT | FI_SELECTIVE_COMPLETION);
	if (ret) {
		RPMEM_FI_ERR(ret, "binding completion queue to endpoint");
		goto err_ep_bind_cq;
	}

	/*
	 * Enable endpoint so it is possible to post inbound/outbound
	 * operations if required.
	 */
	ret = fi_enable(fip->ep);
	if (ret) {
		RPMEM_FI_ERR(ret, "activating endpoint");
		goto err_fi_enable;
	}

	return 0;
err_fi_enable:
err_ep_bind_cq:
err_ep_bind_eq:
err_endpoint:
	return ret;
}
Exemple #29
0
void rdm_str_addr_sr_setup_common(void)
{
	int ret = 0, i = 0, j = 0;
	struct fi_av_attr attr;

	memset(&attr, 0, sizeof(attr));
	attr.type = FI_AV_MAP;
	attr.count = NUMEPS;

	cq_attr.format = FI_CQ_FORMAT_TAGGED;
	cq_attr.size = 1024;
	cq_attr.wait_obj = 0;

	target_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
	assert(target_base);
	target = GNIT_ALIGN_BUFFER(char *, target_base);

	source_base = malloc(GNIT_ALIGN_LEN(BUF_SZ));
	assert(source_base);
	source = GNIT_ALIGN_BUFFER(char *, source_base);

	ret = fi_fabric(fi[0]->fabric_attr, &fab, NULL);
	cr_assert(!ret, "fi_fabric");

	for (i = 0; i < NUMEPS; i++) {
		ret = fi_domain(fab, fi[i], dom + i, NULL);
		cr_assert(!ret, "fi_domain");

		ret = fi_av_open(dom[i], &attr, av + i, NULL);
		cr_assert(!ret, "fi_av_open");

		ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
		cr_assert(!ret, "fi_endpoint");

		ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i, 0);
		cr_assert(!ret, "fi_cq_open");

		ret = fi_ep_bind(ep[i], &msg_cq[i]->fid, FI_SEND | FI_RECV);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_getname(&ep[i]->fid, NULL, &addrlen);
		cr_assert(addrlen > 0);

		ep_name[i] = malloc(addrlen);
		cr_assert(ep_name[i] != NULL);

		ret = fi_getname(&ep[i]->fid, ep_name[i], &addrlen);
		cr_assert(ret == FI_SUCCESS);
	}

	for (i = 0; i < NUMEPS; i++) {
		/*
		 * To test API-1.1: Reporting of unknown source addresses --
		 * only insert addresses into the sender's av
		 */
		if (i < (NUMEPS / 2)) {
			for (j = 0; j < NUMEPS; j++) {
				dbg_printf("Only does src EP insertions\n");
				ret = fi_av_insert(av[i], ep_name[j], 1,
						   &gni_addr[j],
						   0, NULL);
				cr_assert(ret == 1);
			}
		}

		ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
		cr_assert(!ret, "fi_ep_bind");

		ret = fi_enable(ep[i]);
		cr_assert(!ret, "fi_ep_enable");

	}
}
Exemple #30
0
static int alloc_ep_res(struct fi_info *fi)
{
	struct fi_cq_attr cq_attr;
	struct fi_av_attr av_attr;
	int ret;

	buffer_size = opts.user_options & FT_OPT_SIZE ?
			opts.transfer_size : test_size[TEST_CNT - 1].size;
	if (max_msg_size > 0 && buffer_size > max_msg_size) {
		buffer_size = max_msg_size;
	}
	if (buffer_size < fi->src_addrlen) {
		buffer_size = fi->src_addrlen;
	}
	buffer_size += prefix_len;
	buf = malloc(buffer_size);
	if (!buf) {
		perror("malloc");
		return -1;
	}
	buf_ptr = (char *)buf + prefix_len;

	memset(&cq_attr, 0, sizeof cq_attr);
	cq_attr.format = FI_CQ_FORMAT_CONTEXT;
	cq_attr.wait_obj = FI_WAIT_NONE;
	cq_attr.size = max_credits << 1;
	ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
	if (ret) {
		FT_PRINTERR("fi_cq_open", ret);
		goto err1;
	}

	ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
	if (ret) {
		FT_PRINTERR("fi_cq_open", ret);
		goto err2;
	}

	ret = fi_mr_reg(dom, buf, buffer_size, 0, 0, 0, 0, &mr, NULL);
	if (ret) {
		FT_PRINTERR("fi_mr_reg", ret);
		goto err3;
	}

	memset(&av_attr, 0, sizeof(av_attr));
	av_attr.type = fi->domain_attr->av_type ?
			fi->domain_attr->av_type : FI_AV_MAP;
	av_attr.name = NULL;
	av_attr.flags = 0;
	ret = fi_av_open(dom, &av_attr, &av, NULL);
	if (ret) {
		FT_PRINTERR("fi_av_open", ret);
		goto err4;
	}

	ret = fi_endpoint(dom, fi, &ep, NULL);
	if (ret) {
		FT_PRINTERR("fi_endpoint", ret);
		goto err5;
	}

	return 0;

err5:
	fi_close(&av->fid);
err4:
	fi_close(&mr->fid);
err3:
	fi_close(&rcq->fid);
err2:
	fi_close(&scq->fid);
err1:
	free(buf);
	return ret;
}