Beispiel #1
0
static int pp_listen_ctx(struct pingpong_context *ctx)
{
	int rc = 0;

	rc = fi_passive_ep(ctx->fabric, ctx->info, &ctx->lep, NULL);
	if (rc) {
		fprintf(stderr, "Unable to open listener endpoint\n");
		return 1;
	}

	/* Create listener EQ */
	rc = pp_eq_create(ctx);
	if (rc) {
		fprintf(stderr, "Unable to allocate listener resources\n");
		return 1;
	}

	rc = fi_pep_bind(ctx->lep, &ctx->eq->fid, 0);
	if (rc) {
		FT_PRINTERR("fi_pep_bind", rc);
		return 1;
	}

	rc = fi_listen(ctx->lep);
	if (rc) {
		FT_PRINTERR("fi_listen", rc);
		return 1;
	}

	printf("Listening for incoming connections...\n");
	return 0;
}
Beispiel #2
0
int cm_server_start(void)
{
	int ret;
	struct sockaddr_in loc_sa;

	cm_local_ip(&loc_sa);

	srv_hints = fi_allocinfo();
	srv_hints->fabric_attr->name = strdup("gni");
	srv_hints->ep_attr->type = FI_EP_MSG;
	srv_hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;

	ret = fi_getinfo(fi_version(), inet_ntoa(loc_sa.sin_addr),
			 DEF_PORT, FI_SOURCE, srv_hints, &srv_fi);
	cr_assert(!ret);

	ret = fi_fabric(srv_fi->fabric_attr, &srv_fab, NULL);
	cr_assert(!ret);

	ret = fi_eq_open(srv_fab, &eq_attr, &srv_eq, NULL);
	cr_assert(!ret);

	ret = fi_passive_ep(srv_fab, srv_fi, &srv_pep, NULL);
	cr_assert(!ret);

	ret = fi_pep_bind(srv_pep, &srv_eq->fid, 0);
	cr_assert(!ret);

	ret = fi_listen(srv_pep);
	cr_assert(!ret);

	dbg_printf("Server start complete.\n");

	return 0;
}
Beispiel #3
0
static int rxm_ep_msg_res_open(struct fi_info *rxm_info,
		struct util_domain *util_domain, struct rxm_ep *rxm_ep)
{
	struct rxm_fabric *rxm_fabric;
	struct rxm_domain *rxm_domain;
	struct fi_cq_attr cq_attr;
	int ret;

	ret = ofix_getinfo(rxm_prov.version, NULL, NULL, 0, &rxm_util_prov,
			rxm_info, rxm_alter_layer_info, rxm_alter_base_info,
			1, &rxm_ep->msg_info);
	if (ret)
		return ret;

	rxm_domain = container_of(util_domain, struct rxm_domain, util_domain);
	rxm_fabric = container_of(util_domain->fabric, struct rxm_fabric, util_fabric);

	ret = fi_passive_ep(rxm_fabric->msg_fabric, rxm_ep->msg_info, &rxm_ep->msg_pep, rxm_ep);
	if (ret) {
		FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open msg PEP\n");
		goto err1;
	}

	memset(&cq_attr, 0, sizeof(cq_attr));
	cq_attr.size = rxm_info->tx_attr->size + rxm_info->rx_attr->size;
	cq_attr.format = FI_CQ_FORMAT_MSG;

	ret = fi_cq_open(rxm_domain->msg_domain, &cq_attr, &rxm_ep->msg_cq, NULL);
	if (ret) {
		FI_WARN(&rxm_prov, FI_LOG_CQ, "Unable to open MSG CQ\n");
		goto err1;
	}

	ret = fi_srx_context(rxm_domain->msg_domain, rxm_ep->msg_info->rx_attr,
			&rxm_ep->srx_ctx, NULL);
	if (ret) {
		FI_WARN(&rxm_prov, FI_LOG_FABRIC, "Unable to open shared receive context\n");
		goto err2;
	}

	/* We don't care what's in the dest_addr at this point. We go by AV. */
	if (rxm_ep->msg_info->dest_addr) {
		free(rxm_ep->msg_info->dest_addr);
		rxm_ep->msg_info->dest_addr = NULL;
		rxm_ep->msg_info->dest_addrlen = 0;
	}

	/* Zero out the port as we would be creating multiple MSG EPs for a single
	 * RXM EP and we don't want address conflicts. */
	if (rxm_ep->msg_info->src_addr)
		((struct sockaddr_in *)(rxm_ep->msg_info->src_addr))->sin_port = 0;

	return 0;
err2:
	fi_close(&rxm_ep->msg_pep->fid);
err1:
	fi_freeinfo(rxm_ep->msg_info);
	return ret;
}
Beispiel #4
0
static int server_listen(void)
{
	struct fi_info *fi;
	int ret;

	/* Get fabric info */
	ret = fi_getinfo(FT_FIVERSION, NULL, opts.src_port, FI_SOURCE, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	/* Open the fabric */
	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto err0;
	}

	/* Open a passive endpoint */
	ret = fi_passive_ep(fab, fi, &pep, NULL);
	if (ret) {
		FT_PRINTERR("fi_passive_ep", ret);
		goto err1;
	}

	/* Allocate connection management resources */
	ret = alloc_cm_res();
	if (ret)
		goto err2;

	/* Bind EQ to passive endpoint */
	ret = fi_pep_bind(pep, &cmeq->fid, 0);
	if (ret) {
		FT_PRINTERR("fi_pep_bind", ret);
		goto err3;
	}

	/* Listen for incoming connections */
	ret = fi_listen(pep);
	if (ret) {
		FT_PRINTERR("fi_listen", ret);
		goto err3;
	}

	fi_freeinfo(fi);
	return 0;
err3:
	fi_close(&cmeq->fid);
err2:
	fi_close(&pep->fid);
err1:
	fi_close(&fab->fid);
err0:
	fi_freeinfo(fi);
	return ret;
}
Beispiel #5
0
static int server_listen(void)
{
	struct fi_info *fi;
	int ret;

	ret = fi_getinfo(FI_VERSION(1, 0), src_addr, port, FI_SOURCE, &hints, &fi);
	if (ret) {
		printf("fi_getinfo %s\n", strerror(-ret));
		return ret;
	}

	cq_data_size = fi->domain_attr->cq_data_size;

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		printf("fi_fabric %s\n", fi_strerror(-ret));
		goto err0;
	}

	ret = fi_passive_ep(fab, fi, &pep, NULL);
	if (ret) {
		printf("fi_passive_ep %s\n", fi_strerror(-ret));
		goto err1;
	}

	ret = alloc_cm_res();
	if (ret)
		goto err2;

	ret = fi_bind(&pep->fid, &cmeq->fid, 0);
	if (ret) {
		printf("fi_bind %s\n", fi_strerror(-ret));
		goto err3;
	}

	ret = fi_listen(pep);
	if (ret) {
		printf("fi_listen %s\n", fi_strerror(-ret));
		goto err3;
	}

	fi_freeinfo(fi);
	return 0;
err3:
	free_lres();
err2:
	fi_close(&pep->fid);
err1:
	fi_close(&fab->fid);
err0:
	fi_freeinfo(fi);
	return ret;
}
Beispiel #6
0
static int server_listen(void)
{
	struct fi_info *fi;
	int ret;

	ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE,
			hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	ret = fi_fabric(fi->fabric_attr, &fab, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto err0;
	}

	ret = fi_passive_ep(fab, fi, &pep, NULL);
	if (ret) {
		FT_PRINTERR("fi_passive_ep", ret);
		goto err1;
	}

	ret = alloc_cm_res();
	if (ret)
		goto err2;

	ret = fi_pep_bind(pep, &cmeq->fid, 0);
	if (ret) {
		FT_PRINTERR("fi_pep_bind", ret);
		goto err3;
	}

	ret = fi_listen(pep);
	if (ret) {
		FT_PRINTERR("fi_listen", ret);
		goto err3;
	}

	fi_freeinfo(fi);
	return 0;
err3:
	free_lres();
err2:
	fi_close(&pep->fid);
err1:
	fi_close(&fab->fid);
err0:
	fi_freeinfo(fi);
	return ret;
}
Beispiel #7
0
int ft_start_server(void)
{
	int ret;

	ret = fi_getinfo(FT_FIVERSION, opts.src_addr, opts.src_port, FI_SOURCE,
			 hints, &fi_pep);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		return ret;
	}

	ret = fi_fabric(fi_pep->fabric_attr, &fabric, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		return ret;
	}

	ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
	if (ret) {
		FT_PRINTERR("fi_eq_open", ret);
		return ret;
	}

	ret = fi_passive_ep(fabric, fi_pep, &pep, NULL);
	if (ret) {
		FT_PRINTERR("fi_passive_ep", ret);
		return ret;
	}

	ret = fi_pep_bind(pep, &eq->fid, 0);
	if (ret) {
		FT_PRINTERR("fi_pep_bind", ret);
		return ret;
	}

	ret = fi_listen(pep);
	if (ret) {
		FT_PRINTERR("fi_listen", ret);
		return ret;
	}

	return 0;
}
Beispiel #8
0
int ft_start_server(void)
{
	int ret;

	ret = ft_getinfo(hints, &fi_pep);
	if (ret)
		return ret;

	ret = fi_fabric(fi_pep->fabric_attr, &fabric, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		return ret;
	}

	ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
	if (ret) {
		FT_PRINTERR("fi_eq_open", ret);
		return ret;
	}

	ret = fi_passive_ep(fabric, fi_pep, &pep, NULL);
	if (ret) {
		FT_PRINTERR("fi_passive_ep", ret);
		return ret;
	}

	ret = fi_pep_bind(pep, &eq->fid, 0);
	if (ret) {
		FT_PRINTERR("fi_pep_bind", ret);
		return ret;
	}

	ret = fi_listen(pep);
	if (ret) {
		FT_PRINTERR("fi_listen", ret);
		return ret;
	}

	return 0;
}
Beispiel #9
0
static void fas_ep_setup(void)
{
	int ret, i, j;
	size_t addrlen = 0;

	fas_setup_common(fi_version());
	ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->rx_ctx_cnt);
	ctx_cnt = MIN(ctx_cnt, fi[0]->domain_attr->tx_ctx_cnt);

	for (i = 0; i < NUMEPS; i++) {
		fi[i]->ep_attr->tx_ctx_cnt = ctx_cnt;
		fi[i]->ep_attr->rx_ctx_cnt = ctx_cnt;

		ret = fi_domain(fab, fi[i], dom + i, NULL);
		cr_assert(!ret, "fi_domain returned: %s", fi_strerror(-ret));

		ret = fi_cntr_open(dom[i], &cntr_attr, send_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));

		ret = fi_cntr_open(dom[i], &cntr_attr, recv_cntr + i, 0);
		cr_assert(!ret, "fi_cntr_open returned: %s", fi_strerror(-ret));

		switch (ep_type) {
		case EP:
			ret = fi_endpoint(dom[i], fi[i], ep + i, NULL);
			cr_assert(!ret, "fi_endpoint returned: %s",
				  fi_strerror(-ret));
			break;
		case SEP:
			ret = fi_scalable_ep(dom[i], fi[i], ep + i,
					     NULL);
			cr_assert(!ret, "fi_endpoint returned: %s",
				  fi_strerror(-ret));
			break;
		case PEP:
			ret = fi_passive_ep(fab, fi[i], pep + i,
					    NULL);
			cr_assert(!ret, "fi_endpoint returned: %s",
				  fi_strerror(-ret));
			ret = fi_getname(get_fid[ep_type](i), NULL,
					 &addrlen);
			if (use_str_fmt) {
				cr_assert(addrlen == GNIX_FI_ADDR_STR_LEN,
					  "fi_getname returned: %s",
					  fi_strerror(-ret));
			} else {
				cr_assert(addrlen ==
					  sizeof(struct gnix_ep_name),
					  "fi_getname returned: %s",
					  fi_strerror(-ret));
			}
			ep_name_len[i] = addrlen;
			continue;
		default:
			cr_assert_fail("Unknown endpoint type.");
		}

		ret = fi_av_open(dom[i], &attr, av + i, NULL);
		cr_assert(!ret, "fi_av_open returned: %s", fi_strerror(-ret));

		switch (ep_type) {
		case EP:
		case PEP:
			ret = fi_cq_open(dom[i], &cq_attr, msg_cq + i,
					 0);
			cr_assert(!ret, "fi_cq_open returned: %s",
				  fi_strerror(-ret));

			ret = fi_ep_bind(ep[i], &msg_cq[i]->fid,
					 FI_SEND | FI_RECV);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));
			break;
		case SEP:
			dbg_printf(BLUE
					   "ctx_cnt = %d\n"
					   COLOR_RESET,
				   ctx_cnt);

			for (j = 0; j < ctx_cnt; j++) {
				ret = fi_tx_context(ep[i], j, NULL,
						    &tx_ep[i][j], NULL);
				cr_assert(!ret,
					  "fi_tx_context  returned: %s",
					  fi_strerror(-ret));

				ret = fi_cq_open(dom[i], &cq_attr,
						 &tx_cq[i][j],
						 NULL);
				cr_assert(!ret,
					  "fi_cq_open  returned: %s",
					  fi_strerror(-ret));

				ret = fi_rx_context(ep[i], j, NULL,
						    &rx_ep[i][j], NULL);
				cr_assert(!ret,
					  "fi_rx_context  returned: %s",
					  fi_strerror(-ret));

				ret = fi_cq_open(dom[i], &cq_attr,
						 &rx_cq[i][j],
						 NULL);
				cr_assert(!ret,
					  "fi_cq_open  returned: %s",
					  fi_strerror(-ret));
			}
			break;
		default:
			cr_assert_fail("Unknown endpoint type.");
		}

		ret = fi_getname(get_fid[ep_type](i), NULL, &addrlen);
		if (use_str_fmt) {
			cr_assert(addrlen > sizeof(struct gnix_ep_name),
				  "fi_getname returned: %s",
				  fi_strerror(-ret));
		} else {
			cr_assert(addrlen == sizeof(struct gnix_ep_name),
				  "fi_getname returned: %s",
				  fi_strerror(-ret));
		}

		ep_name[i] = malloc(addrlen);
		ep_name_len[i] = addrlen;

		dbg_printf(BLUE
				   "ep_name_len[%d] = %lu\n"
				   COLOR_RESET, i,
			   ep_name_len[i]);
		cr_assert(ep_name[i] != NULL, "malloc returned: %s",
			  strerror(errno));

		ret = fi_getname(get_fid[ep_type](i), ep_name[i], &addrlen);
		cr_assert(ret == FI_SUCCESS, "fi_getname returned: %s",
			  fi_strerror(-ret));
	}

	/* Just testing setname / getname for passive endpoints */
	if (ep_type == PEP)
		return;

	for (i = 0; i < NUMEPS; i++) {
		/*Insert all gni addresses into each av*/
		for (j = 0; j < NUMEPS; j++) {
			ret = fi_av_insert(av[i], ep_name[j], 1, &gni_addr[j],
					   0, NULL);
			cr_assert(ret == 1, "fi_av_insert returned: %s",
				  fi_strerror(-ret));
		}

		switch (ep_type) {
		case EP:
			ret = fi_ep_bind(ep[i], &av[i]->fid, 0);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));

			ret = fi_ep_bind(ep[i], &send_cntr[i]->fid,
					 FI_SEND);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));

			ret = fi_ep_bind(ep[i], &recv_cntr[i]->fid,
					 FI_RECV);
			cr_assert(!ret, "fi_ep_bind returned: %s",
				  fi_strerror(-ret));
			break;
		case SEP:
			ret = fi_scalable_ep_bind(ep[i], &av[i]->fid,
						  0);
			cr_assert(!ret,
				  "fi_scalable_ep_bind returned: %s",
				  fi_strerror(-ret));
			dbg_printf(BLUE
					   "ctx_cnt = %d\n"
					   COLOR_RESET,
				   ctx_cnt);
			for (j = 0; j < ctx_cnt; j++) {
				ret = fi_ep_bind(tx_ep[i][j],
						 &tx_cq[i][j]->fid,
						 FI_TRANSMIT);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_ep_bind(tx_ep[i][j],
						 &send_cntr[i]->fid,
						 FI_SEND);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_enable(tx_ep[i][j]);
				cr_assert(!ret,
					  "fi_enable  returned: %s",
					  fi_strerror(-ret));

				ret = fi_ep_bind(rx_ep[i][j],
						 &rx_cq[i][j]->fid,
						 FI_RECV);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_ep_bind(rx_ep[i][j],
						 &recv_cntr[i]->fid,
						 FI_RECV);
				cr_assert(!ret,
					  "fi_ep_bind  returned: %s",
					  fi_strerror(-ret));

				ret = fi_enable(rx_ep[i][j]);
				cr_assert(!ret,
					  "fi_enable  returned: %s",
					  fi_strerror(-ret));

			}
			break;
		case PEP:
			break;
		default:
			cr_assert_fail("Unknown endpoint type.");
		}

		ret = fi_enable(ep[i]);
		cr_assert(!ret, "fi_ep_enable returned: %s", fi_strerror(-ret));

		if (ep_type != SEP) {
			ret = fi_enable(ep[i]);
			cr_assert_eq(ret, -FI_EOPBADSTATE,
				     "fi_enable returned: %s",
				     fi_strerror(-ret));
		}
	}
}
Beispiel #10
0
static int setup_handle(void)
{
	static char buf[BUFSIZ];
	struct addrinfo *ai, aihints;
	const char *bound_addr_str;
	int ret;

	memset(&aihints, 0, sizeof aihints);
	aihints.ai_flags = AI_PASSIVE;
	ret = getaddrinfo(opts.src_addr, opts.src_port, &aihints, &ai);
	if (ret == EAI_SYSTEM) {
		FT_ERR("getaddrinfo for %s:%s: %s\n",
			opts.src_addr, opts.src_port, strerror(errno));
		return -ret;
	} else if (ret) {
		FT_ERR("getaddrinfo: %s\n", gai_strerror(ret));
		return -FI_ENODATA;
	}

	switch (ai->ai_family) {
	case AF_INET:
		hints->addr_format = FI_SOCKADDR_IN;
		break;
	case AF_INET6:
		hints->addr_format = FI_SOCKADDR_IN6;
		break;
	}

	/* Get fabric info */
	ret = fi_getinfo(FT_FIVERSION, opts.src_addr, NULL, FI_SOURCE, hints, &fi);
	if (ret) {
		FT_PRINTERR("fi_getinfo", ret);
		goto out;
	}
	free(fi->src_addr);
	fi->src_addr = NULL;
	fi->src_addrlen = 0;

	ret = fi_fabric(fi->fabric_attr, &fabric, NULL);
	if (ret) {
		FT_PRINTERR("fi_fabric", ret);
		goto out;
	}

	ret = fi_eq_open(fabric, &eq_attr, &eq, NULL);
	if (ret) {
		FT_PRINTERR("fi_eq_open", ret);
		goto out;
	}

	/* Open a passive endpoint */
	ret = fi_passive_ep(fabric, fi, &pep, NULL);
	if (ret) {
		FT_PRINTERR("fi_passive_ep", ret);
		goto out;
	}

	ret = fi_setname(&pep->fid, ai->ai_addr, ai->ai_addrlen);
	if (ret) {
		FT_PRINTERR("fi_setname", ret);
		goto out;
	}

	ret = fi_getname(&pep->fid, &bound_addr, &bound_addr_len);
	if (ret) {
		FT_PRINTERR("fi_getname", ret);
		goto out;
	}

	/* Verify port number */
	switch (ai->ai_family) {
	case AF_INET:
		if (bound_addr.sin.sin_port == 0) {
			FT_ERR("port number is 0 after fi_setname()\n");
			ret = -FI_EINVAL;
			goto out;
		}
		break;
	case AF_INET6:
		if (bound_addr.sin6.sin6_port == 0) {
			FT_ERR("port number is 0 after fi_setname()\n");
			ret = -FI_EINVAL;
			goto out;
		}
		break;
	}

	bound_addr_str = sockaddrstr(&bound_addr, bound_addr_len, buf, BUFSIZ);
	if (!bound_addr_str) {
		FT_ERR("Unable to get bound_addr as string!\n");
		ret = -FI_EINVAL;
		goto out;
	}
	printf("bound_addr: \"%s\"\n", bound_addr_str);

	hints->handle = &pep->fid;
out:
	freeaddrinfo(ai);
	return ret;
}
Beispiel #11
0
/*
 * rpmemd_fip_init_fabric_res -- initialize common fabric's resources
 */
static int
rpmemd_fip_init_fabric_res(struct rpmemd_fip *fip)
{
	int ret;
	ret = fi_fabric(fip->fi->fabric_attr, &fip->fabric, NULL);
	if (ret) {
		RPMEMD_FI_ERR(ret, "opening fabric domain");
		goto err_fi_fabric;
	}

	ret = fi_domain(fip->fabric, fip->fi, &fip->domain, NULL);
	if (ret) {
		RPMEMD_FI_ERR(ret, "opening fabric access domain");
		goto err_fi_domain;
	}

	struct fi_eq_attr eq_attr = {
		.size = 0,	/* use default */
		.flags = 0,
		.wait_obj = FI_WAIT_UNSPEC,
		.signaling_vector = 0,
		.wait_set = NULL,
	};

	ret = fi_eq_open(fip->fabric, &eq_attr, &fip->eq, NULL);
	if (ret) {
		RPMEMD_FI_ERR(ret, "opening event queue");
		goto err_eq_open;
	}

	ret = fi_passive_ep(fip->fabric, fip->fi, &fip->pep, NULL);
	if (ret) {
		RPMEMD_FI_ERR(ret, "allocating passive endpoint");
		goto err_pep;
	}

	ret = fi_pep_bind(fip->pep, &fip->eq->fid, 0);
	if (ret) {
		RPMEMD_FI_ERR(ret, "binding event queue to passive endpoint");
		goto err_pep_bind_eq;
	}

	return 0;
err_pep_bind_eq:
	RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint");
err_pep:
	RPMEMD_FI_CLOSE(fip->eq, "closing event queue");
err_eq_open:
	RPMEMD_FI_CLOSE(fip->domain, "closing fabric access domain");
err_fi_domain:
	RPMEMD_FI_CLOSE(fip->fabric, "closing fabric domain");
err_fi_fabric:
	return ret;
}

/*
 * rpmemd_fip_fini_fabric_res -- deinitialize common fabric resources
 */
static void
rpmemd_fip_fini_fabric_res(struct rpmemd_fip *fip)
{
	RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint");
	RPMEMD_FI_CLOSE(fip->eq, "closing event queue");
	RPMEMD_FI_CLOSE(fip->domain, "closing fabric access domain");
	RPMEMD_FI_CLOSE(fip->fabric, "closing fabric domain");
}
static void test_connect_with_accept_blocking_on_eq_fq_SERVER(void)
{
    int ret;

    printf("SERVER running\n");

    setup_ofi(NULL, NULL, FI_SOURCE);

#if WANT_FDS
    // Add the EQ FD to the epoll fd
    static struct epoll_event edt;
    memset(&edt, 0, sizeof(edt));
    edt.events = EPOLLIN;
    edt.data.u32 = 2222;
    ret = epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fidev.eq_fd, &edt);
    if (ret < 0) {
        error("server epoll_ctl failed");
    }
#endif

    // Make a PEP
    ret = fi_passive_ep(fidev.fabric, fidev.info, &fidev.pep, NULL);
    if (0 != ret) {
        error("fi_passive_ep failed");
    }

#if WANT_FIXED_PORT
    size_t ss = sizeof(sin);
    ret = fi_getname(&(fidev.pep->fid), &sin, &ss);
    if (0 != ret) {
        error("fi_setname failed");
    }
    sin.sin_port = htons(listen_port);

    // Bind the PEP to listen on a specific port
    ret = fi_setname(&(fidev.pep->fid), &sin, sizeof(sin));
    if (0 != ret) {
        error("fi_setname failed");
    }
#endif

    // Bind the EQ to the PEP
    ret = fi_pep_bind(fidev.pep, &fidev.eq->fid, 0);
    if (0 != ret) {
        error("fi_pep_bind failed");
    }

    // Listen
    ret = fi_listen(fidev.pep);
    if (0 != ret) {
        error("fi_listen failed");
    }

    // Get the actual address of this PEP
    struct sockaddr_in sinout;
    size_t s = sizeof(sinout);
    ret = fi_getname(&(fidev.pep->fid), &sinout, &s);
    if (0 != ret) {
        error("fi_setname failed");
    }
    sin.sin_family = sinout.sin_family;
    sin.sin_addr = sinout.sin_addr;
    sin.sin_port = sinout.sin_port;

    // Print server addr
    printf("SERVER listening on %s\n", addrstr(&sin));

    // Send our node (IP addr) and service (port) to the client
    snprintf(ofi_node, sizeof(ofi_node) - 1, "%s",
             inet_ntoa(sin.sin_addr));
    snprintf(ofi_service, sizeof(ofi_service) - 1, "%d",
             ntohs(sin.sin_port));
    MPI_Send(ofi_node, sizeof(ofi_node) - 1, MPI_CHAR,
             1, 101, MPI_COMM_WORLD);
    MPI_Send(ofi_service, sizeof(ofi_service) - 1, MPI_CHAR,
             1, 102, MPI_COMM_WORLD);
    printf("SERVER sent via MPI to client: %s / %s\n", ofi_node, ofi_service);

#if WANT_FDS
    // Now wait for the listen to complete
    int nevents;
    #define NEVENTS 32
    struct epoll_event events[NEVENTS];
    int timeout = 10000;
    while (1) {
        printf("SERVER blocking on epoll\n");
        nevents = epoll_wait(epoll_fd, events, NEVENTS, timeout);
        if (nevents < 0) {
            if (errno != EINTR) {
                error("server epoll wait failed");
            } else {
                continue;
            }
        } else {
            printf("SERVER successfully woke up from epoll! %d events\n", nevents);
            for (int i = 0; i < nevents; ++i) {
                if (events[i].data.u32 != 2222) {
                    error("server unexpected epoll return type");
                }
            }
            // If we got the expected event, then go read from the EQ
            break;
        }
    }
#endif

    // Wait for the FI_CONNREQ event
    uint32_t event;
    uint8_t *entry_buffer;
    size_t expected_len = sizeof(struct fi_eq_cm_entry) +
        sizeof(client_data);
    entry_buffer = (uint8_t*) calloc(1, expected_len);
    if (NULL == entry_buffer) {
        error("calloc failed");
    }
    struct fi_eq_cm_entry *entry = (struct fi_eq_cm_entry*) entry_buffer;

    while (1) {
        printf("SERVER waiting for FI_CONNREQ\n");
#if WANT_FDS
        ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0);
#else
        ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0);
#endif
        if (-FI_EAVAIL == ret) {
            printf("server fi_eq_sread failed because there's something in the error queue\n");
            char buffer[2048];
            struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer;
            ret = fi_eq_readerr(fidev.eq, err_entry, 0);
            printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno));
            error("sad panda");
        } else if (-EAGAIN == ret) {
            fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n");
            sleep(1);
            continue;
        } else if (ret < 0) {
            fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret);
            error("SERVER fi_eq_sread failed for some random reason");
        } else if (event != FI_CONNREQ) {
            error("SERVER got some unexpected event");
        } else if (ret != expected_len) {
            error("SERVER got wrong length back from fi_eq_sread");
        }

        uint32_t *d = (uint32_t*) entry->data;
        for (int i = 0; i < (sizeof(client_data) / sizeof(uint32_t)); ++i) {
            if (d[i] != client_data[i]) {
                printf("SERVER got wrong CM client data: d[%d]=%d, should be %d\n",
                       i, d[i], client_data[i]);
            }
        }

        printf("SERVER got FI_CONNREQ, correct size, and correct data -- yay!\n");
        break;
    }

    // Silly logistics: setup_ofi_active adds the fd to the epoll set.
    // But we already added it.  So for simplicity, just remove it
    // here so that setup_ofi_active() can re-add it.
#if WANT_FDS
    // Remove the EQ FD from the epoll fd
    ret = epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fidev.eq_fd, &edt);
    if (ret < 0) {
        error("server epoll_ctl DEL failed");
    }
#endif

    // Make an active endpoint
    setup_ofi_active(entry->info, &ficonn.ep);

    // Accept the incoming connection
    ret = fi_accept(ficonn.ep, (void*) server_data, sizeof(server_data));
    if (ret != 0) {
        printf("fi_accept: ret=%d, %s\n", ret, fi_strerror(-ret));
        error("SERVER fi_accept failed\n");
    }

    // Need to read and get a FI_CONNECTED event
    while (1) {
        printf("SERVER waiting for FI_CONNECTED\n");
#if WANT_FDS
        ret = fi_eq_read(fidev.eq, &event, entry, expected_len, 0);
#else
        ret = fi_eq_sread(fidev.eq, &event, entry, expected_len, -1, 0);
#endif
        if (-FI_EAVAIL == ret) {
            printf("server fi_eq_sread failed because there's something in the error queue\n");
            char buffer[2048];
            struct fi_eq_err_entry *err_entry = (struct fi_eq_err_entry*) buffer;
            ret = fi_eq_readerr(fidev.eq, err_entry, 0);
            printf("error code: %d (%s), prov err code: %d (%s)\n", err_entry->err, fi_strerror(err_entry->err), err_entry->prov_errno, fi_strerror(err_entry->prov_errno));
            error("sad panda");
        } else if (-EAGAIN == ret) {
            fprintf(stderr, "SERVER fi_eq_sread fail got -EAGAIN... trying again...\n");
            sleep(1);
            continue;
        } else if (ret < 0) {
            fprintf(stderr, "SERVER fi_eq_sread fail: %s (FI_EAVAIL = %d, -ret = %d)\n", fi_strerror(-ret), FI_EAVAIL, -ret);
            error("SERVER fi_eq_sread failed for some random reason");
        } else if (event != FI_CONNECTED) {
            error("SERVER got some unexpected event");
        }

        printf("SERVER got FI_CONNECTED -- yay!\n");
        break;
    }

    // Post a recv buffer for the client to send
    int msg[4] = { 0 };
    int len = sizeof(msg);
    printf("SERVER receiving len of %d\n", len);

    struct fid_mr no_mr;
    struct fid_mr *mr;
    void *recv_context = (void*) 0x17;
#if 0
    fi_mr_reg(fidev.domain, msg, len, FI_SEND | FI_RECV,
              0, (uint64_t)(uintptr_t) msg, 0, &mr, NULL);
#else
    // Try using no mr, like fi_msg_pingpong...
    memset(&no_mr, 0, sizeof(no_mr));
    mr = &no_mr;
#endif
    ret = fi_recv(ficonn.ep, msg, len,
                  fi_mr_desc(mr), 0, recv_context);
    if (ret < 0) {
        printf("fi_recv failed! %d, %s\n", ret, fi_strerror(-ret));
        MPI_Abort(MPI_COMM_WORLD, 37);
    }

    sleep(1);
    printf("SERVER posted receive -- waiting for client to send\n");
    MPI_Barrier(MPI_COMM_WORLD);

    // Wait for receive completion
    struct fi_cq_entry cqe;
    while (1) {
        ret = fi_cq_sread(ficonn.cq, &cqe, 1, 0, -1);
        if (cqe.op_context == recv_context) {
            printf("SERVER receive completed\n");
            break;
        } else {
            printf("SERVER got some other completion... continuing\n");
        }
    }

    printf("SERVER finished -- waiting for client before teardown\n");
    MPI_Barrier(MPI_COMM_WORLD);

    printf("SERVER tearing down\n");
    fi_close(&(mr->fid));
    teardown_ofi();
}
Beispiel #13
0
static int run_server(void)
{
	int i, ret;

	printf("cmatose: starting server\n");
	ret = fi_passive_ep(fabric, info, &pep, NULL);
	if (ret) {
		FT_PRINTERR("fi_passive_ep", ret);
		return ret;
	}

	ret = fi_pep_bind(pep, &eq->fid, 0);
	if (ret) {
		FT_PRINTERR("fi_ep_bind", ret);
		goto out;
	}

	ret = fi_listen(pep);
	if (ret) {
		FT_PRINTERR("fi_listen", ret);
		goto out;
	}

	ret = connect_events();
	if (ret)
		goto out;

	if (hints->tx_attr->size) {
		printf("initiating data transfers\n");
		for (i = 0; i < connections; i++) {
			ret = post_sends(&nodes[i]);
			if (ret)
				goto out;
		}

		printf("completing sends\n");
		ret = poll_cqs(SEND_CQ_INDEX);
		if (ret)
			goto out;

		printf("receiving data transfers\n");
		ret = poll_cqs(RECV_CQ_INDEX);
		if (ret)
			goto out;
		printf("data transfers complete\n");

	}

	printf("cmatose: disconnecting\n");
	for (i = 0; i < connections; i++) {
		if (!nodes[i].connected)
			continue;

		nodes[i].connected = 0;
		fi_shutdown(nodes[i].ep, 0);
	}

	ret = shutdown_events();
 	printf("disconnected\n");

out:
	fi_close(&pep->fid);
	return ret;
}