static int alloc_ep_res(struct fi_info *fi)
{
int ret;
ret = ft_alloc_ep_res(fi);
if (ret)
return ret;
if (tx_shared_ctx) {
ret = fi_stx_context(domain, fi->tx_attr, &stx_ctx, NULL);
if (ret) {
FT_PRINTERR("fi_stx_context", ret);
return ret;
}
}
if (rx_shared_ctx) {
ret = fi_srx_context(domain, fi->rx_attr, &srx_ctx, NULL);
if (ret) {
FT_PRINTERR("fi_srx_context", ret);
return ret;
}
}
return 0;
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_rx_attr rx_attr;
struct fi_tx_attr tx_attr;
int i, ret = 0;
ret = ft_alloc_bufs();
if (ret)
return ret;
remote_fi_addr = malloc(sizeof(*remote_fi_addr) * ep_cnt);
if (!remote_fi_addr) {
perror("malloc");
return -FI_ENOMEM;
}
av_attr.count = ep_cnt;
ret = ft_alloc_active_res(fi);
if (ret)
return ret;
/* TODO: avoid allocating EP when EP array is used. */
FT_CLOSE_FID(ep);
memset(&tx_attr, 0, sizeof tx_attr);
memset(&rx_attr, 0, sizeof rx_attr);
ret = fi_stx_context(domain, &tx_attr, &stx_ctx, NULL);
if (ret) {
FT_PRINTERR("fi_stx_context", ret);
return ret;
}
ret = fi_srx_context(domain, &rx_attr, &srx_ctx, NULL);
if (ret) {
FT_PRINTERR("fi_srx_context", ret);
return ret;
}
ep_array = calloc(ep_cnt, sizeof(*ep_array));
if (!ep_array) {
perror("malloc");
return ret;
}
for (i = 0; i < ep_cnt; i++) {
ret = fi_endpoint(domain, fi, &ep_array[i], NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
return ret;
}
}
return 0;
}
static inline int allocate_fabric_resources(struct fabric_info *info)
{
int ret = 0;
struct fi_av_attr av_attr = {0};
/* fabric domain: define domain of resources physical and logical*/
ret = fi_fabric(info->p_info->fabric_attr, &shmem_transport_ofi_fabfd, NULL);
if(ret!=0){
OFI_ERRMSG("fabric initialization failed\n");
return ret;
}
/*access domain: define communication resource limits/boundary within fabric domain */
ret = fi_domain(shmem_transport_ofi_fabfd, info->p_info,
&shmem_transport_ofi_domainfd,NULL);
if(ret!=0){
OFI_ERRMSG("domain initialization failed\n");
return ret;
}
/*transmit context: allocate one transmit context for this SHMEM PE
* and share it across different multiple endpoints. Since we have only
* one thread per PE, a single context is sufficient and allows more
* more PEs/node (i.e. doesn't exhaust contexts) */
ret = fi_stx_context(shmem_transport_ofi_domainfd, NULL, /* TODO: fill tx_attr */
&shmem_transport_ofi_stx, NULL);
if(ret!=0) {
OFI_ERRMSG("stx context initialization failed\n");
return ret;
}
/*AV table set-up for PE mapping*/
#ifdef USE_AV_MAP
av_attr.type = FI_AV_MAP;
addr_table = (fi_addr_t*) malloc(info->npes * sizeof(fi_addr_t));
#else
/* open Address Vector and bind the AV to the domain */
av_attr.type = FI_AV_TABLE;
addr_table = NULL;
#endif
ret = fi_av_open(shmem_transport_ofi_domainfd,
&av_attr,
&shmem_transport_ofi_avfd,
NULL);
if(ret!=0){
OFI_ERRMSG("av open failed\n");
return ret;
}
return ret;
}
static int alloc_ep_res(struct fi_info *fi)
{
struct fi_cq_attr cq_attr;
struct fi_rx_attr rx_attr;
struct fi_tx_attr tx_attr;
struct fi_av_attr av_attr;
int i, ret = 0;
buffer_size = test_size[TEST_CNT - 1].size;
buf = malloc(buffer_size);
remote_fi_addr = (fi_addr_t *)malloc(sizeof(*remote_fi_addr) * ep_cnt);
if (!buf || !remote_fi_addr) {
perror("malloc");
goto err1;
}
memset(&cq_attr, 0, sizeof cq_attr);
cq_attr.format = FI_CQ_FORMAT_CONTEXT;
cq_attr.wait_obj = FI_WAIT_NONE;
cq_attr.size = rx_depth;
memset(&tx_attr, 0, sizeof tx_attr);
memset(&rx_attr, 0, sizeof rx_attr);
ret = fi_stx_context(dom, &tx_attr, &stx_ctx, NULL);
if (ret) {
FT_PRINTERR("fi_stx_context", ret);
goto err1;
}
ret = fi_cq_open(dom, &cq_attr, &scq, NULL);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
goto err2;
}
ret = fi_srx_context(dom, &rx_attr, &srx_ctx, NULL);
if (ret) {
FT_PRINTERR("fi_srx_context", ret);
goto err3;
}
ret = fi_cq_open(dom, &cq_attr, &rcq, NULL);
if (ret) {
FT_PRINTERR("fi_cq_open", ret);
goto err4;
}
ret = fi_mr_reg(dom, buf, buffer_size, 0, 0, 0, 0, &mr, NULL);
if (ret) {
FT_PRINTERR("fi_mr_reg", ret);
goto err5;
}
memset(&av_attr, 0, sizeof av_attr);
av_attr.type = fi->domain_attr->av_type ?
fi->domain_attr->av_type : FI_AV_MAP;
av_attr.count = ep_cnt;
ret = fi_av_open(dom, &av_attr, &av, NULL);
if (ret) {
FT_PRINTERR("fi_av_open", ret);
goto err6;
}
ep = calloc(ep_cnt, sizeof(*ep));
if (!ep) {
perror("malloc");
goto err7;
}
for (i = 0; i < ep_cnt; i++) {
ret = fi_endpoint(dom, fi, &ep[i], NULL);
if (ret) {
FT_PRINTERR("fi_endpoint", ret);
goto err8;
}
}
return 0;
err8:
FT_CLOSEV(ep, ep_cnt);
err7:
fi_close(&av->fid);
err6:
fi_close(&mr->fid);
err5:
fi_close(&rcq->fid);
err4:
fi_close(&srx_ctx->fid);
err3:
fi_close(&scq->fid);
err2:
fi_close(&stx_ctx->fid);
err1:
free(buf);
free(remote_fi_addr);
return ret;
}
