int ofi_nd_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fab,
void *context)
{
OFI_UNUSED(context);
if (attr) {
if (attr->name && strcmp(attr->name, ofi_nd_prov.name))
return -FI_EINVAL;
if (attr->prov_name && strcmp(attr->prov_name, ofi_nd_prov.name))
return -FI_EINVAL;
if (attr->prov_version && attr->prov_version != ofi_nd_prov.version)
return -FI_EINVAL;
}
struct nd_fabric *fabric = (struct nd_fabric*)calloc(1, sizeof(*fabric));
if (!fabric)
return -FI_ENOMEM;
struct nd_fabric def = {
.fid = {
.fid = ofi_nd_fid,
.ops = &ofi_nd_fabric_ops
}
};
*fabric = def;
*fab = &fabric->fid;
fi_param_get_int(&ofi_nd_prov, "inlinethr", &gl_data.inline_thr);
fi_param_get_int(&ofi_nd_prov, "prepostcnt", &gl_data.prepost_cnt);
fi_param_get_int(&ofi_nd_prov, "prepostbuftcnt", &gl_data.prepost_buf_cnt);
return FI_SUCCESS;
}
static void rxd_init_env(void)
{
fi_param_get_int(&rxd_prov, "spin_count", &rxd_env.spin_count);
fi_param_get_bool(&rxd_prov, "retry", &rxd_env.retry);
fi_param_get_int(&rxd_prov, "max_peers", &rxd_env.max_peers);
fi_param_get_int(&rxd_prov, "max_unacked", &rxd_env.max_unacked);
}
static void psmx2_init_env(void)
{
if (getenv("OMPI_COMM_WORLD_RANK") || getenv("PMI_RANK"))
psmx2_env.name_server = 0;
fi_param_get_bool(&psmx2_prov, "name_server", &psmx2_env.name_server);
fi_param_get_bool(&psmx2_prov, "tagged_rma", &psmx2_env.tagged_rma);
fi_param_get_str(&psmx2_prov, "uuid", &psmx2_env.uuid);
fi_param_get_int(&psmx2_prov, "delay", &psmx2_env.delay);
fi_param_get_int(&psmx2_prov, "timeout", &psmx2_env.timeout);
fi_param_get_int(&psmx2_prov, "prog_interval", &psmx2_env.prog_interval);
fi_param_get_str(&psmx2_prov, "prog_affinity", &psmx2_env.prog_affinity);
}
static int fi_ibv_get_param_int(char *param_name, char *param_str,
size_t *param_default)
{
char *param_help;
size_t len, ret;
int param;
len = strlen(param_str) + 50;
param_help = malloc(len);
ret = snprintf(param_help, len, "%s (default: %zu)", param_str,
*param_default);
if (ret >= len) {
VERBS_WARN(FI_LOG_EP_DATA,
"param_help string size insufficient!\n");
free(param_help);
assert(0);
return -FI_ETOOSMALL;
}
fi_param_define(&fi_ibv_prov, param_name, FI_PARAM_INT, param_help);
if (!fi_param_get_int(&fi_ibv_prov, param_name, ¶m))
*param_default = param;
free(param_help);
return 0;
}
int rxd_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric,
void *context)
{
struct rxd_fabric *rxd_fabric;
struct fi_info hints, *dg_info;
int ret;
rxd_fabric = calloc(1, sizeof(*rxd_fabric));
if (!rxd_fabric)
return -FI_ENOMEM;
ret = ofi_fabric_init(&rxd_prov, &rxd_fabric_attr, attr,
&rxd_fabric->util_fabric, context);
if (ret)
goto err1;
memset(&hints, 0, sizeof hints);
if (!(hints.fabric_attr = calloc(1, sizeof(*hints.fabric_attr)))) {
ret = -FI_ENOMEM;
goto err2;
}
hints.fabric_attr->name = attr->name;
ret = ofi_get_core_info(attr->api_version, NULL, NULL, 0, &rxd_util_prov,
&hints, rxd_info_to_core, &dg_info);
if (ret) {
ret = -FI_EINVAL;
goto err3;
}
ret = fi_fabric(dg_info->fabric_attr, &rxd_fabric->dg_fabric, context);
if (ret) {
goto err4;
}
*fabric = &rxd_fabric->util_fabric.fabric_fid;
(*fabric)->fid.ops = &rxd_fabric_fi_ops;
(*fabric)->ops = &rxd_fabric_ops;
free(hints.fabric_attr);
fi_freeinfo(dg_info);
fi_param_get_int(&rxd_prov, "spin_count", &rxd_progress_spin_count);
return 0;
err4:
fi_freeinfo(dg_info);
err3:
free(hints.fabric_attr);
err2:
ofi_fabric_close(&rxd_fabric->util_fabric);
err1:
free(rxd_fabric);
return ret;
}
static int rxm_init_info(void)
{
int param;
if (!fi_param_get_int(&rxm_prov, "buffer_size", ¶m)) {
if (param > sizeof(struct rxm_pkt)) {
rxm_info.tx_attr->inject_size = param;
} else {
FI_WARN(&rxm_prov, FI_LOG_CORE,
"Requested buffer size too small\n");
return -FI_EINVAL;
}
} else {
rxm_info.tx_attr->inject_size = RXM_BUF_SIZE;
}
rxm_info.tx_attr->inject_size -= sizeof(struct rxm_pkt);
rxm_util_prov.info = &rxm_info;
return 0;
}
static int fi_ibv_alloc_info(struct ibv_context *ctx, struct fi_info **info,
const struct verbs_ep_domain *ep_dom)
{
struct fi_info *fi;
union ibv_gid gid;
size_t name_len;
int ret;
int param;
if (!(fi = fi_allocinfo()))
return -FI_ENOMEM;
fi->caps = ep_dom->caps;
fi->handle = NULL;
if (ep_dom->type == FI_EP_RDM) {
fi->mode = VERBS_RDM_MODE;
*(fi->tx_attr) = verbs_rdm_tx_attr;
} else {
fi->mode = VERBS_MODE;
*(fi->tx_attr) = verbs_tx_attr;
}
*(fi->rx_attr) = (ep_dom->type == FI_EP_RDM)
? verbs_rdm_rx_attr : verbs_rx_attr;
*(fi->ep_attr) = verbs_ep_attr;
*(fi->domain_attr) = verbs_domain_attr;
*(fi->fabric_attr) = verbs_fabric_attr;
fi->ep_attr->type = ep_dom->type;
fi->tx_attr->caps = ep_dom->caps;
fi->rx_attr->caps = ep_dom->caps;
ret = fi_ibv_get_device_attrs(ctx, fi);
if (ret)
goto err;
if (ep_dom->type == FI_EP_RDM) {
fi->tx_attr->inject_size = FI_IBV_RDM_DFLT_BUFFERED_SSIZE;
fi->tx_attr->iov_limit = 1;
fi->tx_attr->rma_iov_limit = 1;
if (!fi_param_get_int(&fi_ibv_prov, "rdm_buffer_size", ¶m)) {
if (param > sizeof (struct fi_ibv_rdm_tagged_rndv_header)) {
fi->tx_attr->inject_size = param;
} else {
FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
"rdm_buffer_size too small, should be greater then %d\n",
sizeof (struct fi_ibv_rdm_tagged_rndv_header));
ret = -FI_EINVAL;
goto err;
}
}
}
switch (ctx->device->transport_type) {
case IBV_TRANSPORT_IB:
if(ibv_query_gid(ctx, 1, 0, &gid)) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_query_gid", errno);
ret = -errno;
goto err;
}
name_len = strlen(VERBS_IB_PREFIX) + INET6_ADDRSTRLEN;
if (!(fi->fabric_attr->name = calloc(1, name_len + 1))) {
ret = -FI_ENOMEM;
goto err;
}
snprintf(fi->fabric_attr->name, name_len, VERBS_IB_PREFIX "%lx",
gid.global.subnet_prefix);
fi->ep_attr->protocol = (ep_dom == &verbs_msg_domain) ?
FI_PROTO_RDMA_CM_IB_RC : FI_PROTO_IB_RDM;
break;
case IBV_TRANSPORT_IWARP:
fi->fabric_attr->name = strdup(VERBS_IWARP_FABRIC);
if (!fi->fabric_attr->name) {
ret = -FI_ENOMEM;
goto err;
}
if (ep_dom == &verbs_msg_domain) {
fi->ep_attr->protocol = FI_PROTO_IWARP;
fi->tx_attr->op_flags = VERBS_TX_OP_FLAGS_IWARP;
} else {
fi->ep_attr->protocol = FI_PROTO_IWARP_RDM;
fi->tx_attr->op_flags = VERBS_TX_OP_FLAGS_IWARP_RDM;
}
break;
default:
FI_INFO(&fi_ibv_prov, FI_LOG_CORE, "Unknown transport type\n");
ret = -FI_ENODATA;
goto err;
}
name_len = strlen(ctx->device->name) + strlen(ep_dom->suffix);
fi->domain_attr->name = malloc(name_len + 1);
if (!fi->domain_attr->name) {
ret = -FI_ENOMEM;
goto err;
}
snprintf(fi->domain_attr->name, name_len + 1, "%s%s",
ctx->device->name, ep_dom->suffix);
fi->domain_attr->name[name_len] = '\0';
*info = fi;
return 0;
err:
fi_freeinfo(fi);
return ret;
}
