/* A wrapper function to core utility function to check mr_mode bits.
* We need to check some more things for backward compatibility.
*/
int usdf_check_mr_mode(uint32_t version, const struct fi_info *hints,
uint64_t prov_mode)
{
int ret;
ret = ofi_check_mr_mode(version, prov_mode,
hints->domain_attr->mr_mode);
/* If ofi_check_mr_mode fails. */
if (ret) {
/* Is it because the user give 0 as mr_mode? */
if (hints->domain_attr->mr_mode == 0) {
if (FI_VERSION_LT(version, FI_VERSION(1, 5))) {
/* If the version is < 1.5, it is ok.
* We let this slide and catch it later on.
*/
return FI_SUCCESS;
} else if (hints->mode & FI_LOCAL_MR) {
/* If version is >= 1.5, we check fi_info mode
* for FI_LOCAL_MR for backward compatibility.
*/
return FI_SUCCESS;
}
}
}
return ret;
}
int
usdf_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct usdf_fabric *fp;
struct usdf_domain *udp;
struct sockaddr_in *sin;
size_t addrlen;
int ret;
#if ENABLE_DEBUG
char requested[INET_ADDRSTRLEN], actual[INET_ADDRSTRLEN];
#endif
USDF_TRACE_SYS(DOMAIN, "\n");
sin = NULL;
fp = fab_fidtou(fabric);
if (info->domain_attr != NULL) {
/* No versioning information available here. */
if (!usdf_domain_checkname(0, fp->fab_dev_attrs,
info->domain_attr->name)) {
USDF_WARN_SYS(DOMAIN, "domain name mismatch\n");
return -FI_ENODATA;
}
if (ofi_check_mr_mode(fabric->api_version,
OFI_MR_BASIC_MAP | FI_MR_LOCAL,
info->domain_attr->mr_mode)) {
/* the caller ignored our fi_getinfo results */
USDF_WARN_SYS(DOMAIN, "MR mode (%d) not supported\n",
info->domain_attr->mr_mode);
return -FI_ENODATA;
}
}
udp = calloc(1, sizeof *udp);
if (udp == NULL) {
USDF_DBG("unable to alloc mem for domain\n");
ret = -FI_ENOMEM;
goto fail;
}
USDF_DBG("uda_devname=%s\n", fp->fab_dev_attrs->uda_devname);
/*
* Make sure address format is good and matches this fabric
*/
switch (info->addr_format) {
case FI_SOCKADDR:
addrlen = sizeof(struct sockaddr);
sin = info->src_addr;
break;
case FI_SOCKADDR_IN:
addrlen = sizeof(struct sockaddr_in);
sin = info->src_addr;
break;
case FI_ADDR_STR:
sin = usdf_format_to_sin(info, info->src_addr);
goto skip_size_check;
default:
ret = -FI_EINVAL;
goto fail;
}
if (info->src_addrlen != addrlen) {
ret = -FI_EINVAL;
goto fail;
}
skip_size_check:
if (sin->sin_family != AF_INET ||
sin->sin_addr.s_addr != fp->fab_dev_attrs->uda_ipaddr_be) {
USDF_DBG_SYS(DOMAIN, "requested src_addr (%s) != fabric addr (%s)\n",
inet_ntop(AF_INET, &sin->sin_addr.s_addr,
requested, sizeof(requested)),
inet_ntop(AF_INET, &fp->fab_dev_attrs->uda_ipaddr_be,
actual, sizeof(actual)));
ret = -FI_EINVAL;
usdf_free_sin_if_needed(info, sin);
goto fail;
}
usdf_free_sin_if_needed(info, sin);
ret = usd_open(fp->fab_dev_attrs->uda_devname, &udp->dom_dev);
if (ret != 0) {
goto fail;
}
udp->dom_fid.fid.fclass = FI_CLASS_DOMAIN;
udp->dom_fid.fid.context = context;
udp->dom_fid.fid.ops = &usdf_fid_ops;
udp->dom_fid.ops = &usdf_domain_ops;
udp->dom_fid.mr = &usdf_domain_mr_ops;
ret = pthread_spin_init(&udp->dom_progress_lock,
PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&udp->dom_tx_ready);
TAILQ_INIT(&udp->dom_hcq_list);
udp->dom_info = fi_dupinfo(info);
if (udp->dom_info == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
if (udp->dom_info->dest_addr != NULL) {
free(udp->dom_info->dest_addr);
udp->dom_info->dest_addr = NULL;
}
ret = usdf_dom_rdc_alloc_data(udp);
if (ret != 0) {
goto fail;
}
udp->dom_fabric = fp;
LIST_INSERT_HEAD(&fp->fab_domain_list, udp, dom_link);
ofi_atomic_initialize32(&udp->dom_refcnt, 0);
ofi_atomic_inc32(&fp->fab_refcnt);
*domain = &udp->dom_fid;
return 0;
fail:
if (udp != NULL) {
if (udp->dom_info != NULL) {
fi_freeinfo(udp->dom_info);
}
if (udp->dom_dev != NULL) {
usd_close(udp->dom_dev);
}
usdf_dom_rdc_free_data(udp);
free(udp);
}
return ret;
}
int ofi_check_domain_attr(const struct fi_provider *prov, uint32_t api_version,
const struct fi_domain_attr *prov_attr,
const struct fi_info *user_info)
{
const struct fi_domain_attr *user_attr = user_info->domain_attr;
if (prov_attr->name && user_attr->name &&
strcasecmp(user_attr->name, prov_attr->name)) {
FI_INFO(prov, FI_LOG_CORE, "Unknown domain name\n");
FI_INFO_NAME(prov, prov_attr, user_attr);
return -FI_ENODATA;
}
if (fi_thread_level(user_attr->threading) <
fi_thread_level(prov_attr->threading)) {
FI_INFO(prov, FI_LOG_CORE, "Invalid threading model\n");
return -FI_ENODATA;
}
if (fi_progress_level(user_attr->control_progress) <
fi_progress_level(prov_attr->control_progress)) {
FI_INFO(prov, FI_LOG_CORE, "Invalid control progress model\n");
return -FI_ENODATA;
}
if (fi_progress_level(user_attr->data_progress) <
fi_progress_level(prov_attr->data_progress)) {
FI_INFO(prov, FI_LOG_CORE, "Invalid data progress model\n");
return -FI_ENODATA;
}
if (fi_resource_mgmt_level(user_attr->resource_mgmt) <
fi_resource_mgmt_level(prov_attr->resource_mgmt)) {
FI_INFO(prov, FI_LOG_CORE, "Invalid resource mgmt model\n");
return -FI_ENODATA;
}
if ((prov_attr->av_type != FI_AV_UNSPEC) &&
(user_attr->av_type != FI_AV_UNSPEC) &&
(prov_attr->av_type != user_attr->av_type)) {
FI_INFO(prov, FI_LOG_CORE, "Invalid AV type\n");
return -FI_ENODATA;
}
if (user_attr->cq_data_size > prov_attr->cq_data_size) {
FI_INFO(prov, FI_LOG_CORE, "CQ data size too large\n");
FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, cq_data_size);
return -FI_ENODATA;
}
if (ofi_check_mr_mode(prov, api_version, prov_attr->mr_mode, user_info))
return -FI_ENODATA;
if (user_attr->max_ep_stx_ctx > prov_attr->max_ep_stx_ctx) {
FI_INFO(prov, FI_LOG_CORE, "max_ep_stx_ctx greater than supported\n");
FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, max_ep_stx_ctx);
}
if (user_attr->max_ep_srx_ctx > prov_attr->max_ep_srx_ctx) {
FI_INFO(prov, FI_LOG_CORE, "max_ep_srx_ctx greater than supported\n");
FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, max_ep_srx_ctx);
}
/* following checks only apply to api 1.5 and beyond */
if (FI_VERSION_LT(api_version, FI_VERSION(1, 5)))
return 0;
if (user_attr->cntr_cnt > prov_attr->cntr_cnt) {
FI_INFO(prov, FI_LOG_CORE, "Cntr count too large\n");
return -FI_ENODATA;
}
if (user_attr->mr_iov_limit > prov_attr->mr_iov_limit) {
FI_INFO(prov, FI_LOG_CORE, "MR iov limit too large\n");
FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, mr_iov_limit);
return -FI_ENODATA;
}
if (user_attr->caps & ~(prov_attr->caps)) {
FI_INFO(prov, FI_LOG_CORE, "Requested domain caps not supported\n");
FI_INFO_CHECK(prov, prov_attr, user_attr, caps, FI_TYPE_CAPS);
return -FI_ENODATA;
}
if ((user_attr->mode & prov_attr->mode) != prov_attr->mode) {
FI_INFO(prov, FI_LOG_CORE, "Required domain mode missing\n");
FI_INFO_MODE(prov, prov_attr->mode, user_attr->mode);
return -FI_ENODATA;
}
if (user_attr->max_err_data > prov_attr->max_err_data) {
FI_INFO(prov, FI_LOG_CORE, "Max err data too large\n");
FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, max_err_data);
return -FI_ENODATA;
}
if (user_attr->mr_cnt > prov_attr->mr_cnt) {
FI_INFO(prov, FI_LOG_CORE, "MR count too large\n");
FI_INFO_CHECK_VAL(prov, prov_attr, user_attr, mr_cnt);
return -FI_ENODATA;
}
return 0;
}
