static int
usdf_handle_node_and_service(const char *node, const char *service,
uint64_t flags, void **src, void **dest,
const struct fi_info *hints, struct addrinfo **ai)
{
int ret;
struct sockaddr_in *sin;
if (node != NULL || service != NULL) {
if (hints && hints->addr_format == FI_ADDR_STR) {
/* FI_ADDR_STR can't have service param. */
if (service)
return -FI_EINVAL;
sin = usdf_format_to_sin(hints, node);
if (!sin)
/* This could be invalid or no memory. */
return -FI_EINVAL;
} else {
ret = getaddrinfo(node, service, NULL, ai);
if (ret != 0) {
USDF_DBG("getaddrinfo failed: %d: <%s>\n", ret,
gai_strerror(ret));
return ret;
}
sin = (struct sockaddr_in *)(*ai)->ai_addr;
}
if (flags & FI_SOURCE)
*src = usdf_sin_to_format(hints, sin, NULL);
else
*dest = usdf_sin_to_format(hints, sin, NULL);
}
return FI_SUCCESS;
}
/* Check all things related to a device. Make sure it's okay, the source address
* matches the requested address, the destination is reachable from the device,
* the device fabric name matches the requested fabric name, and the device
* domain name matches the requested domain name.
*
* @param version Libfabric API version used to verify the domain / fabric name.
* @param hints Hints passed to fi_getinfo.
* @param src Source address being requested.
* @param dest Destination address to communicate with.
* @param dep usNIC device entry being checked.
*
* @return true on success, false on failure. For debug logging can be enabled
* to see why a device was disqualified.
*/
static bool usdf_check_device(uint32_t version, const struct fi_info *hints,
void *src, void *dest,
struct usdf_dev_entry *dep)
{
char dest_str[INET_ADDRSTRLEN];
char src_str[INET_ADDRSTRLEN];
char dev_str[INET_ADDRSTRLEN];
struct usd_device_attrs *dap;
struct sockaddr_in *sin;
int reachable;
int ret;
reachable = -1;
dap = &dep->ue_dattr;
/* Skip the device if it has problems. */
if (!dep->ue_dev_ok) {
USDF_WARN_SYS(FABRIC, "skipping %s/%s device not ok\n",
dap->uda_devname, dap->uda_ifname);
return false;
}
/* If the given source address is not INADDR_ANY, compare against the
* device.
*/
if (src) {
sin = usdf_format_to_sin(hints, src);
if (sin->sin_addr.s_addr != INADDR_ANY) {
if (sin->sin_addr.s_addr != dap->uda_ipaddr_be) {
inet_ntop(AF_INET, &sin->sin_addr.s_addr,
src_str, sizeof(src_str));
inet_ntop(AF_INET, &dap->uda_ipaddr_be,
dev_str, sizeof(dev_str));
USDF_WARN_SYS(FABRIC,
"src addr<%s> != dev addr<%s>\n",
src_str, dev_str);
goto fail;
}
}
usdf_free_sin_if_needed(hints, sin);
}
/* Check that the given destination address is reachable from the
* interface.
*/
if (dest) {
sin = usdf_format_to_sin(hints, dest);
if (sin->sin_addr.s_addr != INADDR_ANY) {
ret = usdf_get_distance(dap, sin->sin_addr.s_addr,
&reachable);
if (ret) {
inet_ntop(AF_INET,
&sin->sin_addr.s_addr, dest_str,
sizeof(dest_str));
USDF_WARN_SYS(FABRIC,
"get_distance failed @ %s\n",
dest_str);
goto fail;
}
}
if (reachable == -1) {
inet_ntop(AF_INET, &sin->sin_addr.s_addr, dest_str,
sizeof(dest_str));
USDF_WARN_SYS(FABRIC,
"dest %s unreachable from %s/%s, skipping\n",
dest_str, dap->uda_devname,
dap->uda_ifname);
goto fail;
}
usdf_free_sin_if_needed(hints, sin);
}
/* Checks that the fabric name is correct for the given interface. The
* fabric name contains the CIDR notation for the interface.
*/
if (hints && hints->fabric_attr && hints->fabric_attr->name) {
if (!usdf_fabric_checkname(version, dap,
hints->fabric_attr->name))
return false;
}
/* Check that the domain name is correct for the given interface. The
* domain name is the device name.
*/
if (hints && hints->domain_attr && hints->domain_attr->name) {
if (!usdf_domain_checkname(version, dap,
hints->domain_attr->name))
return false;
}
return true;
fail:
usdf_free_sin_if_needed(hints, sin);
return false;
}
int
usdf_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct usdf_fabric *fp;
struct usdf_domain *udp;
struct sockaddr_in *sin;
size_t addrlen;
int ret;
#if ENABLE_DEBUG
char requested[INET_ADDRSTRLEN], actual[INET_ADDRSTRLEN];
#endif
USDF_TRACE_SYS(DOMAIN, "\n");
sin = NULL;
fp = fab_fidtou(fabric);
if (info->domain_attr != NULL) {
/* No versioning information available here. */
if (!usdf_domain_checkname(0, fp->fab_dev_attrs,
info->domain_attr->name)) {
USDF_WARN_SYS(DOMAIN, "domain name mismatch\n");
return -FI_ENODATA;
}
if (ofi_check_mr_mode(fabric->api_version,
OFI_MR_BASIC_MAP | FI_MR_LOCAL,
info->domain_attr->mr_mode)) {
/* the caller ignored our fi_getinfo results */
USDF_WARN_SYS(DOMAIN, "MR mode (%d) not supported\n",
info->domain_attr->mr_mode);
return -FI_ENODATA;
}
}
udp = calloc(1, sizeof *udp);
if (udp == NULL) {
USDF_DBG("unable to alloc mem for domain\n");
ret = -FI_ENOMEM;
goto fail;
}
USDF_DBG("uda_devname=%s\n", fp->fab_dev_attrs->uda_devname);
/*
* Make sure address format is good and matches this fabric
*/
switch (info->addr_format) {
case FI_SOCKADDR:
addrlen = sizeof(struct sockaddr);
sin = info->src_addr;
break;
case FI_SOCKADDR_IN:
addrlen = sizeof(struct sockaddr_in);
sin = info->src_addr;
break;
case FI_ADDR_STR:
sin = usdf_format_to_sin(info, info->src_addr);
goto skip_size_check;
default:
ret = -FI_EINVAL;
goto fail;
}
if (info->src_addrlen != addrlen) {
ret = -FI_EINVAL;
goto fail;
}
skip_size_check:
if (sin->sin_family != AF_INET ||
sin->sin_addr.s_addr != fp->fab_dev_attrs->uda_ipaddr_be) {
USDF_DBG_SYS(DOMAIN, "requested src_addr (%s) != fabric addr (%s)\n",
inet_ntop(AF_INET, &sin->sin_addr.s_addr,
requested, sizeof(requested)),
inet_ntop(AF_INET, &fp->fab_dev_attrs->uda_ipaddr_be,
actual, sizeof(actual)));
ret = -FI_EINVAL;
usdf_free_sin_if_needed(info, sin);
goto fail;
}
usdf_free_sin_if_needed(info, sin);
ret = usd_open(fp->fab_dev_attrs->uda_devname, &udp->dom_dev);
if (ret != 0) {
goto fail;
}
udp->dom_fid.fid.fclass = FI_CLASS_DOMAIN;
udp->dom_fid.fid.context = context;
udp->dom_fid.fid.ops = &usdf_fid_ops;
udp->dom_fid.ops = &usdf_domain_ops;
udp->dom_fid.mr = &usdf_domain_mr_ops;
ret = pthread_spin_init(&udp->dom_progress_lock,
PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&udp->dom_tx_ready);
TAILQ_INIT(&udp->dom_hcq_list);
udp->dom_info = fi_dupinfo(info);
if (udp->dom_info == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
if (udp->dom_info->dest_addr != NULL) {
free(udp->dom_info->dest_addr);
udp->dom_info->dest_addr = NULL;
}
ret = usdf_dom_rdc_alloc_data(udp);
if (ret != 0) {
goto fail;
}
udp->dom_fabric = fp;
LIST_INSERT_HEAD(&fp->fab_domain_list, udp, dom_link);
ofi_atomic_initialize32(&udp->dom_refcnt, 0);
ofi_atomic_inc32(&fp->fab_refcnt);
*domain = &udp->dom_fid;
return 0;
fail:
if (udp != NULL) {
if (udp->dom_info != NULL) {
fi_freeinfo(udp->dom_info);
}
if (udp->dom_dev != NULL) {
usd_close(udp->dom_dev);
}
usdf_dom_rdc_free_data(udp);
free(udp);
}
return ret;
}
