ssize_t
usdf_msg_inject(struct fid_ep *fep, const void *buf, size_t len,
fi_addr_t dest_addr)
{
struct usdf_ep *ep;
struct usdf_tx *tx;
struct usdf_msg_qe *wqe;
struct usdf_domain *udp;
if (len > USDF_MSG_MAX_INJECT_SIZE) {
USDF_WARN_SYS(EP_DATA,
"cannot inject more than inject_size bytes\n");
return -EINVAL;
}
ep = ep_ftou(fep);
tx = ep->ep_tx;
udp = ep->ep_domain;
if (TAILQ_EMPTY(&tx->t.msg.tx_free_wqe)) {
return -FI_EAGAIN;
}
pthread_spin_lock(&udp->dom_progress_lock);
wqe = usdf_msg_get_tx_wqe(tx);
wqe->ms_context = NULL;
memcpy(wqe->ms_inject_buf, buf, len);
wqe->ms_iov[0].iov_base = wqe->ms_inject_buf;
wqe->ms_iov[0].iov_len = len;
wqe->ms_last_iov = 0;
wqe->ms_cur_iov = 0;
wqe->ms_cur_ptr = buf;
wqe->ms_iov_resid = len;
wqe->ms_resid = len;
wqe->ms_length = len;
/* fi_inject() never signals a completion */
wqe->ms_signal_comp = 0;
/* add send to EP, and add EP to TX list if not present */
TAILQ_INSERT_TAIL(&ep->e.msg.ep_posted_wqe, wqe, ms_link);
usdf_msg_ep_ready(ep);
pthread_spin_unlock(&udp->dom_progress_lock);
usdf_domain_progress(udp);
return 0;
}
/* Given a connection request structure containing data, make a copy of the data
* that can be accessed in error entries on the EQ. The return value is the size
* of the data stored in the error entry. If the return value is a non-negative
* value, then the function has suceeded and the size and output data can be
* assumed to be valid. If the function fails, then the data will be NULL and
* the size will be a negative error value.
*/
static int usdf_cm_generate_err_data(struct usdf_eq *eq,
struct usdf_connreq *crp, void **data)
{
struct usdf_err_data_entry *err_data_entry;
struct usdf_connreq_msg *reqp;
size_t entry_size;
size_t data_size;
if (!eq || !crp || !data) {
USDF_DBG_SYS(EP_CTRL,
"eq, crp, or data is NULL.\n");
return -FI_EINVAL;
}
/* Initialize to NULL so data can't be used in the error case. */
*data = NULL;
reqp = (struct usdf_connreq_msg *) crp->cr_data;
/* This is a normal case, maybe there was no data. */
if (!reqp || !reqp->creq_datalen)
return 0;
data_size = reqp->creq_datalen;
entry_size = sizeof(*err_data_entry) + data_size;
err_data_entry = calloc(1, entry_size);
if (!err_data_entry) {
USDF_WARN_SYS(EP_CTRL,
"failed to allocate err data entry\n");
return -FI_ENOMEM;
}
/* This data should be copied and owned by the provider. Keep
* track of it in the EQ, this will be freed in the next EQ read
* call after it has been read.
*/
memcpy(err_data_entry->err_data, reqp->creq_data, data_size);
slist_insert_tail(&err_data_entry->entry, &eq->eq_err_data);
*data = err_data_entry->err_data;
return data_size;
}
static int usdf_pep_getname(fid_t fid, void *addr, size_t *addrlen)
{
int ret;
struct usdf_pep *pep;
size_t copylen;
USDF_TRACE_SYS(EP_CTRL, "\n");
ret = FI_SUCCESS;
pep = pep_fidtou(fid);
copylen = sizeof(pep->pep_src_addr);
memcpy(addr, &pep->pep_src_addr, MIN(copylen, *addrlen));
if (*addrlen < copylen) {
USDF_WARN_SYS(EP_CTRL, "*addrlen is too short\n");
ret = -FI_ETOOSMALL;
}
*addrlen = copylen;
return ret;
}
int
usdf_pep_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_pep **pep_o, void *context)
{
struct usdf_pep *pep;
struct usdf_fabric *fp;
struct sockaddr_in *sin;
int ret;
int optval;
USDF_TRACE_SYS(EP_CTRL, "\n");
if (!info) {
USDF_DBG_SYS(EP_CTRL, "null fi_info struct is invalid\n");
return -FI_EINVAL;
}
if (info->ep_attr->type != FI_EP_MSG) {
return -FI_ENODEV;
}
if ((info->caps & ~USDF_MSG_CAPS) != 0) {
return -FI_EBADF;
}
switch (info->addr_format) {
case FI_SOCKADDR:
if (((struct sockaddr *)info->src_addr)->sa_family != AF_INET) {
USDF_WARN_SYS(EP_CTRL, "non-AF_INET src_addr specified\n");
return -FI_EINVAL;
}
break;
case FI_SOCKADDR_IN:
break;
default:
USDF_WARN_SYS(EP_CTRL, "unknown/unsupported addr_format\n");
return -FI_EINVAL;
}
if (info->src_addrlen &&
info->src_addrlen != sizeof(struct sockaddr_in)) {
USDF_WARN_SYS(EP_CTRL, "unexpected src_addrlen\n");
return -FI_EINVAL;
}
fp = fab_ftou(fabric);
pep = calloc(1, sizeof(*pep));
if (pep == NULL) {
return -FI_ENOMEM;
}
pep->pep_fid.fid.fclass = FI_CLASS_PEP;
pep->pep_fid.fid.context = context;
pep->pep_fid.fid.ops = &usdf_pep_ops;
pep->pep_fid.ops = &usdf_pep_base_ops;
pep->pep_fid.cm = &usdf_pep_cm_ops;
pep->pep_fabric = fp;
pep->pep_state = USDF_PEP_UNBOUND;
pep->pep_sock = socket(AF_INET, SOCK_STREAM, 0);
if (pep->pep_sock == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_GETFL, 0);
if (ret == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_SETFL, ret | O_NONBLOCK);
if (ret == -1) {
ret = -errno;
goto fail;
}
/* set SO_REUSEADDR to prevent annoying "Address already in use" errors
* on successive runs of programs listening on a well known port */
optval = 1;
ret = setsockopt(pep->pep_sock, SOL_SOCKET, SO_REUSEADDR, &optval,
sizeof(optval));
if (ret == -1) {
ret = -errno;
goto fail;
}
pep->pep_info = fi_dupinfo(info);
if (!pep->pep_info) {
ret = -FI_ENOMEM;
goto fail;
}
if (info->src_addrlen == 0) {
/* Copy the source address information from the device
* attributes.
*/
pep->pep_info->src_addrlen = sizeof(struct sockaddr_in);
sin = calloc(1, pep->pep_info->src_addrlen);
if (!sin) {
USDF_WARN_SYS(EP_CTRL,
"calloc for src address failed\n");
goto fail;
}
sin->sin_family = AF_INET;
sin->sin_addr.s_addr = fp->fab_dev_attrs->uda_ipaddr_be;
pep->pep_info->src_addr = sin;
}
memcpy(&pep->pep_src_addr, pep->pep_info->src_addr,
pep->pep_info->src_addrlen);
/* initialize connreq freelist */
ret = pthread_spin_init(&pep->pep_cr_lock, PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&pep->pep_cr_free);
TAILQ_INIT(&pep->pep_cr_pending);
pep->pep_backlog = 10;
ret = usdf_pep_grow_backlog(pep);
if (ret != 0) {
goto fail;
}
atomic_initialize(&pep->pep_refcnt, 0);
atomic_inc(&fp->fab_refcnt);
*pep_o = pep_utof(pep);
return 0;
fail:
if (pep != NULL) {
usdf_pep_free_cr_lists(pep);
if (pep->pep_sock != -1) {
close(pep->pep_sock);
}
fi_freeinfo(pep->pep_info);
free(pep);
}
return ret;
}
static int usdf_pep_setname(fid_t fid, void *addr, size_t addrlen)
{
int ret;
struct usdf_pep *pep;
uint32_t req_addr_be;
socklen_t socklen;
char namebuf[INET_ADDRSTRLEN];
char servbuf[INET_ADDRSTRLEN];
USDF_TRACE_SYS(EP_CTRL, "\n");
pep = pep_fidtou(fid);
if (pep->pep_state != USDF_PEP_UNBOUND) {
USDF_WARN_SYS(EP_CTRL, "PEP cannot be bound\n");
return -FI_EOPBADSTATE;
}
if (((struct sockaddr *)addr)->sa_family != AF_INET) {
USDF_WARN_SYS(EP_CTRL, "non-AF_INET address given\n");
return -FI_EINVAL;
}
if (addrlen != sizeof(struct sockaddr_in)) {
USDF_WARN_SYS(EP_CTRL, "unexpected src_addrlen\n");
return -FI_EINVAL;
}
req_addr_be = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
namebuf[0] = '\0';
servbuf[0] = '\0';
ret = getnameinfo((struct sockaddr*)addr, addrlen,
namebuf, sizeof(namebuf),
servbuf, sizeof(servbuf),
NI_NUMERICHOST|NI_NUMERICSERV);
if (ret != 0)
USDF_WARN_SYS(EP_CTRL, "unable to getnameinfo(0x%x)\n",
req_addr_be);
if (req_addr_be != pep->pep_fabric->fab_dev_attrs->uda_ipaddr_be) {
USDF_WARN_SYS(EP_CTRL, "requested addr (%s:%s) does not match fabric addr\n",
namebuf, servbuf);
return -FI_EADDRNOTAVAIL;
}
ret = bind(pep->pep_sock, (struct sockaddr *)addr,
sizeof(struct sockaddr_in));
if (ret == -1) {
return -errno;
}
pep->pep_state = USDF_PEP_BOUND;
/* store the resulting port so that can implement getname() properly */
socklen = sizeof(pep->pep_src_addr);
ret = getsockname(pep->pep_sock, &pep->pep_src_addr, &socklen);
if (ret == -1) {
ret = -errno;
USDF_WARN_SYS(EP_CTRL, "getsockname failed %d (%s), PEP may be in bad state\n",
ret, strerror(-ret));
return ret;
}
return 0;
}
static int
usdf_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, const struct fi_info *hints, struct fi_info **info)
{
struct usdf_usnic_info *dp;
struct usdf_dev_entry *dep;
struct usd_device_attrs *dap;
struct fi_info *fi_first;
struct fi_info *fi_last;
struct addrinfo *ai;
void *src;
void *dest;
enum fi_ep_type ep_type;
int d;
int ret;
USDF_TRACE("\n");
fi_first = NULL;
fi_last = NULL;
ai = NULL;
src = NULL;
dest = NULL;
/*
* Get and cache usNIC device info
*/
if (__usdf_devinfo == NULL) {
ret = usdf_get_devinfo();
if (ret != 0) {
USDF_WARN("failed to usdf_get_devinfo, ret=%d (%s)\n",
ret, fi_strerror(-ret));
if (ret == -FI_ENODEV)
ret = -FI_ENODATA;
goto fail;
}
}
dp = __usdf_devinfo;
/* Check the hints up front and fail if they're invalid. */
if (hints) {
ret = usdf_validate_hints(version, hints);
if (ret) {
USDF_WARN_SYS(FABRIC, "hints failed to validate\n");
goto fail;
}
}
/* Get the src and dest if user specified. */
ret = usdf_handle_node_and_service(node, service, flags,
&src, &dest, hints, &ai);
if (ret) {
USDF_WARN_SYS(FABRIC, "failed to handle node and service.\n");
goto fail;
}
if (hints != NULL) {
if (dest == NULL && hints->dest_addr != NULL)
dest = hints->dest_addr;
if (src == NULL && hints->src_addr != NULL)
src = hints->src_addr;
}
for (d = 0; d < dp->uu_num_devs; ++d) {
dep = &dp->uu_info[d];
dap = &dep->ue_dattr;
/* If the device has an issue or the hints don't match the
* device information, then skip.
*/
if (!usdf_check_device(version, hints, src, dest, dep))
continue;
if (hints && hints->ep_attr)
ep_type = hints->ep_attr->type;
else
ep_type = FI_EP_UNSPEC;
if (ep_type == FI_EP_DGRAM || ep_type == FI_EP_UNSPEC) {
ret = usdf_fill_info_dgram(version, hints, src, dest,
dap, &fi_first, &fi_last);
if (ret != 0 && ret != -FI_ENODATA) {
goto fail;
}
}
if (ep_type == FI_EP_MSG || ep_type == FI_EP_UNSPEC) {
ret = usdf_fill_info_msg(version, hints, src, dest,
dap, &fi_first, &fi_last);
if (ret != 0 && ret != -FI_ENODATA) {
goto fail;
}
}
if (ep_type == FI_EP_RDM || ep_type == FI_EP_UNSPEC) {
ret = usdf_fill_info_rdm(version, hints, src, dest,
dap, &fi_first, &fi_last);
if (ret != 0 && ret != -FI_ENODATA) {
goto fail;
}
}
}
if (fi_first != NULL) {
*info = fi_first;
ret = 0;
} else {
ret = -FI_ENODATA;
}
fail:
if (ai)
freeaddrinfo(ai);
if (ret != 0) {
fi_freeinfo(fi_first);
USDF_INFO("returning %d (%s)\n", ret, fi_strerror(-ret));
}
return ret;
}
/* Check all things related to a device. Make sure it's okay, the source address
* matches the requested address, the destination is reachable from the device,
* the device fabric name matches the requested fabric name, and the device
* domain name matches the requested domain name.
*
* @param version Libfabric API version used to verify the domain / fabric name.
* @param hints Hints passed to fi_getinfo.
* @param src Source address being requested.
* @param dest Destination address to communicate with.
* @param dep usNIC device entry being checked.
*
* @return true on success, false on failure. For debug logging can be enabled
* to see why a device was disqualified.
*/
static bool usdf_check_device(uint32_t version, const struct fi_info *hints,
void *src, void *dest,
struct usdf_dev_entry *dep)
{
char dest_str[INET_ADDRSTRLEN];
char src_str[INET_ADDRSTRLEN];
char dev_str[INET_ADDRSTRLEN];
struct usd_device_attrs *dap;
struct sockaddr_in *sin;
int reachable;
int ret;
reachable = -1;
dap = &dep->ue_dattr;
/* Skip the device if it has problems. */
if (!dep->ue_dev_ok) {
USDF_WARN_SYS(FABRIC, "skipping %s/%s device not ok\n",
dap->uda_devname, dap->uda_ifname);
return false;
}
/* If the given source address is not INADDR_ANY, compare against the
* device.
*/
if (src) {
sin = usdf_format_to_sin(hints, src);
if (sin->sin_addr.s_addr != INADDR_ANY) {
if (sin->sin_addr.s_addr != dap->uda_ipaddr_be) {
inet_ntop(AF_INET, &sin->sin_addr.s_addr,
src_str, sizeof(src_str));
inet_ntop(AF_INET, &dap->uda_ipaddr_be,
dev_str, sizeof(dev_str));
USDF_WARN_SYS(FABRIC,
"src addr<%s> != dev addr<%s>\n",
src_str, dev_str);
goto fail;
}
}
usdf_free_sin_if_needed(hints, sin);
}
/* Check that the given destination address is reachable from the
* interface.
*/
if (dest) {
sin = usdf_format_to_sin(hints, dest);
if (sin->sin_addr.s_addr != INADDR_ANY) {
ret = usdf_get_distance(dap, sin->sin_addr.s_addr,
&reachable);
if (ret) {
inet_ntop(AF_INET,
&sin->sin_addr.s_addr, dest_str,
sizeof(dest_str));
USDF_WARN_SYS(FABRIC,
"get_distance failed @ %s\n",
dest_str);
goto fail;
}
}
if (reachable == -1) {
inet_ntop(AF_INET, &sin->sin_addr.s_addr, dest_str,
sizeof(dest_str));
USDF_WARN_SYS(FABRIC,
"dest %s unreachable from %s/%s, skipping\n",
dest_str, dap->uda_devname,
dap->uda_ifname);
goto fail;
}
usdf_free_sin_if_needed(hints, sin);
}
/* Checks that the fabric name is correct for the given interface. The
* fabric name contains the CIDR notation for the interface.
*/
if (hints && hints->fabric_attr && hints->fabric_attr->name) {
if (!usdf_fabric_checkname(version, dap,
hints->fabric_attr->name))
return false;
}
/* Check that the domain name is correct for the given interface. The
* domain name is the device name.
*/
if (hints && hints->domain_attr && hints->domain_attr->name) {
if (!usdf_domain_checkname(version, dap,
hints->domain_attr->name))
return false;
}
return true;
fail:
usdf_free_sin_if_needed(hints, sin);
return false;
}
static int usdf_validate_hints(uint32_t version, const struct fi_info *hints)
{
struct fi_fabric_attr *fattrp;
size_t size;
switch (hints->addr_format) {
case FI_FORMAT_UNSPEC:
case FI_SOCKADDR_IN:
size = sizeof(struct sockaddr_in);
break;
case FI_SOCKADDR:
size = sizeof(struct sockaddr);
break;
case FI_ADDR_STR:
if (hints->src_addr != NULL &&
strlen((char *)hints->src_addr) > USDF_ADDR_STR_LEN)
return -FI_ENODATA;
if (hints->dest_addr != NULL &&
strlen((char *)hints->dest_addr) > USDF_ADDR_STR_LEN)
return -FI_ENODATA;
goto skip_sockaddr_size_check;
default:
return -FI_ENODATA;
}
if (hints->src_addr != NULL && hints->src_addrlen < size) {
return -FI_ENODATA;
}
if (hints->dest_addr != NULL && hints->dest_addrlen < size) {
return -FI_ENODATA;
}
skip_sockaddr_size_check:
if (hints->ep_attr != NULL) {
switch (hints->ep_attr->protocol) {
case FI_PROTO_UNSPEC:
case FI_PROTO_UDP:
case FI_PROTO_RUDP:
break;
default:
return -FI_ENODATA;
}
if (hints->ep_attr->auth_key || hints->ep_attr->auth_key_size) {
USDF_WARN_SYS(EP_CTRL,
"\"authorization key\" is not supported in this provider.\n");
return -FI_ENODATA;
}
}
fattrp = hints->fabric_attr;
if (fattrp != NULL) {
if (fattrp->prov_version != 0 &&
fattrp->prov_version != USDF_PROV_VERSION) {
return -FI_ENODATA;
}
}
return FI_SUCCESS;
}
int
usdf_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct usdf_fabric *fp;
struct usdf_domain *udp;
struct sockaddr_in *sin;
size_t addrlen;
int ret;
#if ENABLE_DEBUG
char requested[INET_ADDRSTRLEN], actual[INET_ADDRSTRLEN];
#endif
USDF_TRACE_SYS(DOMAIN, "\n");
sin = NULL;
fp = fab_fidtou(fabric);
if (info->domain_attr != NULL) {
/* No versioning information available here. */
if (!usdf_domain_checkname(0, fp->fab_dev_attrs,
info->domain_attr->name)) {
USDF_WARN_SYS(DOMAIN, "domain name mismatch\n");
return -FI_ENODATA;
}
if (ofi_check_mr_mode(fabric->api_version,
OFI_MR_BASIC_MAP | FI_MR_LOCAL,
info->domain_attr->mr_mode)) {
/* the caller ignored our fi_getinfo results */
USDF_WARN_SYS(DOMAIN, "MR mode (%d) not supported\n",
info->domain_attr->mr_mode);
return -FI_ENODATA;
}
}
udp = calloc(1, sizeof *udp);
if (udp == NULL) {
USDF_DBG("unable to alloc mem for domain\n");
ret = -FI_ENOMEM;
goto fail;
}
USDF_DBG("uda_devname=%s\n", fp->fab_dev_attrs->uda_devname);
/*
* Make sure address format is good and matches this fabric
*/
switch (info->addr_format) {
case FI_SOCKADDR:
addrlen = sizeof(struct sockaddr);
sin = info->src_addr;
break;
case FI_SOCKADDR_IN:
addrlen = sizeof(struct sockaddr_in);
sin = info->src_addr;
break;
case FI_ADDR_STR:
sin = usdf_format_to_sin(info, info->src_addr);
goto skip_size_check;
default:
ret = -FI_EINVAL;
goto fail;
}
if (info->src_addrlen != addrlen) {
ret = -FI_EINVAL;
goto fail;
}
skip_size_check:
if (sin->sin_family != AF_INET ||
sin->sin_addr.s_addr != fp->fab_dev_attrs->uda_ipaddr_be) {
USDF_DBG_SYS(DOMAIN, "requested src_addr (%s) != fabric addr (%s)\n",
inet_ntop(AF_INET, &sin->sin_addr.s_addr,
requested, sizeof(requested)),
inet_ntop(AF_INET, &fp->fab_dev_attrs->uda_ipaddr_be,
actual, sizeof(actual)));
ret = -FI_EINVAL;
usdf_free_sin_if_needed(info, sin);
goto fail;
}
usdf_free_sin_if_needed(info, sin);
ret = usd_open(fp->fab_dev_attrs->uda_devname, &udp->dom_dev);
if (ret != 0) {
goto fail;
}
udp->dom_fid.fid.fclass = FI_CLASS_DOMAIN;
udp->dom_fid.fid.context = context;
udp->dom_fid.fid.ops = &usdf_fid_ops;
udp->dom_fid.ops = &usdf_domain_ops;
udp->dom_fid.mr = &usdf_domain_mr_ops;
ret = pthread_spin_init(&udp->dom_progress_lock,
PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&udp->dom_tx_ready);
TAILQ_INIT(&udp->dom_hcq_list);
udp->dom_info = fi_dupinfo(info);
if (udp->dom_info == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
if (udp->dom_info->dest_addr != NULL) {
free(udp->dom_info->dest_addr);
udp->dom_info->dest_addr = NULL;
}
ret = usdf_dom_rdc_alloc_data(udp);
if (ret != 0) {
goto fail;
}
udp->dom_fabric = fp;
LIST_INSERT_HEAD(&fp->fab_domain_list, udp, dom_link);
ofi_atomic_initialize32(&udp->dom_refcnt, 0);
ofi_atomic_inc32(&fp->fab_refcnt);
*domain = &udp->dom_fid;
return 0;
fail:
if (udp != NULL) {
if (udp->dom_info != NULL) {
fi_freeinfo(udp->dom_info);
}
if (udp->dom_dev != NULL) {
usd_close(udp->dom_dev);
}
usdf_dom_rdc_free_data(udp);
free(udp);
}
return ret;
}
int usdf_msg_fill_dom_attr(uint32_t version, const struct fi_info *hints,
struct fi_info *fi, struct usd_device_attrs *dap)
{
int ret;
struct fi_domain_attr defaults;
defaults = msg_dflt_domain_attr;
ret = usdf_domain_getname(version, dap, &defaults.name);
if (ret < 0)
return -FI_ENODATA;
if (!hints || !hints->domain_attr)
goto catch;
/* how to handle fi_thread_fid, fi_thread_completion, etc?
*/
switch (hints->domain_attr->threading) {
case FI_THREAD_UNSPEC:
case FI_THREAD_ENDPOINT:
break;
default:
return -FI_ENODATA;
}
/* how to handle fi_progress_manual?
*/
switch (hints->domain_attr->control_progress) {
case FI_PROGRESS_UNSPEC:
case FI_PROGRESS_AUTO:
break;
default:
return -FI_ENODATA;
}
switch (hints->domain_attr->data_progress) {
case FI_PROGRESS_UNSPEC:
case FI_PROGRESS_MANUAL:
break;
default:
return -FI_ENODATA;
}
switch (hints->domain_attr->resource_mgmt) {
case FI_RM_UNSPEC:
case FI_RM_DISABLED:
break;
default:
return -FI_ENODATA;
}
switch (hints->domain_attr->caps) {
case 0:
case FI_REMOTE_COMM:
break;
default:
USDF_WARN_SYS(DOMAIN,
"invalid domain capabilities\n");
return -FI_ENODATA;
}
if (usdf_check_mr_mode(version, hints, defaults.mr_mode))
return -FI_ENODATA;
if (hints->domain_attr->mr_cnt <= USDF_MSG_MR_CNT) {
defaults.mr_cnt = hints->domain_attr->mr_cnt;
} else {
USDF_DBG_SYS(DOMAIN, "mr_count exceeded provider limit\n");
return -FI_ENODATA;
}
catch:
static int
usdf_pep_listen(struct fid_pep *fpep)
{
struct usdf_pep *pep;
struct epoll_event ev;
struct usdf_fabric *fp;
socklen_t socklen;
int ret;
USDF_TRACE_SYS(EP_CTRL, "\n");
pep = pep_ftou(fpep);
fp = pep->pep_fabric;
switch (pep->pep_state) {
case USDF_PEP_UNBOUND:
case USDF_PEP_BOUND:
break;
case USDF_PEP_LISTENING:
USDF_WARN_SYS(EP_CTRL, "PEP already LISTENING!\n");
return -FI_EOPBADSTATE;
case USDF_PEP_ROBBED:
USDF_WARN_SYS(EP_CTRL,
"PEP already consumed, you may only fi_close() now\n");
return -FI_EOPBADSTATE;
default:
USDF_WARN_SYS(EP_CTRL, "unhandled case!\n");
abort();
}
/* we could already be bound if the user called fi_setname() or if we
* already did the bind in a previous call to usdf_pep_listen() and the
* listen(2) call failed */
if (pep->pep_state == USDF_PEP_UNBOUND) {
ret = bind(pep->pep_sock, (struct sockaddr *)&pep->pep_src_addr,
sizeof(struct sockaddr_in));
if (ret == -1) {
return -errno;
}
/* Get the actual port (since we may have requested
* port 0)
*/
socklen = sizeof(pep->pep_src_addr);
ret = getsockname(pep->pep_sock, &pep->pep_src_addr,
&socklen);
if (ret == -1)
return -errno;
pep->pep_state = USDF_PEP_BOUND;
}
ret = listen(pep->pep_sock, pep->pep_backlog);
if (ret != 0) {
return -errno;
}
pep->pep_state = USDF_PEP_LISTENING;
pep->pep_pollitem.pi_rtn = usdf_pep_listen_cb;
pep->pep_pollitem.pi_context = pep;
ev.events = EPOLLIN;
ev.data.ptr = &pep->pep_pollitem;
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_ADD, pep->pep_sock, &ev);
if (ret == -1) {
return -errno;
}
return 0;
}