static int
usdf_fabric_close(fid_t fid)
{
struct usdf_fabric *fp;
int ret;
void *rv;
fp = fab_fidtou(fid);
if (atomic_get(&fp->fab_refcnt) > 0) {
return -FI_EBUSY;
}
/* Tell progression thread to exit */
fp->fab_exit = 1;
ret = usdf_fabric_wake_thread(fp);
if (ret != 0) {
return ret;
}
pthread_join(fp->fab_thread, &rv);
usdf_timer_deinit(fp);
close(fp->fab_eventfd);
close(fp->fab_epollfd);
close(fp->fab_arp_sockfd);
free(fp);
return 0;
}
static int
usdf_fabric_close(fid_t fid)
{
struct usdf_fabric *fp;
int ret;
void *rv;
USDF_TRACE("\n");
fp = fab_fidtou(fid);
if (ofi_atomic_get32(&fp->fab_refcnt) > 0) {
return -FI_EBUSY;
}
/* Tell progression thread to exit */
fp->fab_exit = 1;
free(fp->fab_attr.name);
free(fp->fab_attr.prov_name);
if (fp->fab_thread) {
ret = usdf_fabric_wake_thread(fp);
if (ret != 0) {
return ret;
}
pthread_join(fp->fab_thread, &rv);
}
usdf_timer_deinit(fp);
if (fp->fab_epollfd != -1) {
close(fp->fab_epollfd);
}
if (fp->fab_eventfd != -1) {
close(fp->fab_eventfd);
}
if (fp->fab_arp_sockfd != -1) {
close(fp->fab_arp_sockfd);
}
free(fp);
return 0;
}
static int
usdf_fabric_open(struct fi_fabric_attr *fattrp, struct fid_fabric **fabric,
void *context)
{
struct usdf_fabric *fp;
struct usdf_usnic_info *dp;
struct usdf_dev_entry *dep;
struct epoll_event ev;
struct sockaddr_in sin;
int ret;
int d;
/* Make sure this fabric exists */
dp = __usdf_devinfo;
for (d = 0; d < dp->uu_num_devs; ++d) {
dep = &dp->uu_info[d];
if (dep->ue_dev_ok &&
strcmp(fattrp->name, dep->ue_dattr.uda_devname) == 0) {
break;
}
}
if (d >= dp->uu_num_devs) {
USDF_INFO("device \"%s\" does not exit, returning -FI_ENODEV\n",
fattrp->name);
return -FI_ENODEV;
}
fp = calloc(1, sizeof(*fp));
if (fp == NULL) {
USDF_INFO("unable to allocate memory for fabric\n");
return -FI_ENOMEM;
}
fp->fab_epollfd = -1;
fp->fab_arp_sockfd = -1;
LIST_INIT(&fp->fab_domain_list);
fp->fab_attr.fabric = fab_utof(fp);
fp->fab_attr.name = strdup(fattrp->name);
fp->fab_attr.prov_name = strdup(USDF_PROV_NAME);
fp->fab_attr.prov_version = USDF_PROV_VERSION;
if (fp->fab_attr.name == NULL ||
fp->fab_attr.prov_name == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
fp->fab_fid.fid.fclass = FI_CLASS_FABRIC;
fp->fab_fid.fid.context = context;
fp->fab_fid.fid.ops = &usdf_fi_ops;
fp->fab_fid.ops = &usdf_ops_fabric;
fp->fab_dev_attrs = &dep->ue_dattr;
fp->fab_epollfd = epoll_create(1024);
if (fp->fab_epollfd == -1) {
ret = -errno;
USDF_INFO("unable to allocate epoll fd\n");
goto fail;
}
fp->fab_eventfd = eventfd(0, EFD_NONBLOCK | EFD_SEMAPHORE);
if (fp->fab_eventfd == -1) {
ret = -errno;
USDF_INFO("unable to allocate event fd\n");
goto fail;
}
fp->fab_poll_item.pi_rtn = usdf_fabric_progression_cb;
fp->fab_poll_item.pi_context = fp;
ev.events = EPOLLIN;
ev.data.ptr = &fp->fab_poll_item;
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_ADD, fp->fab_eventfd, &ev);
if (ret == -1) {
ret = -errno;
USDF_INFO("unable to EPOLL_CTL_ADD\n");
goto fail;
}
ret = pthread_create(&fp->fab_thread, NULL,
usdf_fabric_progression_thread, fp);
if (ret != 0) {
ret = -ret;
USDF_INFO("unable to create progress thread\n");
goto fail;
}
/* initialize timer subsystem */
ret = usdf_timer_init(fp);
if (ret != 0) {
USDF_INFO("unable to initialize timer\n");
goto fail;
}
/* create and bind socket for ARP resolution */
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = fp->fab_dev_attrs->uda_ipaddr_be;
fp->fab_arp_sockfd = socket(AF_INET, SOCK_DGRAM, 0);
if (fp->fab_arp_sockfd == -1) {
USDF_INFO("unable to create socket\n");
goto fail;
}
ret = bind(fp->fab_arp_sockfd, (struct sockaddr *) &sin, sizeof(sin));
if (ret == -1) {
ret = -errno;
goto fail;
}
atomic_init(&fp->fab_refcnt, 0);
fattrp->fabric = fab_utof(fp);
fattrp->prov_version = USDF_PROV_VERSION;
*fabric = fab_utof(fp);
USDF_INFO("successfully opened %s/%s\n", fattrp->name,
fp->fab_dev_attrs->uda_ifname);
return 0;
fail:
if (fp != NULL) {
if (fp->fab_epollfd != -1) {
close(fp->fab_epollfd);
}
if (fp->fab_eventfd != -1) {
close(fp->fab_eventfd);
}
if (fp->fab_arp_sockfd != -1) {
close(fp->fab_arp_sockfd);
}
usdf_timer_deinit(fp);
free(fp);
}
USDF_DEBUG("returning %d (%s)\n", ret, fi_strerror(-ret));
return ret;
}
