int fi_ibv_process_xrc_connreq(struct fi_ibv_ep *ep,
struct fi_ibv_connreq *connreq)
{
struct fi_ibv_xrc_ep *xrc_ep = container_of(ep, struct fi_ibv_xrc_ep,
base_ep);
int ret;
assert(ep->info->src_addr);
assert(ep->info->dest_addr);
xrc_ep->conn_setup = calloc(1, sizeof(*xrc_ep->conn_setup));
if (!xrc_ep->conn_setup)
return -FI_ENOMEM;
/* This endpoint was created on the passive side of a connection
* request. The reciprocal connection request will go back to the
* passive port indicated by the active side */
ofi_addr_set_port(ep->info->src_addr, 0);
ofi_addr_set_port(ep->info->dest_addr, connreq->xrc.port);
ret = fi_ibv_create_ep(NULL, NULL, 0, ep->info, NULL, &ep->id);
if (ret) {
VERBS_WARN(FI_LOG_EP_CTRL,
"Creation of INI cm_id failed %d\n", ret);
goto create_err;
}
xrc_ep->tgt_id = connreq->id;
xrc_ep->tgt_id->context = &ep->util_ep.ep_fid.fid;
return FI_SUCCESS;
create_err:
free(xrc_ep->conn_setup);
return ret;
}
int fi_ibv_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct rdma_cm_id *id = NULL;
struct rdma_addrinfo *rai;
int ret;
ret = fi_ibv_init_info();
if (ret)
goto out;
ret = fi_ibv_create_ep(node, service, flags, hints, &rai, &id);
if (ret)
goto out;
if (id->verbs) {
ret = fi_ibv_get_matching_info(ibv_get_device_name(id->verbs->device),
hints, rai, info);
} else {
ret = fi_ibv_get_matching_info(NULL, hints, rai, info);
}
ofi_alter_info(*info, hints);
fi_ibv_destroy_ep(rai, &id);
out:
if (!ret || ret == -FI_ENOMEM || ret == -FI_ENODEV)
return ret;
else
return -FI_ENODATA;
}
int fi_ibv_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct rdma_cm_id *id = NULL;
struct rdma_addrinfo *rai;
struct fi_ibv_rdm_cm rdm_cm;
int ret;
ret = fi_ibv_init_info();
if (ret)
goto out;
if (FI_IBV_EP_TYPE_IS_RDM(hints)) {
memset(&rdm_cm, 0, sizeof(struct fi_ibv_rdm_cm));
ret = fi_ibv_create_ep(node, service, flags, hints, &rai,
&(rdm_cm.listener));
id = rdm_cm.listener;
} else {
ret = fi_ibv_create_ep(node, service, flags, hints, &rai, &id);
}
if (ret)
goto out;
if (id->verbs) {
ret = fi_ibv_get_matching_info(ibv_get_device_name(id->verbs->device),
hints, rai, info);
} else {
ret = fi_ibv_get_matching_info(NULL, hints, rai, info);
}
ofi_alter_info(*info, hints);
if (hints && hints->ep_attr)
fi_ibv_destroy_ep(hints->ep_attr->type, rai,
FI_IBV_EP_TYPE_IS_RDM(hints) ? &(rdm_cm.listener) : &id);
out:
if (!ret || ret == -FI_ENOMEM || ret == -FI_ENODEV)
return ret;
else
return -FI_ENODATA;
}
int fi_ibv_open_ep(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct fi_ibv_domain *dom;
struct fi_ibv_msg_ep *_ep;
struct fi_ibv_connreq *connreq;
struct fi_ibv_pep *pep;
struct fi_info *fi;
int ret;
dom = container_of(domain, struct fi_ibv_domain, domain_fid);
if (strcmp(dom->verbs->device->name, info->domain_attr->name)) {
VERBS_INFO(FI_LOG_DOMAIN, "Invalid info->domain_attr->name\n");
return -FI_EINVAL;
}
fi = fi_ibv_get_verbs_info(info->domain_attr->name);
if (!fi) {
VERBS_INFO(FI_LOG_DOMAIN, "Unable to find matching verbs_info\n");
return -FI_EINVAL;
}
if (info->ep_attr) {
ret = fi_ibv_check_ep_attr(info->ep_attr, fi);
if (ret)
return ret;
}
if (info->tx_attr) {
ret = fi_ibv_check_tx_attr(info->tx_attr, info, fi);
if (ret)
return ret;
}
if (info->rx_attr) {
ret = fi_ibv_check_rx_attr(info->rx_attr, info, fi);
if (ret)
return ret;
}
_ep = fi_ibv_alloc_msg_ep(info);
if (!_ep)
return -FI_ENOMEM;
if (!info->handle) {
ret = fi_ibv_create_ep(NULL, NULL, 0, info, NULL, &_ep->id);
if (ret)
goto err;
} else if (info->handle->fclass == FI_CLASS_CONNREQ) {
connreq = container_of(info->handle, struct fi_ibv_connreq, handle);
_ep->id = connreq->id;
} else if (info->handle->fclass == FI_CLASS_PEP) {
int fi_ibv_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct rdma_cm_id *id = NULL;
struct rdma_addrinfo *rai;
int ret;
ret = fi_ibv_init_info();
if (ret)
goto out;
ret = fi_ibv_create_ep(node, service, flags, hints, &rai, &id);
if (ret)
goto out;
if (id->verbs) {
ret = fi_ibv_get_matching_info(ibv_get_device_name(id->verbs->device),
hints, rai, info);
} else {
ret = fi_ibv_get_matching_info(NULL, hints, rai, info);
if (!ret && !(flags & FI_SOURCE) && !node
&& (!hints || (!hints->src_addr && !hints->dest_addr))) {
ret = fi_ibv_getifaddrs(service, flags, *info);
if (ret) {
fi_freeinfo(*info);
fi_ibv_destroy_ep(rai, &id);
goto out;
}
}
}
if (!ret) {
ret = fi_ibv_rdm_remove_nonaddr_info(info);
}
ofi_alter_info(*info, hints);
fi_ibv_destroy_ep(rai, &id);
out:
if (!ret || ret == -FI_ENOMEM || ret == -FI_ENODEV)
return ret;
else
return -FI_ENODATA;
}
static int fi_ibv_msg_ep_setname(fid_t ep_fid, void *addr, size_t addrlen)
{
struct fi_ibv_msg_ep *ep;
void *save_addr;
struct rdma_cm_id *id;
int ret;
ep = container_of(ep_fid, struct fi_ibv_msg_ep, ep_fid);
if (addrlen != ep->info->src_addrlen) {
FI_INFO(&fi_ibv_prov, FI_LOG_EP_CTRL,"addrlen expected: %d, got: %d.\n",
ep->info->src_addrlen, addrlen);
return -FI_EINVAL;
}
save_addr = ep->info->src_addr;
ep->info->src_addr = malloc(ep->info->src_addrlen);
if (!ep->info->src_addr) {
ret = -FI_ENOMEM;
goto err1;
}
memcpy(ep->info->src_addr, addr, ep->info->src_addrlen);
ret = fi_ibv_create_ep(NULL, NULL, 0, ep->info, NULL, &id);
if (ret)
goto err2;
if (ep->id)
rdma_destroy_ep(ep->id);
ep->id = id;
free(save_addr);
return 0;
err2:
free(ep->info->src_addr);
err1:
ep->info->src_addr = save_addr;
return ret;
}
/* Builds a list of interfaces that correspond to active verbs devices */
static int fi_ibv_getifaddrs(struct dlist_entry *verbs_devs)
{
struct ifaddrs *ifaddr, *ifa;
char name[INET6_ADDRSTRLEN];
struct rdma_addrinfo *rai;
struct rdma_cm_id *id;
const char *ret_ptr;
int ret, num_verbs_ifs = 0;
char *iface = NULL;
size_t iface_len = 0;
int exact_match = 0;
ret = getifaddrs(&ifaddr);
if (ret) {
VERBS_WARN(FI_LOG_FABRIC,
"Unable to get interface addresses\n");
return ret;
}
/* select best iface name based on user's input */
if (fi_param_get_str(&fi_ibv_prov, "iface", &iface) == FI_SUCCESS) {
iface_len = strlen(iface);
if (iface_len > IFNAMSIZ) {
VERBS_INFO(FI_LOG_EP_CTRL,
"Too long iface name: %s, max: %d\n",
iface, IFNAMSIZ);
return -FI_EINVAL;
}
for (ifa = ifaddr; ifa && !exact_match; ifa = ifa->ifa_next)
exact_match = !strcmp(ifa->ifa_name, iface);
}
for (ifa = ifaddr; ifa; ifa = ifa->ifa_next) {
if (!ifa->ifa_addr || !(ifa->ifa_flags & IFF_UP) ||
!strcmp(ifa->ifa_name, "lo"))
continue;
if(iface) {
if(exact_match) {
if(strcmp(ifa->ifa_name, iface))
continue;
} else {
if(strncmp(ifa->ifa_name, iface, iface_len))
continue;
}
}
switch (ifa->ifa_addr->sa_family) {
case AF_INET:
ret_ptr = inet_ntop(AF_INET, &ofi_sin_addr(ifa->ifa_addr),
name, INET6_ADDRSTRLEN);
break;
case AF_INET6:
ret_ptr = inet_ntop(AF_INET6, &ofi_sin6_addr(ifa->ifa_addr),
name, INET6_ADDRSTRLEN);
break;
default:
continue;
}
if (!ret_ptr) {
VERBS_WARN(FI_LOG_FABRIC,
"inet_ntop failed: %s(%d)\n",
strerror(errno), errno);
goto err1;
}
ret = fi_ibv_create_ep(name, NULL, FI_NUMERICHOST | FI_SOURCE,
NULL, &rai, &id);
if (ret)
continue;
ret = fi_ibv_add_rai(verbs_devs, id, rai);
if (ret)
goto err2;
VERBS_DBG(FI_LOG_FABRIC, "Found active interface for verbs device: "
"%s with address: %s\n",
ibv_get_device_name(id->verbs->device), name);
rdma_destroy_ep(id);
num_verbs_ifs++;
}
freeifaddrs(ifaddr);
return num_verbs_ifs ? 0 : -FI_ENODATA;
err2:
rdma_destroy_ep(id);
err1:
fi_ibv_verbs_devs_free(verbs_devs);
freeifaddrs(ifaddr);
return ret;
}
