static void fi_efa_fini(void)
{
struct efa_context **ctx_list;
int num_devices;
int i;
fi_freeinfo((void *)efa_util_prov.info);
efa_util_prov.info = NULL;
if (efa_mr_cache_enable)
efa_mem_notifier_finalize();
ctx_list = efa_device_get_context_list(&num_devices);
for (i = 0; i < num_devices; i++)
efa_dealloc_ctx(ctx_list[i]);
efa_device_free_context_list(ctx_list);
efa_device_free();
}
static int fi_ibv_domain_close(fid_t fid)
{
struct fi_ibv_domain *domain;
int ret;
domain = container_of(fid, struct fi_ibv_domain, domain_fid.fid);
if (domain->pd) {
ret = ibv_dealloc_pd(domain->pd);
if (ret)
return -ret;
domain->pd = NULL;
}
fi_freeinfo(domain->info);
free(domain);
return 0;
}
int fi_ibv_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct rdma_cm_id *id = NULL;
struct rdma_addrinfo *rai;
int ret;
ret = fi_ibv_init_info();
if (ret)
goto out;
ret = fi_ibv_create_ep(node, service, flags, hints, &rai, &id);
if (ret)
goto out;
if (id->verbs) {
ret = fi_ibv_get_matching_info(ibv_get_device_name(id->verbs->device),
hints, rai, info);
} else {
ret = fi_ibv_get_matching_info(NULL, hints, rai, info);
if (!ret && !(flags & FI_SOURCE) && !node
&& (!hints || (!hints->src_addr && !hints->dest_addr))) {
ret = fi_ibv_getifaddrs(service, flags, *info);
if (ret) {
fi_freeinfo(*info);
fi_ibv_destroy_ep(rai, &id);
goto out;
}
}
}
if (!ret) {
ret = fi_ibv_rdm_remove_nonaddr_info(info);
}
ofi_alter_info(*info, hints);
fi_ibv_destroy_ep(rai, &id);
out:
if (!ret || ret == -FI_ENOMEM || ret == -FI_ENODEV)
return ret;
else
return -FI_ENODATA;
}
void vc_lookup_setup(int av_type, int av_size)
{
int ret = 0;
struct fi_av_attr attr;
hints = fi_allocinfo();
hints->mode = mode_bits;
hints->domain_attr->mr_mode = GNIX_DEFAULT_MR_MODE;
hints->fabric_attr->prov_name = strdup("gni");
/* Create endpoint */
ret = fi_getinfo(fi_version(), NULL, 0, 0, hints, &fi);
cr_assert(!ret, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fab, NULL);
cr_assert(!ret, "fi_fabric");
ret = fi_domain(fab, fi, &dom, NULL);
cr_assert(!ret, "fi_domain");
attr.type = av_type;
attr.count = av_size;
ret = fi_av_open(dom, &attr, &av, NULL);
cr_assert(!ret, "fi_av_open");
ret = fi_endpoint(dom, fi, &ep, NULL);
cr_assert(!ret, "fi_endpoint");
gnix_ep = container_of(ep, struct gnix_fid_ep, ep_fid);
ret = fi_getname(&ep->fid, NULL, &ep_name_len);
ret = fi_getname(&ep->fid, &ep_name, &ep_name_len);
cr_assert(ret == FI_SUCCESS);
ret = fi_ep_bind(ep, &av->fid, 0);
cr_assert(!ret, "fi_ep_bind");
ret = fi_enable(ep);
cr_assert(!ret, "fi_ep_enable");
fi_freeinfo(hints);
}
static void setup(void)
{
int ret;
struct fi_info *hints;
hints = fi_allocinfo();
cr_assert(hints, "fi_allocinfo");
hints->fabric_attr->name = strdup("gni");
ret = fi_getinfo(FI_VERSION(1, 0), NULL, 0, 0, hints, &fi);
cr_assert(ret == FI_SUCCESS, "fi_getinfo");
ret = fi_fabric(fi->fabric_attr, &fabric, NULL);
cr_assert(ret == FI_SUCCESS, "fi_fabric");
fi_freeinfo(hints);
}
void vc_lookup_teardown(void)
{
int ret = 0;
ret = fi_close(&ep->fid);
cr_assert(!ret, "failure in closing ep[0].");
ret = fi_close(&av->fid);
cr_assert(!ret, "failure in closing dom[0] av.");
ret = fi_close(&dom->fid);
cr_assert(!ret, "failure in closing domain dom[0].");
ret = fi_close(&fab->fid);
cr_assert(!ret, "failure in closing fabric.");
fi_freeinfo(fi);
}
static int run(struct fi_info *hints, char *node, char *port)
{
struct fi_info *cur;
struct fi_info *info;
int ret;
ret = fi_getinfo(FI_VERSION(1, 0), node, port, 0, hints, &info);
if (ret) {
printf("fi_getinfo %s\n", strerror(-ret));
return ret;
}
for (cur = info; cur; cur = cur->next)
printf("%s\n", fi_tostr(cur, FI_TYPE_INFO));
fi_freeinfo(info);
return EXIT_SUCCESS;
}
static int run(struct fi_info *hints, char *node, char *port)
{
struct fi_info *info;
int ret;
ret = fi_getinfo(FI_VERSION(FI_MAJOR_VERSION, FI_MINOR_VERSION), node, port, 0, hints, &info);
if (ret) {
fprintf(stderr, "fi_getinfo: %d\n", ret);
return ret;
}
if (verbose)
ret = print_long_info(info);
else
ret = print_short_info(info);
fi_freeinfo(info);
return ret;
}
static int
rx_size_left_err(void)
{
int ret;
int testret;
struct fid_ep *ep;
struct fi_info *myfi;
testret = FAIL;
ep = NULL;
myfi = fi_dupinfo(fi);
/* datapath operation, not expected to be caught by libfabric */
#if 0
ret = fi_rx_size_left(NULL);
if (ret != -FI_EINVAL) {
goto fail;
}
#endif
ret = fi_endpoint(domain, myfi, &ep, NULL);
if (ret != 0) {
printf("fi_endpoint %s\n", fi_strerror(-ret));
goto fail;
}
/* ep starts in a non-enabled state, may fail, should not SEGV */
fi_rx_size_left(ep);
testret = PASS;
fail:
if (ep != NULL) {
ret = fi_close(&ep->fid);
if (ret != 0)
printf("fi_close %s\n", fi_strerror(-ret));
ep = NULL;
}
if (myfi != NULL) {
fi_freeinfo(myfi);
}
return testret;
}
/*
* rpmem_fip_init -- initialize fabric provider
*/
struct rpmem_fip *
rpmem_fip_init(const char *node, const char *service,
struct rpmem_fip_attr *attr, unsigned *nlanes)
{
int ret;
struct rpmem_fip *fip = calloc(1, sizeof(*fip));
if (!fip) {
RPMEM_LOG(ERR, "!allocating fabric handle");
return NULL;
}
ret = rpmem_fip_getinfo(fip, node, service, attr->provider);
if (ret)
goto err_getinfo;
rpmem_fip_set_attr(fip, attr);
*nlanes = fip->nlanes;
ret = rpmem_fip_init_fabric_res(fip);
if (ret)
goto err_init_fabric_res;
ret = rpmem_fip_init_memory(fip);
if (ret)
goto err_init_memory;
ret = rpmem_fip_init_lanes(fip);
if (ret)
goto err_init_lanes;
return fip;
err_init_lanes:
rpmem_fip_fini_memory(fip);
err_init_memory:
rpmem_fip_fini_fabric_res(fip);
err_init_fabric_res:
fi_freeinfo(fip->fi);
err_getinfo:
free(fip);
return NULL;
}
int fi_getinfo_1_1(uint32_t version, const char *node, const char *service,
uint64_t flags, const struct fi_info_1_1 *hints_1_1,
struct fi_info_1_1 **info)
{
struct fi_info *hints;
int ret;
if (hints_1_1) {
hints = (struct fi_info *) fi_dupinfo_1_1(hints_1_1);
if (!hints)
return -FI_ENOMEM;
} else {
hints = NULL;
}
ret = fi_getinfo(version, node, service, flags, hints,
(struct fi_info **) info);
fi_freeinfo(hints);
return ret;
}
static int fi_ib_freeinfo(struct fi_info *info)
{
print_trace("in\n");
if (!info)
return -EINVAL;
return -ENOSYS;
#if 0
/* prevent recursion via def of fi_freeinfo() */
if (info->fabric_attr) {
kfree(info->fabric_attr->name);
kfree(info->fabric_attr->prov_name);
kfree(info->fabric_attr);
info->fabric_attr = NULL;
}
fi_freeinfo(info);
return 0;
#endif
}
void cm_stop_client(void)
{
int ret;
ret = fi_close(&cli_cq->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&cli_ep->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&cli_dom->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&cli_eq->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&cli_fab->fid);
cr_assert_eq(ret, FI_SUCCESS);
fi_freeinfo(cli_fi);
}
static int rxm_getinfo(uint32_t version, const char *node, const char *service,
uint64_t flags, struct fi_info *hints, struct fi_info **info)
{
struct fi_info *cur, *dup;
int ret;
ret = ofix_getinfo(version, node, service, flags, &rxm_util_prov, hints,
rxm_info_to_core, rxm_info_to_rxm, info);
if (ret)
return ret;
/* If app supports FI_MR_LOCAL, prioritize requiring it for
* better performance. */
if (hints && hints->domain_attr &&
(RXM_MR_LOCAL(hints))) {
for (cur = *info; cur; cur = cur->next) {
if (!RXM_MR_LOCAL(cur))
continue;
if (!(dup = fi_dupinfo(cur))) {
fi_freeinfo(*info);
return -FI_ENOMEM;
}
if (FI_VERSION_LT(version, FI_VERSION(1, 5)))
dup->mode &= ~FI_LOCAL_MR;
else
dup->domain_attr->mr_mode &= ~FI_MR_LOCAL;
dup->next = cur->next;
cur->next = dup;
cur = dup;
}
} else {
for (cur = *info; cur; cur = cur->next) {
if (FI_VERSION_LT(version, FI_VERSION(1, 5)))
cur->mode &= ~FI_LOCAL_MR;
else
cur->domain_attr->mr_mode &= ~FI_MR_LOCAL;
}
}
return 0;
}
/*
* Make a dummy info object for each provider, and copy in the
* provider name and version. We report utility providers directly
* to export their version.
*/
static int ofi_getprovinfo(struct fi_info **info)
{
struct ofi_prov *prov;
struct fi_info *tail, *cur;
int ret = -FI_ENODATA;
*info = tail = NULL;
for (prov = prov_head; prov; prov = prov->next) {
if (!prov->provider)
continue;
cur = fi_allocinfo();
if (!cur) {
ret = -FI_ENOMEM;
goto err;
}
cur->fabric_attr->prov_name = strdup(prov->provider->name);
cur->fabric_attr->prov_version = prov->provider->version;
if (!*info) {
*info = tail = cur;
} else {
tail->next = cur;
}
tail = cur;
ret = 0;
}
return ret;
err:
while (tail) {
cur = tail->next;
fi_freeinfo(tail);
tail = cur;
}
return ret;
}
static struct fi_info *
__fi_eq_cm_getinfo(struct __fid_fabric *fab, struct rdma_cm_event *event)
{
struct fi_info *fi;
fi = calloc(1, sizeof *fi);
if (!fi)
return NULL;
fi->type = FID_MSG;
if (event->id->verbs->device->transport_type == IBV_TRANSPORT_IWARP) {
fi->protocol = FI_PROTO_IWARP;
} else {
fi->protocol = FI_PROTO_IB_RC;
}
fi->ep_cap = FI_MSG | FI_RMA;
fi->src_addrlen = fi_sockaddr_len(rdma_get_local_addr(event->id));
if (!(fi->src_addr = malloc(fi->src_addrlen)))
goto err;
memcpy(fi->src_addr, rdma_get_local_addr(event->id), fi->src_addrlen);
fi->dest_addrlen = fi_sockaddr_len(rdma_get_peer_addr(event->id));
if (!(fi->dest_addr = malloc(fi->dest_addrlen)))
goto err;
memcpy(fi->dest_addr, rdma_get_peer_addr(event->id), fi->dest_addrlen);
if (!(fi->fabric_name = strdup(fab->name)))
goto err;
if (!(fi->domain_name = strdup(event->id->verbs->device->name)))
goto err;
fi->datalen = sizeof event->id;
fi->data = event->id;
return fi;
err:
fi_freeinfo(fi);
return NULL;
}
struct fi_info *ofi_allocinfo_internal(void)
{
struct fi_info *info;
info = calloc(1, sizeof(*info));
if (!info)
return NULL;
info->tx_attr = calloc(1, sizeof(*info->tx_attr));
info->rx_attr = calloc(1, sizeof(*info->rx_attr));
info->ep_attr = calloc(1, sizeof(*info->ep_attr));
info->domain_attr = calloc(1, sizeof(*info->domain_attr));
info->fabric_attr = calloc(1, sizeof(*info->fabric_attr));
if (!info->tx_attr|| !info->rx_attr || !info->ep_attr ||
!info->domain_attr || !info->fabric_attr)
goto err;
return info;
err:
fi_freeinfo(info);
return NULL;
}
/* if there are multiple fi_info in the provider:
* check and duplicate provider's info */
int ofi_prov_check_dup_info(const struct util_prov *util_prov,
uint32_t api_version,
const struct fi_info *user_info,
struct fi_info **info)
{
const struct fi_info *prov_info = util_prov->info;
const struct fi_provider *prov = util_prov->prov;
struct fi_info *fi, *tail;
int ret;
if (!info)
return -FI_EINVAL;
*info = tail = NULL;
for ( ; prov_info; prov_info = prov_info->next) {
ret = ofi_check_info(util_prov, prov_info,
api_version, user_info);
if (ret)
continue;
if (!(fi = fi_dupinfo(prov_info))) {
ret = -FI_ENOMEM;
goto err;
}
if (!*info)
*info = fi;
else
tail->next = fi;
tail = fi;
}
return (!*info ? -FI_ENODATA : FI_SUCCESS);
err:
fi_freeinfo(*info);
FI_INFO(prov, FI_LOG_CORE,
"cannot copy info\n");
return ret;
}
void cm_stop_server(void)
{
int ret;
ret = fi_close(&srv_cq->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&srv_ep->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&srv_dom->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&srv_pep->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&srv_eq->fid);
cr_assert_eq(ret, FI_SUCCESS);
ret = fi_close(&srv_fab->fid);
cr_assert_eq(ret, FI_SUCCESS);
fi_freeinfo(srv_fi);
}
static int rxm_ep_close(struct fid *fid)
{
struct rxm_ep *rxm_ep;
int ret, retv = 0;
rxm_ep = container_of(fid, struct rxm_ep, util_ep.ep_fid.fid);
if (rxm_ep->util_ep.av) {
ofi_cmap_free(rxm_ep->cmap);
atomic_dec(&rxm_ep->util_ep.av->ref);
}
ret = fi_close(&rxm_ep->srx_ctx->fid);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg shared ctx\n");
retv = ret;
}
ret = fi_close(&rxm_ep->msg_pep->fid);
if (ret) {
FI_WARN(&rxm_prov, FI_LOG_EP_CTRL, "Unable to close msg passive EP\n");
retv = ret;
}
fi_freeinfo(rxm_ep->msg_info);
if (rxm_ep->util_ep.rx_cq)
atomic_dec(&rxm_ep->util_ep.rx_cq->ref);
if (rxm_ep->util_ep.tx_cq)
atomic_dec(&rxm_ep->util_ep.tx_cq->ref);
atomic_dec(&rxm_ep->util_ep.domain->ref);
free(rxm_ep);
return retv;
}
struct fi_info *DEFAULT_SYMVER_PRE(fi_dupinfo)(const struct fi_info *info)
{
struct fi_info *dup;
if (!info)
return ofi_allocinfo_internal();
dup = mem_dup(info, sizeof(*dup));
if (dup == NULL) {
return NULL;
}
dup->src_addr = NULL;
dup->dest_addr = NULL;
dup->tx_attr = NULL;
dup->rx_attr = NULL;
dup->ep_attr = NULL;
dup->domain_attr = NULL;
dup->fabric_attr = NULL;
dup->next = NULL;
if (info->src_addr != NULL) {
dup->src_addr = mem_dup(info->src_addr, info->src_addrlen);
if (dup->src_addr == NULL)
goto fail;
}
if (info->dest_addr != NULL) {
dup->dest_addr = mem_dup(info->dest_addr, info->dest_addrlen);
if (dup->dest_addr == NULL)
goto fail;
}
if (info->tx_attr != NULL) {
dup->tx_attr = mem_dup(info->tx_attr, sizeof(*info->tx_attr));
if (dup->tx_attr == NULL)
goto fail;
}
if (info->rx_attr != NULL) {
dup->rx_attr = mem_dup(info->rx_attr, sizeof(*info->rx_attr));
if (dup->rx_attr == NULL)
goto fail;
}
if (info->ep_attr != NULL) {
dup->ep_attr = mem_dup(info->ep_attr, sizeof(*info->ep_attr));
if (dup->ep_attr == NULL)
goto fail;
}
if (info->domain_attr) {
dup->domain_attr = mem_dup(info->domain_attr,
sizeof(*info->domain_attr));
if (dup->domain_attr == NULL)
goto fail;
if (info->domain_attr->name != NULL) {
dup->domain_attr->name = strdup(info->domain_attr->name);
if (dup->domain_attr->name == NULL)
goto fail;
}
}
if (info->fabric_attr) {
dup->fabric_attr = mem_dup(info->fabric_attr,
sizeof(*info->fabric_attr));
if (dup->fabric_attr == NULL)
goto fail;
dup->fabric_attr->name = NULL;
dup->fabric_attr->prov_name = NULL;
if (info->fabric_attr->name != NULL) {
dup->fabric_attr->name = strdup(info->fabric_attr->name);
if (dup->fabric_attr->name == NULL)
goto fail;
}
if (info->fabric_attr->prov_name != NULL) {
dup->fabric_attr->prov_name = strdup(info->fabric_attr->prov_name);
if (dup->fabric_attr->prov_name == NULL)
goto fail;
}
}
return dup;
fail:
fi_freeinfo(dup);
return NULL;
}
static void fas_teardown_common(void)
{
int ret = 0, i = 0, j;
for (; i < NUMEPS; i++) {
if (ep_type == SEP || ep_type == EP) {
ret = fi_close(&recv_cntr[i]->fid);
cr_assert(!ret, "failure in closing recv cntr.");
ret = fi_close(&send_cntr[i]->fid);
cr_assert(!ret, "failure in closing send cntr.");
}
switch (ep_type) {
case EP:
ret = fi_close(&msg_cq[i]->fid);
cr_assert(!ret, "failure in closing msg cq.");
break;
case SEP:
for (j = 0; j < ctx_cnt; j++) {
ret = fi_close(&tx_ep[i][j]->fid);
cr_assert(!ret,
"failure closing tx_ep.");
ret = fi_close(&rx_ep[i][j]->fid);
cr_assert(!ret,
"failure closing rx_ep.");
ret = fi_close(&tx_cq[i][j]->fid);
cr_assert(!ret,
"failure closing tx cq.");
ret = fi_close(&rx_cq[i][j]->fid);
cr_assert(!ret,
"failure closing rx cq.");
}
break;
case PEP:
ret = fi_close(get_fid[ep_type](i));
cr_assert(!ret, "failure in closing ep.");
continue;
break;
default:
cr_assert_fail("Unknown endpoint type.");
break;
}
ret = fi_close(get_fid[ep_type](i));
cr_assert(!ret, "failure in closing ep.");
ret = fi_close(&av[i]->fid);
cr_assert(!ret, "failure in closing av.");
ret = fi_close(&dom[i]->fid);
cr_assert(!ret, "failure in closing domain.");
free(ep_name[i]);
free(target[i]);
free(source[i]);
free(tx_cq[i]);
free(tx_ep[i]);
free(rx_cq[i]);
free(rx_ep[i]);
fi_freeinfo(fi[i]);
}
ret = fi_close(&fab->fid);
cr_assert(!ret, "failure in closing fabric.");
fi_freeinfo(hints);
}
static void fas_getinfo_teardown(void)
{
fi_freeinfo(hints);
}
static int server_connect(void)
{
struct fi_eq_cm_entry entry;
uint32_t event;
struct fi_info *info = NULL;
ssize_t rd;
int ret;
/* Wait for connection request from client */
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
return (int) rd;
}
info = entry.info;
if (event != FI_CONNREQ) {
FT_ERR("Unexpected CM event %d\n", event);
ret = -FI_EOTHER;
goto err;
}
ret = fi_domain(fabric, info, &domain, NULL);
if (ret) {
FT_PRINTERR("fi_domain", ret);
goto err;
}
ret = ft_alloc_active_res(info);
if (ret)
goto err;
ret = ft_init_ep();
if (ret)
goto err;
/* Accept the incoming connection. Also transitions endpoint to active state */
ret = fi_accept(ep, NULL, 0);
if (ret) {
FT_PRINTERR("fi_accept", ret);
goto err;
}
/* Wait for the connection to be established */
rd = fi_eq_sread(eq, &event, &entry, sizeof entry, -1, 0);
if (rd != sizeof entry) {
FT_PRINTERR("fi_eq_sread", rd);
goto err;
}
if (event != FI_CONNECTED || entry.fid != &ep->fid) {
FT_ERR("Unexpected CM event %d fid %p (ep %p)\n", event, entry.fid, ep);
ret = -FI_EOTHER;
goto err;
}
ret = check_address(&ep->fid, "accept");
if (ret) {
goto err;
}
fi_freeinfo(info);
return 0;
err:
fi_reject(pep, info->handle, NULL, 0);
fi_freeinfo(info);
return ret;
}
/*
* rpmemd_fip_init -- initialize fabric provider
*/
struct rpmemd_fip *
rpmemd_fip_init(const char *node, const char *service,
struct rpmemd_fip_attr *attr, struct rpmem_resp_attr *resp,
enum rpmem_err *err)
{
int ret;
RPMEMD_ASSERT(resp);
RPMEMD_ASSERT(err);
RPMEMD_ASSERT(attr);
RPMEMD_ASSERT(attr->persist);
RPMEMD_ASSERT(attr->nthreads);
struct rpmemd_fip *fip = calloc(1, sizeof(*fip));
if (!fip) {
RPMEMD_LOG(ERR, "!allocating fabric handle");
*err = RPMEM_ERR_FATAL;
return NULL;
}
ret = rpmemd_fip_getinfo(fip, service, node, attr->provider);
if (ret) {
*err = RPMEM_ERR_BADPROVIDER;
goto err_getinfo;
}
rpmemd_fip_set_attr(fip, attr);
ret = rpmemd_fip_init_fabric_res(fip);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_init_fabric_res;
}
ret = rpmemd_fip_init_memory(fip);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_init_memory;
}
ret = fip->ops->init(fip);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_init;
}
ret = fi_listen(fip->pep);
if (ret) {
*err = RPMEM_ERR_FATAL_CONN;
goto err_fi_listen;
}
ret = rpmemd_fip_set_resp(fip, resp);
if (ret) {
*err = RPMEM_ERR_FATAL;
goto err_set_resp;
}
return fip;
err_set_resp:
RPMEMD_FI_CLOSE(fip->pep, "closing passive endpoint");
err_fi_listen:
fip->ops->fini(fip);
err_init:
rpmemd_fip_fini_memory(fip);
err_init_memory:
rpmemd_fip_fini_fabric_res(fip);
err_init_fabric_res:
fi_freeinfo(fip->fi);
err_getinfo:
free(fip);
return NULL;
}
static int efa_alloc_info(struct efa_context *ctx, struct fi_info **info,
const struct efa_ep_domain *ep_dom)
{
struct fi_info *fi;
union ibv_gid gid;
size_t name_len;
int ret;
fi = fi_allocinfo();
if (!fi)
return -FI_ENOMEM;
fi->caps = ep_dom->caps;
fi->handle = NULL;
*fi->ep_attr = efa_ep_attr;
if (ep_dom->type == FI_EP_RDM) {
*fi->tx_attr = efa_rdm_tx_attr;
*fi->rx_attr = efa_rdm_rx_attr;
} else if (ep_dom->type == FI_EP_DGRAM) {
fi->mode |= FI_MSG_PREFIX;
fi->ep_attr->msg_prefix_size = 40;
*fi->tx_attr = efa_dgrm_tx_attr;
*fi->rx_attr = efa_dgrm_rx_attr;
}
*fi->domain_attr = efa_domain_attr;
*fi->fabric_attr = efa_fabric_attr;
fi->ep_attr->protocol = FI_PROTO_EFA;
fi->ep_attr->type = ep_dom->type;
fi->tx_attr->caps = ep_dom->caps;
fi->rx_attr->caps = ep_dom->caps;
ret = efa_get_device_attrs(ctx, fi);
if (ret)
goto err_free_info;
ret = efa_cmd_query_gid(ctx, 1, 0, &gid);
if (ret) {
EFA_INFO_ERRNO(FI_LOG_FABRIC, "efa_cmd_query_gid", errno);
ret = -errno;
goto err_free_info;
}
name_len = strlen(EFA_FABRIC_PREFIX) + INET6_ADDRSTRLEN;
fi->fabric_attr->name = calloc(1, name_len + 1);
if (!fi->fabric_attr->name) {
ret = -FI_ENOMEM;
goto err_free_info;
}
efa_addr_to_str(gid.raw, fi->fabric_attr->name);
name_len = strlen(ctx->ibv_ctx.device->name) + strlen(ep_dom->suffix);
fi->domain_attr->name = malloc(name_len + 1);
if (!fi->domain_attr->name) {
ret = -FI_ENOMEM;
goto err_free_fab_name;
}
snprintf(fi->domain_attr->name, name_len + 1, "%s%s",
ctx->ibv_ctx.device->name, ep_dom->suffix);
fi->domain_attr->name[name_len] = '\0';
fi->addr_format = FI_ADDR_EFA;
fi->src_addr = calloc(1, EFA_EP_ADDR_LEN);
if (!fi->src_addr) {
ret = -FI_ENOMEM;
goto err_free_dom_name;
}
fi->src_addrlen = EFA_EP_ADDR_LEN;
ret = efa_get_addr(ctx, fi->src_addr);
if (ret)
goto err_free_src;
fi->domain_attr->av_type = FI_AV_TABLE;
*info = fi;
return 0;
err_free_src:
free(fi->src_addr);
err_free_dom_name:
free(fi->domain_attr->name);
err_free_fab_name:
free(fi->fabric_attr->name);
err_free_info:
fi_freeinfo(fi);
return ret;
}
static int fi_ibv_alloc_info(struct ibv_context *ctx, struct fi_info **info,
const struct verbs_ep_domain *ep_dom)
{
struct fi_info *fi;
union ibv_gid gid;
size_t name_len;
int ret;
int param;
if (!(fi = fi_allocinfo()))
return -FI_ENOMEM;
fi->caps = ep_dom->caps;
fi->handle = NULL;
if (ep_dom->type == FI_EP_RDM) {
fi->mode = VERBS_RDM_MODE;
*(fi->tx_attr) = verbs_rdm_tx_attr;
} else {
fi->mode = VERBS_MODE;
*(fi->tx_attr) = verbs_tx_attr;
}
*(fi->rx_attr) = (ep_dom->type == FI_EP_RDM)
? verbs_rdm_rx_attr : verbs_rx_attr;
*(fi->ep_attr) = verbs_ep_attr;
*(fi->domain_attr) = verbs_domain_attr;
*(fi->fabric_attr) = verbs_fabric_attr;
fi->ep_attr->type = ep_dom->type;
fi->tx_attr->caps = ep_dom->caps;
fi->rx_attr->caps = ep_dom->caps;
ret = fi_ibv_get_device_attrs(ctx, fi);
if (ret)
goto err;
if (ep_dom->type == FI_EP_RDM) {
fi->tx_attr->inject_size = FI_IBV_RDM_DFLT_BUFFERED_SSIZE;
fi->tx_attr->iov_limit = 1;
fi->tx_attr->rma_iov_limit = 1;
if (!fi_param_get_int(&fi_ibv_prov, "rdm_buffer_size", ¶m)) {
if (param > sizeof (struct fi_ibv_rdm_tagged_rndv_header)) {
fi->tx_attr->inject_size = param;
} else {
FI_INFO(&fi_ibv_prov, FI_LOG_CORE,
"rdm_buffer_size too small, should be greater then %d\n",
sizeof (struct fi_ibv_rdm_tagged_rndv_header));
ret = -FI_EINVAL;
goto err;
}
}
}
switch (ctx->device->transport_type) {
case IBV_TRANSPORT_IB:
if(ibv_query_gid(ctx, 1, 0, &gid)) {
VERBS_INFO_ERRNO(FI_LOG_FABRIC, "ibv_query_gid", errno);
ret = -errno;
goto err;
}
name_len = strlen(VERBS_IB_PREFIX) + INET6_ADDRSTRLEN;
if (!(fi->fabric_attr->name = calloc(1, name_len + 1))) {
ret = -FI_ENOMEM;
goto err;
}
snprintf(fi->fabric_attr->name, name_len, VERBS_IB_PREFIX "%lx",
gid.global.subnet_prefix);
fi->ep_attr->protocol = (ep_dom == &verbs_msg_domain) ?
FI_PROTO_RDMA_CM_IB_RC : FI_PROTO_IB_RDM;
break;
case IBV_TRANSPORT_IWARP:
fi->fabric_attr->name = strdup(VERBS_IWARP_FABRIC);
if (!fi->fabric_attr->name) {
ret = -FI_ENOMEM;
goto err;
}
if (ep_dom == &verbs_msg_domain) {
fi->ep_attr->protocol = FI_PROTO_IWARP;
fi->tx_attr->op_flags = VERBS_TX_OP_FLAGS_IWARP;
} else {
fi->ep_attr->protocol = FI_PROTO_IWARP_RDM;
fi->tx_attr->op_flags = VERBS_TX_OP_FLAGS_IWARP_RDM;
}
break;
default:
FI_INFO(&fi_ibv_prov, FI_LOG_CORE, "Unknown transport type\n");
ret = -FI_ENODATA;
goto err;
}
name_len = strlen(ctx->device->name) + strlen(ep_dom->suffix);
fi->domain_attr->name = malloc(name_len + 1);
if (!fi->domain_attr->name) {
ret = -FI_ENOMEM;
goto err;
}
snprintf(fi->domain_attr->name, name_len + 1, "%s%s",
ctx->device->name, ep_dom->suffix);
fi->domain_attr->name[name_len] = '\0';
*info = fi;
return 0;
err:
fi_freeinfo(fi);
return ret;
}
void fi_ibv_free_info(void)
{
fi_freeinfo(verbs_info);
}
int
usdf_pep_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_pep **pep_o, void *context)
{
struct usdf_pep *pep;
struct usdf_fabric *fp;
struct sockaddr_in *sin;
int ret;
int optval;
USDF_TRACE_SYS(EP_CTRL, "\n");
if (!info) {
USDF_DBG_SYS(EP_CTRL, "null fi_info struct is invalid\n");
return -FI_EINVAL;
}
if (info->ep_attr->type != FI_EP_MSG) {
return -FI_ENODEV;
}
if ((info->caps & ~USDF_MSG_CAPS) != 0) {
return -FI_EBADF;
}
switch (info->addr_format) {
case FI_SOCKADDR:
if (((struct sockaddr *)info->src_addr)->sa_family != AF_INET) {
USDF_WARN_SYS(EP_CTRL, "non-AF_INET src_addr specified\n");
return -FI_EINVAL;
}
break;
case FI_SOCKADDR_IN:
break;
default:
USDF_WARN_SYS(EP_CTRL, "unknown/unsupported addr_format\n");
return -FI_EINVAL;
}
if (info->src_addrlen &&
info->src_addrlen != sizeof(struct sockaddr_in)) {
USDF_WARN_SYS(EP_CTRL, "unexpected src_addrlen\n");
return -FI_EINVAL;
}
fp = fab_ftou(fabric);
pep = calloc(1, sizeof(*pep));
if (pep == NULL) {
return -FI_ENOMEM;
}
pep->pep_fid.fid.fclass = FI_CLASS_PEP;
pep->pep_fid.fid.context = context;
pep->pep_fid.fid.ops = &usdf_pep_ops;
pep->pep_fid.ops = &usdf_pep_base_ops;
pep->pep_fid.cm = &usdf_pep_cm_ops;
pep->pep_fabric = fp;
pep->pep_state = USDF_PEP_UNBOUND;
pep->pep_sock = socket(AF_INET, SOCK_STREAM, 0);
if (pep->pep_sock == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_GETFL, 0);
if (ret == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_SETFL, ret | O_NONBLOCK);
if (ret == -1) {
ret = -errno;
goto fail;
}
/* set SO_REUSEADDR to prevent annoying "Address already in use" errors
* on successive runs of programs listening on a well known port */
optval = 1;
ret = setsockopt(pep->pep_sock, SOL_SOCKET, SO_REUSEADDR, &optval,
sizeof(optval));
if (ret == -1) {
ret = -errno;
goto fail;
}
pep->pep_info = fi_dupinfo(info);
if (!pep->pep_info) {
ret = -FI_ENOMEM;
goto fail;
}
if (info->src_addrlen == 0) {
/* Copy the source address information from the device
* attributes.
*/
pep->pep_info->src_addrlen = sizeof(struct sockaddr_in);
sin = calloc(1, pep->pep_info->src_addrlen);
if (!sin) {
USDF_WARN_SYS(EP_CTRL,
"calloc for src address failed\n");
goto fail;
}
sin->sin_family = AF_INET;
sin->sin_addr.s_addr = fp->fab_dev_attrs->uda_ipaddr_be;
pep->pep_info->src_addr = sin;
}
memcpy(&pep->pep_src_addr, pep->pep_info->src_addr,
pep->pep_info->src_addrlen);
/* initialize connreq freelist */
ret = pthread_spin_init(&pep->pep_cr_lock, PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&pep->pep_cr_free);
TAILQ_INIT(&pep->pep_cr_pending);
pep->pep_backlog = 10;
ret = usdf_pep_grow_backlog(pep);
if (ret != 0) {
goto fail;
}
atomic_initialize(&pep->pep_refcnt, 0);
atomic_inc(&fp->fab_refcnt);
*pep_o = pep_utof(pep);
return 0;
fail:
if (pep != NULL) {
usdf_pep_free_cr_lists(pep);
if (pep->pep_sock != -1) {
close(pep->pep_sock);
}
fi_freeinfo(pep->pep_info);
free(pep);
}
return ret;
}
static mca_mtl_base_module_t*
ompi_mtl_ofi_component_init(bool enable_progress_threads,
bool enable_mpi_threads)
{
int ret, fi_version;
struct fi_info *hints;
struct fi_info *providers = NULL, *prov = NULL;
struct fi_cq_attr cq_attr = {0};
struct fi_av_attr av_attr = {0};
char ep_name[FI_NAME_MAX] = {0};
size_t namelen;
/**
* Hints to filter providers
* See man fi_getinfo for a list of all filters
* mode: Select capabilities MTL is prepared to support.
* In this case, MTL will pass in context into communication calls
* ep_type: reliable datagram operation
* caps: Capabilities required from the provider.
* Tag matching is specified to implement MPI semantics.
* msg_order: Guarantee that messages with same tag are ordered.
*/
hints = fi_allocinfo();
if (!hints) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: Could not allocate fi_info\n",
__FILE__, __LINE__);
goto error;
}
hints->mode = FI_CONTEXT;
hints->ep_attr->type = FI_EP_RDM; /* Reliable datagram */
hints->caps = FI_TAGGED; /* Tag matching interface */
hints->tx_attr->msg_order = FI_ORDER_SAS;
hints->rx_attr->msg_order = FI_ORDER_SAS;
hints->domain_attr->threading = FI_THREAD_UNSPEC;
if (MTL_OFI_PROG_AUTO == control_progress) {
hints->domain_attr->control_progress = FI_PROGRESS_AUTO;
} else {
hints->domain_attr->control_progress = FI_PROGRESS_MANUAL;
}
if (MTL_OFI_PROG_MANUAL == data_progress) {
hints->domain_attr->data_progress = FI_PROGRESS_MANUAL;
} else {
hints->domain_attr->data_progress = FI_PROGRESS_AUTO;
}
if (MTL_OFI_AV_TABLE == av_type) {
hints->domain_attr->av_type = FI_AV_TABLE;
} else {
hints->domain_attr->av_type = FI_AV_MAP;
}
hints->domain_attr->resource_mgmt = FI_RM_ENABLED;
/**
* FI_VERSION provides binary backward and forward compatibility support
* Specify the version of OFI is coded to, the provider will select struct
* layouts that are compatible with this version.
*/
fi_version = FI_VERSION(1, 0);
/**
* fi_getinfo: returns information about fabric services for reaching a
* remote node or service. this does not necessarily allocate resources.
* Pass NULL for name/service because we want a list of providers supported.
*/
ret = fi_getinfo(fi_version, /* OFI version requested */
NULL, /* Optional name or fabric to resolve */
NULL, /* Optional service name or port to request */
0ULL, /* Optional flag */
hints, /* In: Hints to filter providers */
&providers); /* Out: List of matching providers */
if (0 != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_getinfo failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* Select a provider from the list returned by fi_getinfo().
*/
prov = select_ofi_provider(providers);
if (!prov) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: select_ofi_provider: no provider found\n",
__FILE__, __LINE__);
goto error;
}
/**
* Open fabric
* The getinfo struct returns a fabric attribute struct that can be used to
* instantiate the virtual or physical network. This opens a "fabric
* provider". See man fi_fabric for details.
*/
ret = fi_fabric(prov->fabric_attr, /* In: Fabric attributes */
&ompi_mtl_ofi.fabric, /* Out: Fabric handle */
NULL); /* Optional context for fabric events */
if (0 != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_fabric failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* Create the access domain, which is the physical or virtual network or
* hardware port/collection of ports. Returns a domain object that can be
* used to create endpoints. See man fi_domain for details.
*/
ret = fi_domain(ompi_mtl_ofi.fabric, /* In: Fabric object */
prov, /* In: Provider */
&ompi_mtl_ofi.domain, /* Out: Domain oject */
NULL); /* Optional context for domain events */
if (0 != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_domain failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* Create a transport level communication endpoint. To use the endpoint,
* it must be bound to completion counters or event queues and enabled,
* and the resources consumed by it, such as address vectors, counters,
* completion queues, etc.
* see man fi_endpoint for more details.
*/
ret = fi_endpoint(ompi_mtl_ofi.domain, /* In: Domain object */
prov, /* In: Provider */
&ompi_mtl_ofi.ep, /* Out: Endpoint object */
NULL); /* Optional context */
if (0 != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_endpoint failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* Save the maximum inject size.
*/
ompi_mtl_ofi.max_inject_size = prov->tx_attr->inject_size;
/**
* Create the objects that will be bound to the endpoint.
* The objects include:
* - completion queue for events
* - address vector of other endpoint addresses
* - dynamic memory-spanning memory region
*/
cq_attr.format = FI_CQ_FORMAT_TAGGED;
ret = fi_cq_open(ompi_mtl_ofi.domain, &cq_attr, &ompi_mtl_ofi.cq, NULL);
if (ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_cq_open failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* The remote fi_addr will be stored in the ofi_endpoint struct.
*/
av_attr.type = (MTL_OFI_AV_TABLE == av_type) ? FI_AV_TABLE: FI_AV_MAP;
ret = fi_av_open(ompi_mtl_ofi.domain, &av_attr, &ompi_mtl_ofi.av, NULL);
if (ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_av_open failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* Bind the CQ and AV to the endpoint object.
*/
ret = fi_ep_bind(ompi_mtl_ofi.ep,
(fid_t)ompi_mtl_ofi.cq,
FI_SEND | FI_RECV);
if (0 != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_bind CQ-EP failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
ret = fi_ep_bind(ompi_mtl_ofi.ep,
(fid_t)ompi_mtl_ofi.av,
0);
if (0 != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_bind AV-EP failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* Enable the endpoint for communication
* This commits the bind operations.
*/
ret = fi_enable(ompi_mtl_ofi.ep);
if (0 != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_enable failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
/**
* Free providers info since it's not needed anymore.
*/
fi_freeinfo(hints);
hints = NULL;
fi_freeinfo(providers);
providers = NULL;
/**
* Get our address and publish it with modex.
*/
namelen = sizeof(ep_name);
ret = fi_getname((fid_t)ompi_mtl_ofi.ep, &ep_name[0], &namelen);
if (ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: fi_getname failed: %s\n",
__FILE__, __LINE__, fi_strerror(-ret));
goto error;
}
OFI_COMPAT_MODEX_SEND(ret,
&mca_mtl_ofi_component.super.mtl_version,
&ep_name,
namelen);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: modex_send failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
ompi_mtl_ofi.epnamelen = namelen;
/**
* Set the ANY_SRC address.
*/
ompi_mtl_ofi.any_addr = FI_ADDR_UNSPEC;
/**
* Activate progress callback.
*/
ret = opal_progress_register(ompi_mtl_ofi_progress_no_inline);
if (OMPI_SUCCESS != ret) {
opal_output_verbose(1, ompi_mtl_base_framework.framework_output,
"%s:%d: opal_progress_register failed: %d\n",
__FILE__, __LINE__, ret);
goto error;
}
return &ompi_mtl_ofi.base;
error:
if (providers) {
(void) fi_freeinfo(providers);
}
if (hints) {
(void) fi_freeinfo(hints);
}
if (ompi_mtl_ofi.av) {
(void) fi_close((fid_t)ompi_mtl_ofi.av);
}
if (ompi_mtl_ofi.cq) {
(void) fi_close((fid_t)ompi_mtl_ofi.cq);
}
if (ompi_mtl_ofi.ep) {
(void) fi_close((fid_t)ompi_mtl_ofi.ep);
}
if (ompi_mtl_ofi.domain) {
(void) fi_close((fid_t)ompi_mtl_ofi.domain);
}
if (ompi_mtl_ofi.fabric) {
(void) fi_close((fid_t)ompi_mtl_ofi.fabric);
}
return NULL;
}
