static void continuous_insert_remove(int num_inserters, int num_removers,
int num_insertions)
{
int i, ret;
unsigned long pret;
atomic_t done;
fi_addr_t addresses[MT_ADDR_COUNT];
atomic_t fe[MT_ADDR_COUNT];
const int addrs_per_thread = MT_ADDR_COUNT/num_inserters;
const int num_threads = num_inserters + num_removers;
pthread_t threads[num_threads];
void *info[num_threads][6];
ret = pthread_barrier_init(&mtbar, NULL, num_threads);
cr_assert_eq(ret, 0);
init_av_lock();
atomic_initialize(&done, 0);
for (i = 0; i < MT_ADDR_COUNT; i++) {
atomic_initialize(&fe[i], state_empty);
}
for (i = 0; i < num_inserters; i++) {
info[i][0] = (void *) &simple_ep_names[i*addrs_per_thread];
info[i][1] = (void *) &addresses[i*addrs_per_thread];
info[i][2] = (void *) &fe[i*addrs_per_thread];
info[i][3] = (void *) (uint64_t) num_insertions;
info[i][4] = (void *) (uint64_t) addrs_per_thread;
info[i][5] = (void *) &done;
ret = pthread_create(&threads[i], NULL,
continuous_insert, &info[i]);
cr_assert_eq(ret, 0);
}
for (i = num_inserters; i < num_threads; i++) {
info[i][0] = (void *) addresses;
info[i][1] = (void *) fe;
info[i][2] = (void *) (uint64_t)
(num_inserters*addrs_per_thread);
info[i][3] = (void *) &done;
ret = pthread_create(&threads[i], NULL,
continuous_remove, &info[i]);
cr_assert_eq(ret, 0);
}
for (i = 0; i < num_threads; i++) {
if (i == num_inserters) {
atomic_set(&done, 1);
}
ret = pthread_join(threads[i], (void **) &pret);
cr_assert_eq(ret, 0);
cr_assert_eq(pret, 0UL, "thread %d failed\n", i);
}
ret = pthread_barrier_destroy(&mtbar);
cr_assert_eq(ret, 0);
}
void gnix_init(void)
{
__setup_precomputed_crcs();
atomic_initialize(&gnix_id_counter, 0);
atomic_initialize(&file_id_counter, 0);
#ifndef NDEBUG
atomic_initialize(&gnix_debug_next_tid, 0);
#endif
}
static void cntr_setup_mt(void)
{
int i, ret;
cntr_setup_eps();
cntr_setup_av();
cntr_setup_cqs();
cntr_setup_bind_cqs();
for (i = 0; i < NUM_EPS; i++) {
ret = fi_cntr_open(dom, &cntr_attr, &ep_write_cntrs[i], 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &ep_write_cntrs[i]->fid,
FI_WRITE | FI_SEND);
cr_assert(!ret, "fi_ep_bind cntr");
ret = fi_cntr_open(dom, &cntr_attr, &ep_read_cntrs[i], 0);
cr_assert(!ret, "fi_cntr_open");
ret = fi_ep_bind(ep[i], &ep_read_cntrs[i]->fid, FI_READ);
cr_assert(!ret, "fi_ep_bind cntr");
}
cntr_setup_enable_ep();
cntr_setup_mr();
atomic_initialize(&cntr_test_next_tid, 0);
}
int fi_wait_init(struct util_fabric *fabric, struct fi_wait_attr *attr,
struct util_wait *wait)
{
struct fid_poll *poll_fid;
struct fi_poll_attr poll_attr;
int ret;
wait->prov = fabric->prov;
atomic_initialize(&wait->ref, 0);
wait->wait_fid.fid.fclass = FI_CLASS_WAIT;
switch (attr->wait_obj) {
case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
wait->wait_obj = FI_WAIT_FD;
break;
case FI_WAIT_MUTEX_COND:
wait->wait_obj = FI_WAIT_MUTEX_COND;
break;
default:
assert(0);
return -FI_EINVAL;
}
memset(&poll_attr, 0, sizeof poll_attr);
ret = fi_poll_create_(fabric->prov, NULL, &poll_attr, &poll_fid);
if (ret)
return ret;
wait->pollset = container_of(poll_fid, struct util_poll, poll_fid);
wait->fabric = fabric;
atomic_inc(&fabric->ref);
return 0;
}
static void util_fabric_init(struct util_fabric *fabric, const char *name)
{
atomic_initialize(&fabric->ref, 0);
dlist_init(&fabric->domain_list);
fastlock_init(&fabric->lock);
fabric->name = name;
}
int
usdf_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
struct fid_cq **cq_o, void *context)
{
struct usdf_cq *cq;
struct usdf_domain *udp;
int ret;
USDF_TRACE_SYS(CQ, "\n");
udp = dom_ftou(domain);
ret = usdf_cq_process_attr(attr, udp);
if (ret != 0) {
return ret;
}
cq = calloc(1, sizeof(*cq));
if (cq == NULL) {
return -FI_ENOMEM;
}
cq->cq_domain = udp;
cq->cq_fid.fid.fclass = FI_CLASS_CQ;
cq->cq_fid.fid.context = context;
cq->cq_fid.fid.ops = &usdf_cq_fi_ops;
atomic_initialize(&cq->cq_refcnt, 0);
switch (attr->format) {
case FI_CQ_FORMAT_CONTEXT:
cq->cq_fid.ops = &usdf_cq_context_ops;
break;
case FI_CQ_FORMAT_MSG:
cq->cq_fid.ops = &usdf_cq_msg_ops;
break;
case FI_CQ_FORMAT_DATA:
cq->cq_fid.ops = &usdf_cq_data_ops;
break;
default:
ret = -FI_ENOSYS;
goto fail;
}
cq->cq_attr = *attr;
*cq_o = &cq->cq_fid;
return 0;
fail:
if (cq != NULL) {
if (cq->c.hard.cq_cq != NULL) {
usd_destroy_cq(cq->c.hard.cq_cq);
}
free(cq);
}
return ret;
}
static int util_domain_init(struct util_domain *domain,
const struct fi_info *info)
{
atomic_initialize(&domain->ref, 0);
fastlock_init(&domain->lock);
domain->caps = info->caps;
domain->mode = info->mode;
domain->addr_format = info->addr_format;
domain->av_type = info->domain_attr->av_type;
domain->name = strdup(info->domain_attr->name);
return domain->name ? 0 : -FI_ENOMEM;
}
int fi_rbv_rdm_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
struct fid_cntr **cntr_fid, void *context)
{
struct fi_ibv_rdm_cntr *cntr;
struct fi_ibv_domain *dom =
container_of(domain, struct fi_ibv_domain, domain_fid);
if (attr) {
switch (attr->events) {
case FI_CNTR_EVENTS_COMP:
break;
default:
return -FI_ENOSYS;
}
switch (attr->wait_obj) {
case FI_WAIT_NONE:
case FI_WAIT_UNSPEC:
break;
case FI_WAIT_MUTEX_COND:
case FI_WAIT_SET:
case FI_WAIT_FD:
default:
return -FI_ENOSYS;
}
if (attr->flags) {
return -FI_EINVAL;
}
}
cntr = calloc(1, sizeof(*cntr));
if (!cntr)
return -FI_ENOMEM;
if (attr) {
assert(sizeof(cntr->attr) == sizeof(*attr));
memcpy(&cntr->attr, attr, sizeof(*attr));
}
cntr->fid.fid.fclass = FI_CLASS_CNTR;
cntr->fid.fid.context = context;
cntr->fid.fid.ops = &fi_ibv_rdm_cntr_fi_ops;
cntr->fid.ops = &fi_ibv_rdm_cntr_ops;
cntr->domain = dom;
atomic_initialize(&cntr->ep_ref, 0);
*cntr_fid = &cntr->fid;
return FI_SUCCESS;
}
static void continuous_lookup(void)
{
int i, ret;
pthread_t thread;
fi_addr_t addresses[MT_ADDR_COUNT];
atomic_t done;
void *info[2];
const int iters = 17;
ret = pthread_barrier_init(&mtbar, NULL, 2);
cr_assert_eq(ret, 0);
init_av_lock();
atomic_initialize(&done, 0);
memset(addresses, 0, MT_ADDR_COUNT*sizeof(fi_addr_t));
info[0] = (void *) addresses;
info[1] = (void *) &done;
ret = pthread_create(&thread, NULL, lookup_random, info);
cr_assert_eq(ret, 0);
ret = pthread_barrier_wait(&mtbar);
cr_assert((ret == PTHREAD_BARRIER_SERIAL_THREAD) || (ret == 0));
for (i = 0; i < iters; i++) {
for (i = 0; i < MT_ADDR_COUNT; i++) {
ret = fi_av_insert(av, (void *) &simple_ep_names[i], 1,
&addresses[i], 0, NULL);
cr_assert_eq(ret, 1);
}
for (i = 0; i < MT_ADDR_COUNT; i++) {
ret = fi_av_remove(av, &addresses[i], 1, 0);
cr_assert_eq(ret, FI_SUCCESS);
}
}
atomic_set(&done, 1);
ret = pthread_join(thread, NULL);
cr_assert_eq(ret, 0);
ret = pthread_barrier_destroy(&mtbar);
cr_assert_eq(ret, 0);
}
static int util_av_init(struct util_av *av, const struct fi_av_attr *attr,
const struct util_av_attr *util_attr)
{
int *entry, i, ret = 0;
atomic_initialize(&av->ref, 0);
fastlock_init(&av->lock);
av->count = attr->count ? attr->count : UTIL_DEFAULT_AV_SIZE;
av->count = roundup_power_of_two(av->count);
av->addrlen = util_attr->addrlen;
av->flags = util_attr->flags | attr->flags;
FI_INFO(av->prov, FI_LOG_AV, "AV size %zu\n", av->count);
/* TODO: Handle FI_READ */
/* TODO: Handle mmap - shared AV */
if (util_attr->flags & FI_SOURCE) {
av->hash.slots = av->count;
av->hash.total_count = av->count + util_attr->overhead;
FI_INFO(av->prov, FI_LOG_AV,
"FI_SOURCE requested, hash size %zu\n", av->hash.total_count);
}
av->data = malloc((av->count * util_attr->addrlen) +
(av->hash.total_count * sizeof(*av->hash.table)));
if (!av->data)
return -FI_ENOMEM;
for (i = 0; i < av->count - 1; i++) {
entry = util_av_get_data(av, i);
*entry = i + 1;
}
entry = util_av_get_data(av, av->count - 1);
*entry = UTIL_NO_ENTRY;
if (util_attr->flags & FI_SOURCE) {
av->hash.table = util_av_get_data(av, av->count);
util_av_hash_init(&av->hash);
}
return ret;
}
static int
usdf_dom_rdc_alloc_data(struct usdf_domain *udp)
{
struct usdf_rdm_connection *rdc;
int ret;
int i;
udp->dom_rdc_hashtab = calloc(USDF_RDM_HASH_SIZE,
sizeof(*udp->dom_rdc_hashtab));
if (udp->dom_rdc_hashtab == NULL) {
return -FI_ENOMEM;
}
SLIST_INIT(&udp->dom_rdc_free);
atomic_initialize(&udp->dom_rdc_free_cnt, 0);
for (i = 0; i < USDF_RDM_FREE_BLOCK; ++i) {
rdc = calloc(1, sizeof(*rdc));
if (rdc == NULL) {
return -FI_ENOMEM;
}
ret = usdf_timer_alloc(usdf_rdm_rdc_timeout, rdc,
&rdc->dc_timer);
if (ret != 0) {
free(rdc);
return ret;
}
rdc->dc_flags = USDF_DCS_UNCONNECTED | USDF_DCF_NEW_RX;
rdc->dc_next_rx_seq = 0;
rdc->dc_next_tx_seq = 0;
rdc->dc_last_rx_ack = rdc->dc_next_tx_seq - 1;
TAILQ_INIT(&rdc->dc_wqe_posted);
TAILQ_INIT(&rdc->dc_wqe_sent);
SLIST_INSERT_HEAD(&udp->dom_rdc_free, rdc, dc_addr_link);
atomic_inc(&udp->dom_rdc_free_cnt);
}
udp->dom_rdc_total = USDF_RDM_FREE_BLOCK;
return 0;
}
int
usdf_pep_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_pep **pep_o, void *context)
{
struct usdf_pep *pep;
struct usdf_fabric *fp;
struct sockaddr_in *sin;
int ret;
int optval;
USDF_TRACE_SYS(EP_CTRL, "\n");
if (!info) {
USDF_DBG_SYS(EP_CTRL, "null fi_info struct is invalid\n");
return -FI_EINVAL;
}
if (info->ep_attr->type != FI_EP_MSG) {
return -FI_ENODEV;
}
if ((info->caps & ~USDF_MSG_CAPS) != 0) {
return -FI_EBADF;
}
switch (info->addr_format) {
case FI_SOCKADDR:
if (((struct sockaddr *)info->src_addr)->sa_family != AF_INET) {
USDF_WARN_SYS(EP_CTRL, "non-AF_INET src_addr specified\n");
return -FI_EINVAL;
}
break;
case FI_SOCKADDR_IN:
break;
default:
USDF_WARN_SYS(EP_CTRL, "unknown/unsupported addr_format\n");
return -FI_EINVAL;
}
if (info->src_addrlen &&
info->src_addrlen != sizeof(struct sockaddr_in)) {
USDF_WARN_SYS(EP_CTRL, "unexpected src_addrlen\n");
return -FI_EINVAL;
}
fp = fab_ftou(fabric);
pep = calloc(1, sizeof(*pep));
if (pep == NULL) {
return -FI_ENOMEM;
}
pep->pep_fid.fid.fclass = FI_CLASS_PEP;
pep->pep_fid.fid.context = context;
pep->pep_fid.fid.ops = &usdf_pep_ops;
pep->pep_fid.ops = &usdf_pep_base_ops;
pep->pep_fid.cm = &usdf_pep_cm_ops;
pep->pep_fabric = fp;
pep->pep_state = USDF_PEP_UNBOUND;
pep->pep_sock = socket(AF_INET, SOCK_STREAM, 0);
if (pep->pep_sock == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_GETFL, 0);
if (ret == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_SETFL, ret | O_NONBLOCK);
if (ret == -1) {
ret = -errno;
goto fail;
}
/* set SO_REUSEADDR to prevent annoying "Address already in use" errors
* on successive runs of programs listening on a well known port */
optval = 1;
ret = setsockopt(pep->pep_sock, SOL_SOCKET, SO_REUSEADDR, &optval,
sizeof(optval));
if (ret == -1) {
ret = -errno;
goto fail;
}
pep->pep_info = fi_dupinfo(info);
if (!pep->pep_info) {
ret = -FI_ENOMEM;
goto fail;
}
if (info->src_addrlen == 0) {
/* Copy the source address information from the device
* attributes.
*/
pep->pep_info->src_addrlen = sizeof(struct sockaddr_in);
sin = calloc(1, pep->pep_info->src_addrlen);
if (!sin) {
USDF_WARN_SYS(EP_CTRL,
"calloc for src address failed\n");
goto fail;
}
sin->sin_family = AF_INET;
sin->sin_addr.s_addr = fp->fab_dev_attrs->uda_ipaddr_be;
pep->pep_info->src_addr = sin;
}
memcpy(&pep->pep_src_addr, pep->pep_info->src_addr,
pep->pep_info->src_addrlen);
/* initialize connreq freelist */
ret = pthread_spin_init(&pep->pep_cr_lock, PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&pep->pep_cr_free);
TAILQ_INIT(&pep->pep_cr_pending);
pep->pep_backlog = 10;
ret = usdf_pep_grow_backlog(pep);
if (ret != 0) {
goto fail;
}
atomic_initialize(&pep->pep_refcnt, 0);
atomic_inc(&fp->fab_refcnt);
*pep_o = pep_utof(pep);
return 0;
fail:
if (pep != NULL) {
usdf_pep_free_cr_lists(pep);
if (pep->pep_sock != -1) {
close(pep->pep_sock);
}
fi_freeinfo(pep->pep_info);
free(pep);
}
return ret;
}
int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_ep *ep_priv;
int err;
uint64_t ep_cap;
if (info)
ep_cap = info->caps;
else
ep_cap = FI_TAGGED;
domain_priv = container_of(domain, struct psmx_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
return -FI_EINVAL;
err = psmx_domain_check_features(domain_priv, ep_cap);
if (err)
return err;
ep_priv = (struct psmx_fid_ep *) calloc(1, sizeof *ep_priv);
if (!ep_priv)
return -FI_ENOMEM;
ep_priv->ep.fid.fclass = FI_CLASS_EP;
ep_priv->ep.fid.context = context;
ep_priv->ep.fid.ops = &psmx_fi_ops;
ep_priv->ep.ops = &psmx_ep_ops;
ep_priv->ep.cm = &psmx_cm_ops;
ep_priv->domain = domain_priv;
atomic_initialize(&ep_priv->ref, 0);
PSMX_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX_NOCOMP_SEND_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv;
PSMX_CTXT_TYPE(&ep_priv->nocomp_recv_context) = PSMX_NOCOMP_RECV_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_recv_context) = ep_priv;
if (ep_cap & FI_TAGGED)
ep_priv->ep.tagged = &psmx_tagged_ops;
if (ep_cap & FI_MSG)
ep_priv->ep.msg = &psmx_msg_ops;
if ((ep_cap & FI_MSG) && psmx_env.am_msg)
ep_priv->ep.msg = &psmx_msg2_ops;
if (ep_cap & FI_RMA)
ep_priv->ep.rma = &psmx_rma_ops;
if (ep_cap & FI_ATOMICS)
ep_priv->ep.atomic = &psmx_atomic_ops;
ep_priv->caps = ep_cap;
err = psmx_domain_enable_ep(domain_priv, ep_priv);
if (err) {
free(ep_priv);
return err;
}
psmx_domain_acquire(domain_priv);
if (info) {
if (info->tx_attr)
ep_priv->tx_flags = info->tx_attr->op_flags;
if (info->rx_attr)
ep_priv->rx_flags = info->rx_attr->op_flags;
}
psmx_ep_optimize_ops(ep_priv);
*ep = &ep_priv->ep;
return 0;
}
int gnix_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
int ret = FI_SUCCESS;
struct gnix_fid_domain *domain_priv;
struct gnix_fid_ep *ep_priv;
gnix_hashtable_attr_t gnix_ht_attr;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if ((domain == NULL) || (info == NULL) || (ep == NULL))
return -FI_EINVAL;
if (info->ep_attr->type != FI_EP_RDM)
return -FI_ENOSYS;
domain_priv = container_of(domain, struct gnix_fid_domain, domain_fid);
ep_priv = calloc(1, sizeof *ep_priv);
if (!ep_priv)
return -FI_ENOMEM;
ep_priv->ep_fid.fid.fclass = FI_CLASS_EP;
ep_priv->ep_fid.fid.context = context;
ep_priv->ep_fid.fid.ops = &gnix_ep_fi_ops;
ep_priv->ep_fid.ops = &gnix_ep_ops;
ep_priv->domain = domain_priv;
ep_priv->type = info->ep_attr->type;
fastlock_init(&ep_priv->vc_list_lock);
dlist_init(&ep_priv->wc_vc_list);
atomic_initialize(&ep_priv->active_fab_reqs, 0);
atomic_initialize(&ep_priv->ref_cnt, 0);
fastlock_init(&ep_priv->recv_queue_lock);
slist_init(&ep_priv->unexp_recv_queue);
slist_init(&ep_priv->posted_recv_queue);
slist_init(&ep_priv->pending_recv_comp_queue);
if (info->tx_attr)
ep_priv->op_flags = info->tx_attr->op_flags;
if (info->rx_attr)
ep_priv->op_flags |= info->rx_attr->op_flags;
ret = __fr_freelist_init(ep_priv);
if (ret != FI_SUCCESS) {
GNIX_ERR(FI_LOG_EP_CTRL,
"Error allocating gnix_fab_req freelist (%s)",
fi_strerror(-ret));
goto err1;
}
ep_priv->ep_fid.msg = &gnix_ep_msg_ops;
ep_priv->ep_fid.rma = &gnix_ep_rma_ops;
ep_priv->ep_fid.tagged = &gnix_ep_tagged_ops;
ep_priv->ep_fid.atomic = NULL;
ep_priv->ep_fid.cm = &gnix_cm_ops;
/*
* TODO, initialize vc hash table
*/
if (ep_priv->type == FI_EP_RDM) {
ret = _gnix_cm_nic_alloc(domain_priv,
&ep_priv->cm_nic);
if (ret != FI_SUCCESS)
goto err;
gnix_ht_attr.ht_initial_size = domain_priv->params.ct_init_size;
gnix_ht_attr.ht_maximum_size = domain_priv->params.ct_max_size;
gnix_ht_attr.ht_increase_step = domain_priv->params.ct_step;
gnix_ht_attr.ht_increase_type = GNIX_HT_INCREASE_MULT;
gnix_ht_attr.ht_collision_thresh = 500;
gnix_ht_attr.ht_hash_seed = 0xdeadbeefbeefdead;
gnix_ht_attr.ht_internal_locking = 1;
ep_priv->vc_ht = calloc(1, sizeof(struct gnix_hashtable));
if (ep_priv->vc_ht == NULL)
goto err;
ret = _gnix_ht_init(ep_priv->vc_ht, &gnix_ht_attr);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"gnix_ht_init call returned %d\n",
ret);
goto err;
}
} else {
ep_priv->cm_nic = NULL;
ep_priv->vc = NULL;
}
ep_priv->progress_fn = NULL;
ep_priv->rx_progress_fn = NULL;
ret = gnix_nic_alloc(domain_priv, &ep_priv->nic);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_nic_alloc call returned %d\n",
ret);
goto err;
}
/*
* if smsg callbacks not present hook them up now
*/
if (ep_priv->nic->smsg_callbacks == NULL)
ep_priv->nic->smsg_callbacks = gnix_ep_smsg_callbacks;
atomic_inc(&domain_priv->ref_cnt);
*ep = &ep_priv->ep_fid;
return ret;
err1:
__fr_freelist_destroy(ep_priv);
err:
if (ep_priv->vc_ht != NULL) {
_gnix_ht_destroy(ep_priv->vc_ht); /* may not be initialized but
okay */
free(ep_priv->vc_ht);
ep_priv->vc_ht = NULL;
}
if (ep_priv->cm_nic != NULL)
ret = _gnix_cm_nic_free(ep_priv->cm_nic);
free(ep_priv);
return ret;
}
int sock_domain(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **dom, void *context)
{
struct sock_domain *sock_domain;
struct sock_fabric *fab;
int ret;
fab = container_of(fabric, struct sock_fabric, fab_fid);
if (info && info->domain_attr) {
ret = sock_verify_domain_attr(info->domain_attr);
if (ret)
return -FI_EINVAL;
}
sock_domain = calloc(1, sizeof(*sock_domain));
if (!sock_domain)
return -FI_ENOMEM;
fastlock_init(&sock_domain->lock);
atomic_initialize(&sock_domain->ref, 0);
if (info) {
sock_domain->info = *info;
} else {
SOCK_LOG_ERROR("invalid fi_info\n");
goto err;
}
sock_domain->dom_fid.fid.fclass = FI_CLASS_DOMAIN;
sock_domain->dom_fid.fid.context = context;
sock_domain->dom_fid.fid.ops = &sock_dom_fi_ops;
sock_domain->dom_fid.ops = &sock_dom_ops;
sock_domain->dom_fid.mr = &sock_dom_mr_ops;
if (!info->domain_attr ||
info->domain_attr->data_progress == FI_PROGRESS_UNSPEC)
sock_domain->progress_mode = FI_PROGRESS_AUTO;
else
sock_domain->progress_mode = info->domain_attr->data_progress;
sock_domain->pe = sock_pe_init(sock_domain);
if (!sock_domain->pe) {
SOCK_LOG_ERROR("Failed to init PE\n");
goto err;
}
sock_domain->mr_heap = rbtNew(&sock_compare_mr_keys);
if (!sock_domain->mr_heap) {
goto err;
}
sock_domain->fab = fab;
*dom = &sock_domain->dom_fid;
if (info->domain_attr)
sock_domain->attr = *(info->domain_attr);
else
sock_domain->attr = sock_domain_attr;
sock_dom_add_to_list(sock_domain);
return 0;
err:
free(sock_domain);
return -FI_EINVAL;
}
int
usdf_cq_make_soft(struct usdf_cq *cq)
{
struct fi_ops_cq *hard_ops;
struct fi_ops_cq *soft_ops;
struct usdf_cq_hard *hcq;
struct usd_cq *ucq;
void (*rtn)(struct usdf_cq_hard *hcq);
switch (cq->cq_attr.format) {
case FI_CQ_FORMAT_CONTEXT:
hard_ops = &usdf_cq_context_ops;
soft_ops = &usdf_cq_context_soft_ops;
break;
case FI_CQ_FORMAT_MSG:
hard_ops = &usdf_cq_msg_ops;
soft_ops = &usdf_cq_msg_soft_ops;
break;
case FI_CQ_FORMAT_DATA:
hard_ops = &usdf_cq_data_ops;
soft_ops = &usdf_cq_data_soft_ops;
break;
default:
return 0;
}
rtn = usdf_progress_hard_cq;
if (cq->cq_fid.ops == hard_ops) {
/* save the CQ before we trash the union */
ucq = cq->c.hard.cq_cq;
/* fill in the soft part of union */
TAILQ_INIT(&cq->c.soft.cq_list);
cq->c.soft.cq_comps = calloc(cq->cq_attr.size,
sizeof(struct usdf_cq_soft_entry));
if (cq->c.soft.cq_comps == NULL) {
return -FI_ENOMEM;
}
cq->c.soft.cq_end = cq->c.soft.cq_comps + cq->cq_attr.size;
cq->c.soft.cq_head = cq->c.soft.cq_comps;
cq->c.soft.cq_tail = cq->c.soft.cq_comps;
/* need to add hard queue to list? */
if (ucq != NULL) {
hcq = malloc(sizeof(*hcq));
if (hcq == NULL) {
free(cq->c.soft.cq_comps);
cq->c.hard.cq_cq = ucq; /* restore */
return -FI_ENOMEM;
}
hcq->cqh_cq = cq;
hcq->cqh_ucq = ucq;
hcq->cqh_progress = rtn;
atomic_initialize(&hcq->cqh_refcnt,
atomic_get(&cq->cq_refcnt));
TAILQ_INSERT_HEAD(&cq->c.soft.cq_list, hcq, cqh_link);
}
cq->cq_fid.ops = soft_ops;
}
return 0;
}
int psmx_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx_fid_domain *domain_priv;
struct psmx_fid_ep *ep_priv;
int err;
uint64_t ep_cap;
if (info)
ep_cap = info->caps;
else
ep_cap = FI_TAGGED;
domain_priv = container_of(domain, struct psmx_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
return -FI_EINVAL;
if (info && info->ep_attr && info->ep_attr->auth_key) {
if (info->ep_attr->auth_keylen != sizeof(psm_uuid_t)) {
FI_WARN(&psmx_prov, FI_LOG_EP_CTRL,
"Invalid auth_key_len %d, should be %d.\n",
info->ep_attr->auth_keylen,
sizeof(psm_uuid_t));
return -FI_EINVAL;
}
if (memcmp(domain_priv->fabric->uuid, info->ep_attr->auth_key,
sizeof(psm_uuid_t))) {
FI_WARN(&psmx_prov, FI_LOG_EP_CTRL,
"Invalid auth_key: %s\n",
psmx_uuid_to_string((void *)info->ep_attr->auth_key));
return -FI_EINVAL;
}
}
err = psmx_domain_check_features(domain_priv, ep_cap);
if (err)
return err;
ep_priv = (struct psmx_fid_ep *) calloc(1, sizeof *ep_priv);
if (!ep_priv)
return -FI_ENOMEM;
ep_priv->ep.fid.fclass = FI_CLASS_EP;
ep_priv->ep.fid.context = context;
ep_priv->ep.fid.ops = &psmx_fi_ops;
ep_priv->ep.ops = &psmx_ep_ops;
ep_priv->ep.cm = &psmx_cm_ops;
ep_priv->domain = domain_priv;
atomic_initialize(&ep_priv->ref, 0);
PSMX_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX_NOCOMP_SEND_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv;
PSMX_CTXT_TYPE(&ep_priv->nocomp_recv_context) = PSMX_NOCOMP_RECV_CONTEXT;
PSMX_CTXT_EP(&ep_priv->nocomp_recv_context) = ep_priv;
if (ep_cap & FI_TAGGED)
ep_priv->ep.tagged = &psmx_tagged_ops;
if (ep_cap & FI_MSG)
ep_priv->ep.msg = &psmx_msg_ops;
if ((ep_cap & FI_MSG) && psmx_env.am_msg)
ep_priv->ep.msg = &psmx_msg2_ops;
if (ep_cap & FI_RMA)
ep_priv->ep.rma = &psmx_rma_ops;
if (ep_cap & FI_ATOMICS)
ep_priv->ep.atomic = &psmx_atomic_ops;
ep_priv->caps = ep_cap;
err = psmx_domain_enable_ep(domain_priv, ep_priv);
if (err) {
free(ep_priv);
return err;
}
psmx_domain_acquire(domain_priv);
if (info) {
if (info->tx_attr)
ep_priv->tx_flags = info->tx_attr->op_flags;
if (info->rx_attr)
ep_priv->rx_flags = info->rx_attr->op_flags;
}
psmx_ep_optimize_ops(ep_priv);
ep_priv->service = PSMX_ANY_SERVICE;
if (info && info->src_addr)
ep_priv->service = ((struct psmx_src_name *)info->src_addr)->service;
if (ep_priv->service == PSMX_ANY_SERVICE)
ep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)ep_priv & 0xFFFF);
psmx_ns_add_local_name(ep_priv->service, domain_priv->psm_epid);
*ep = &ep_priv->ep;
return 0;
}
ssize_t _gnix_rma(struct gnix_fid_ep *ep, enum gnix_fab_req_type fr_type,
uint64_t loc_addr, size_t len, void *mdesc,
uint64_t dest_addr, uint64_t rem_addr, uint64_t mkey,
void *context, uint64_t flags, uint64_t data)
{
struct gnix_vc *vc;
struct gnix_fab_req *req;
struct gnix_fid_mem_desc *md = NULL;
int rc;
int rdma;
struct fid_mr *auto_mr = NULL;
if (!ep) {
return -FI_EINVAL;
}
if ((flags & FI_INJECT) && (len > GNIX_INJECT_SIZE)) {
GNIX_INFO(FI_LOG_EP_DATA,
"RMA length %d exceeds inject max size: %d\n",
len, GNIX_INJECT_SIZE);
return -FI_EINVAL;
}
/* find VC for target */
rc = _gnix_ep_get_vc(ep, dest_addr, &vc);
if (rc) {
GNIX_INFO(FI_LOG_EP_DATA,
"_gnix_ep_get_vc() failed, addr: %lx, rc:\n",
dest_addr, rc);
return rc;
}
/* setup fabric request */
req = _gnix_fr_alloc(ep);
if (!req) {
GNIX_INFO(FI_LOG_EP_DATA, "_gnix_fr_alloc() failed\n");
return -FI_ENOSPC;
}
rdma = len >= ep->domain->params.rma_rdma_thresh;
req->type = fr_type;
req->gnix_ep = ep;
req->vc = vc;
req->user_context = context;
req->work_fn = _gnix_rma_post_req;
atomic_initialize(&req->rma.outstanding_txds, 0);
if (fr_type == GNIX_FAB_RQ_RDMA_READ &&
(rem_addr & GNI_READ_ALIGN_MASK || len & GNI_READ_ALIGN_MASK)) {
if (len >= GNIX_RMA_UREAD_CHAINED_THRESH) {
GNIX_INFO(FI_LOG_EP_DATA,
"Using CT for unaligned GET, req: %p\n",
req);
flags |= GNIX_RMA_CHAINED;
} else {
GNIX_INFO(FI_LOG_EP_DATA,
"Using tmp buf for unaligned GET, req: %p\n",
req);
flags |= GNIX_RMA_INDIRECT;
}
if (rdma)
req->work_fn = _gnix_rma_post_rdma_chain_req;
}
if (!(flags & GNIX_RMA_INDIRECT) && !mdesc &&
(rdma || fr_type == GNIX_FAB_RQ_RDMA_READ)) {
/* We need to auto-register the source buffer. */
rc = gnix_mr_reg(&ep->domain->domain_fid.fid, (void *)loc_addr,
len, FI_READ | FI_WRITE, 0, 0, 0, &auto_mr,
NULL);
if (rc != FI_SUCCESS) {
GNIX_INFO(FI_LOG_EP_DATA,
"Failed to auto-register local buffer: %d\n",
rc);
goto err_auto_reg;
}
flags |= FI_LOCAL_MR;
mdesc = (void *)auto_mr;
GNIX_INFO(FI_LOG_EP_DATA, "auto-reg MR: %p\n", auto_mr);
}
if (mdesc)
md = container_of(mdesc, struct gnix_fid_mem_desc, mr_fid);
req->rma.loc_md = (void *)md;
req->rma.rem_addr = rem_addr;
req->rma.rem_mr_key = mkey;
req->rma.len = len;
req->rma.imm = data;
req->flags = flags;
if (req->flags & FI_INJECT) {
memcpy(req->inject_buf, (void *)loc_addr, len);
req->rma.loc_addr = (uint64_t)req->inject_buf;
} else {
req->rma.loc_addr = loc_addr;
}
/* Inject interfaces always suppress completions. If
* SELECTIVE_COMPLETION is set, honor any setting. Otherwise, always
* deliver a completion. */
if ((flags & GNIX_SUPPRESS_COMPLETION) ||
(ep->send_selective_completion && !(flags & FI_COMPLETION))) {
req->flags &= ~FI_COMPLETION;
} else {
req->flags |= FI_COMPLETION;
}
if (rdma) {
req->flags |= GNIX_RMA_RDMA;
}
GNIX_INFO(FI_LOG_EP_DATA, "Queuing (%p %p %d)\n",
(void *)loc_addr, (void *)rem_addr, len);
return _gnix_vc_queue_tx_req(req);
err_auto_reg:
_gnix_fr_free(req->vc->ep, req);
return rc;
}
int _gnix_vc_alloc(struct gnix_fid_ep *ep_priv,
struct gnix_av_addr_entry *entry, struct gnix_vc **vc)
{
int ret = FI_SUCCESS;
int remote_id;
struct gnix_vc *vc_ptr = NULL;
struct gnix_cm_nic *cm_nic = NULL;
struct gnix_nic *nic = NULL;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
nic = ep_priv->nic;
if (nic == NULL)
return -FI_EINVAL;
cm_nic = ep_priv->cm_nic;
if (cm_nic == NULL)
return -FI_EINVAL;
vc_ptr = calloc(1, sizeof(*vc_ptr));
if (!vc_ptr)
return -FI_ENOMEM;
vc_ptr->conn_state = GNIX_VC_CONN_NONE;
if (entry) {
memcpy(&vc_ptr->peer_addr,
&entry->gnix_addr,
sizeof(struct gnix_address));
vc_ptr->peer_cm_nic_addr.device_addr =
entry->gnix_addr.device_addr;
vc_ptr->peer_cm_nic_addr.cdm_id =
entry->cm_nic_cdm_id;
} else {
vc_ptr->peer_addr.device_addr = -1;
vc_ptr->peer_addr.cdm_id = -1;
vc_ptr->peer_cm_nic_addr.device_addr = -1;
vc_ptr->peer_cm_nic_addr.cdm_id = -1;
}
vc_ptr->ep = ep_priv;
slist_init(&vc_ptr->work_queue);
fastlock_init(&vc_ptr->work_queue_lock);
slist_init(&vc_ptr->tx_queue);
fastlock_init(&vc_ptr->tx_queue_lock);
dlist_init(&vc_ptr->rx_list);
dlist_init(&vc_ptr->work_list);
dlist_init(&vc_ptr->tx_list);
vc_ptr->peer_fi_addr = FI_ADDR_NOTAVAIL;
atomic_initialize(&vc_ptr->outstanding_tx_reqs, 0);
ret = _gnix_alloc_bitmap(&vc_ptr->flags, 1);
assert(!ret);
/*
* we need an id for the vc to allow for quick lookup
* based on GNI_CQ_GET_INST_ID
*/
ret = _gnix_nic_get_rem_id(nic, &remote_id, vc_ptr);
if (ret != FI_SUCCESS)
goto err;
vc_ptr->vc_id = remote_id;
*vc = vc_ptr;
return ret;
err:
if (vc_ptr)
free(vc_ptr);
return ret;
}
int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av, void *context)
{
int ret = 0;
struct sock_domain *dom;
struct sock_av *_av;
size_t table_sz, i;
uint64_t flags = O_RDWR;
if (!attr || sock_verify_av_attr(attr))
return -FI_EINVAL;
dom = container_of(domain, struct sock_domain, dom_fid);
if (dom->attr.av_type != FI_AV_UNSPEC && attr &&
dom->attr.av_type != attr->type)
return -FI_EINVAL;
_av = calloc(1, sizeof(*_av));
if (!_av)
return -FI_ENOMEM;
_av->attr = *attr;
_av->attr.count = (attr->count) ? attr->count : sock_av_def_sz;
table_sz = sizeof(struct sock_av_table_hdr) +
_av->attr.count * sizeof(struct sock_av_addr);
if (attr->name) {
_av->name = strdup(attr->name);
if (!_av->name) {
ret = -FI_ENOMEM;
goto err1;
}
if (!(attr->flags & FI_READ))
flags |= O_CREAT;
for (i = 0; i < strlen(_av->name); i++)
if (_av->name[i] == ' ')
_av->name[i] = '_';
SOCK_LOG_DBG("Creating shm segment :%s (size: %lu)\n",
_av->name, table_sz);
_av->shared_fd = shm_open(_av->name, flags, S_IRUSR | S_IWUSR);
if (_av->shared_fd < 0) {
SOCK_LOG_ERROR("shm_open failed\n");
ret = -FI_EINVAL;
goto err2;
}
if (ftruncate(_av->shared_fd, table_sz) == -1) {
SOCK_LOG_ERROR("ftruncate failed\n");
shm_unlink(_av->name);
ret = -FI_EINVAL;
goto err2;
}
_av->table_hdr = mmap(NULL, table_sz, PROT_READ | PROT_WRITE,
MAP_SHARED, _av->shared_fd, 0);
if (attr->flags & FI_READ) {
if (_av->table_hdr->size != _av->attr.count) {
ret = -FI_EINVAL;
goto err2;
}
} else {
_av->table_hdr->size = _av->attr.count;
_av->table_hdr->stored = 0;
}
if (_av->table_hdr == MAP_FAILED) {
SOCK_LOG_ERROR("mmap failed\n");
shm_unlink(_av->name);
ret = -FI_EINVAL;
goto err2;
}
} else {
_av->table_hdr = calloc(1, table_sz);
if (!_av->table_hdr) {
ret = -FI_ENOMEM;
goto err3;
}
_av->table_hdr->size = _av->attr.count;
_av->table_hdr->req_sz = attr->count;
}
_av->table = (struct sock_av_addr *)((char *)_av->table_hdr +
sizeof(struct sock_av_table_hdr));
_av->av_fid.fid.fclass = FI_CLASS_AV;
_av->av_fid.fid.context = context;
_av->av_fid.fid.ops = &sock_av_fi_ops;
switch (attr->type) {
case FI_AV_MAP:
_av->av_fid.ops = &sock_am_ops;
break;
case FI_AV_TABLE:
_av->av_fid.ops = &sock_at_ops;
break;
default:
ret = -FI_EINVAL;
goto err3;
}
atomic_initialize(&_av->ref, 0);
atomic_inc(&dom->ref);
_av->domain = dom;
switch (dom->info.addr_format) {
case FI_SOCKADDR_IN:
_av->addrlen = sizeof(struct sockaddr_in);
break;
default:
SOCK_LOG_ERROR("Invalid address format: only IPv4 supported\n");
ret = -FI_EINVAL;
goto err3;
}
_av->rx_ctx_bits = attr->rx_ctx_bits;
_av->mask = attr->rx_ctx_bits ?
((uint64_t)1 << (64 - attr->rx_ctx_bits)) - 1 : ~0;
*av = &_av->av_fid;
return 0;
err3:
free(_av->table_hdr);
err2:
free(_av->name);
err1:
free(_av);
return ret;
}
static int
usdf_fabric_open(struct fi_fabric_attr *fattrp, struct fid_fabric **fabric,
void *context)
{
struct fid_fabric *ff;
struct usdf_fabric *fp;
struct usdf_usnic_info *dp;
struct usdf_dev_entry *dep;
struct epoll_event ev;
struct sockaddr_in sin;
int ret;
int d;
USDF_TRACE("\n");
/* Make sure this fabric exists */
dp = __usdf_devinfo;
for (d = 0; d < dp->uu_num_devs; ++d) {
dep = &dp->uu_info[d];
if (dep->ue_dev_ok &&
strcmp(fattrp->name, dep->ue_dattr.uda_devname) == 0) {
break;
}
}
if (d >= dp->uu_num_devs) {
USDF_INFO("device \"%s\" does not exit, returning -FI_ENODEV\n",
fattrp->name);
return -FI_ENODEV;
}
fp = calloc(1, sizeof(*fp));
if (fp == NULL) {
USDF_INFO("unable to allocate memory for fabric\n");
return -FI_ENOMEM;
}
fp->fab_epollfd = -1;
fp->fab_arp_sockfd = -1;
LIST_INIT(&fp->fab_domain_list);
fp->fab_attr.fabric = fab_utof(fp);
fp->fab_attr.name = strdup(fattrp->name);
fp->fab_attr.prov_name = strdup(USDF_PROV_NAME);
fp->fab_attr.prov_version = USDF_PROV_VERSION;
if (fp->fab_attr.name == NULL ||
fp->fab_attr.prov_name == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
fp->fab_fid.fid.fclass = FI_CLASS_FABRIC;
fp->fab_fid.fid.context = context;
fp->fab_fid.fid.ops = &usdf_fi_ops;
fp->fab_fid.ops = &usdf_ops_fabric;
fp->fab_dev_attrs = &dep->ue_dattr;
fp->fab_epollfd = epoll_create(1024);
if (fp->fab_epollfd == -1) {
ret = -errno;
USDF_INFO("unable to allocate epoll fd\n");
goto fail;
}
fp->fab_eventfd = eventfd(0, EFD_NONBLOCK | EFD_SEMAPHORE);
if (fp->fab_eventfd == -1) {
ret = -errno;
USDF_INFO("unable to allocate event fd\n");
goto fail;
}
fp->fab_poll_item.pi_rtn = usdf_fabric_progression_cb;
fp->fab_poll_item.pi_context = fp;
ev.events = EPOLLIN;
ev.data.ptr = &fp->fab_poll_item;
ret = epoll_ctl(fp->fab_epollfd, EPOLL_CTL_ADD, fp->fab_eventfd, &ev);
if (ret == -1) {
ret = -errno;
USDF_INFO("unable to EPOLL_CTL_ADD\n");
goto fail;
}
/* initialize timer subsystem */
ret = usdf_timer_init(fp);
if (ret != 0) {
USDF_INFO("unable to initialize timer\n");
goto fail;
}
ret = pthread_create(&fp->fab_thread, NULL,
usdf_fabric_progression_thread, fp);
if (ret != 0) {
ret = -ret;
USDF_INFO("unable to create progress thread\n");
goto fail;
}
/* create and bind socket for ARP resolution */
memset(&sin, 0, sizeof(sin));
sin.sin_family = AF_INET;
sin.sin_addr.s_addr = fp->fab_dev_attrs->uda_ipaddr_be;
fp->fab_arp_sockfd = socket(AF_INET, SOCK_DGRAM, 0);
if (fp->fab_arp_sockfd == -1) {
USDF_INFO("unable to create socket\n");
goto fail;
}
ret = bind(fp->fab_arp_sockfd, (struct sockaddr *) &sin, sizeof(sin));
if (ret == -1) {
ret = -errno;
goto fail;
}
atomic_initialize(&fp->fab_refcnt, 0);
fattrp->fabric = fab_utof(fp);
fattrp->prov_version = USDF_PROV_VERSION;
*fabric = fab_utof(fp);
USDF_INFO("successfully opened %s/%s\n", fattrp->name,
fp->fab_dev_attrs->uda_ifname);
return 0;
fail:
ff = fab_utof(fp);
usdf_fabric_close(&ff->fid);
USDF_DBG("returning %d (%s)\n", ret, fi_strerror(-ret));
return ret;
}
int sock_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
struct fid_cntr **cntr, void *context)
{
int ret;
struct sock_domain *dom;
struct sock_cntr *_cntr;
struct fi_wait_attr wait_attr;
struct sock_fid_list *list_entry;
struct sock_wait *wait;
dom = container_of(domain, struct sock_domain, dom_fid);
if (attr && sock_cntr_verify_attr(attr))
return -FI_ENOSYS;
_cntr = calloc(1, sizeof(*_cntr));
if (!_cntr)
return -FI_ENOMEM;
ret = pthread_cond_init(&_cntr->cond, NULL);
if (ret)
goto err;
if (attr == NULL)
memcpy(&_cntr->attr, &sock_cntr_add, sizeof(sock_cntr_attr));
else
memcpy(&_cntr->attr, attr, sizeof(sock_cntr_attr));
switch (_cntr->attr.wait_obj) {
case FI_WAIT_NONE:
case FI_WAIT_UNSPEC:
case FI_WAIT_MUTEX_COND:
_cntr->signal = 0;
break;
case FI_WAIT_FD:
wait_attr.flags = 0;
wait_attr.wait_obj = FI_WAIT_FD;
ret = sock_wait_open(&dom->fab->fab_fid, &wait_attr,
&_cntr->waitset);
if (ret) {
ret = FI_EINVAL;
goto err;
}
_cntr->signal = 1;
break;
case FI_WAIT_SET:
if (!attr) {
ret = FI_EINVAL;
goto err;
}
_cntr->waitset = attr->wait_set;
_cntr->signal = 1;
wait = container_of(attr->wait_set, struct sock_wait, wait_fid);
list_entry = calloc(1, sizeof(*list_entry));
dlist_init(&list_entry->entry);
list_entry->fid = &_cntr->cntr_fid.fid;
dlist_insert_after(&list_entry->entry, &wait->fid_list);
break;
default:
break;
}
pthread_mutex_init(&_cntr->mut, NULL);
fastlock_init(&_cntr->list_lock);
atomic_initialize(&_cntr->ref, 0);
atomic_initialize(&_cntr->err_cnt, 0);
atomic_initialize(&_cntr->value, 0);
atomic_initialize(&_cntr->threshold, ~0);
dlist_init(&_cntr->tx_list);
dlist_init(&_cntr->rx_list);
dlist_init(&_cntr->trigger_list);
fastlock_init(&_cntr->trigger_lock);
_cntr->cntr_fid.fid.fclass = FI_CLASS_CNTR;
_cntr->cntr_fid.fid.context = context;
_cntr->cntr_fid.fid.ops = &sock_cntr_fi_ops;
_cntr->cntr_fid.ops = &sock_cntr_ops;
atomic_inc(&dom->ref);
_cntr->domain = dom;
*cntr = &_cntr->cntr_fid;
return 0;
err:
free(_cntr);
return -ret;
}
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_context *item;
uint8_t vlane;
uint64_t ep_cap;
int err = -FI_EINVAL;
int i;
if (info)
ep_cap = info->caps;
else
ep_cap = FI_TAGGED;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
err = psmx2_domain_check_features(domain_priv, ep_cap);
if (err)
goto errout;
err = psmx2_alloc_vlane(domain_priv, &vlane);
if (err)
goto errout;
ep_priv = (struct psmx2_fid_ep *) calloc(1, sizeof *ep_priv);
if (!ep_priv) {
err = -FI_ENOMEM;
goto errout_free_vlane;
}
ep_priv->ep.fid.fclass = FI_CLASS_EP;
ep_priv->ep.fid.context = context;
ep_priv->ep.fid.ops = &psmx2_fi_ops;
ep_priv->ep.ops = &psmx2_ep_ops;
ep_priv->ep.cm = &psmx2_cm_ops;
ep_priv->domain = domain_priv;
ep_priv->vlane = vlane;
atomic_initialize(&ep_priv->ref, 0);
PSMX2_CTXT_TYPE(&ep_priv->nocomp_send_context) = PSMX2_NOCOMP_SEND_CONTEXT;
PSMX2_CTXT_EP(&ep_priv->nocomp_send_context) = ep_priv;
PSMX2_CTXT_TYPE(&ep_priv->nocomp_recv_context) = PSMX2_NOCOMP_RECV_CONTEXT;
PSMX2_CTXT_EP(&ep_priv->nocomp_recv_context) = ep_priv;
if (ep_cap & FI_TAGGED)
ep_priv->ep.tagged = &psmx2_tagged_ops;
if (ep_cap & FI_MSG)
ep_priv->ep.msg = &psmx2_msg_ops;
if (ep_cap & FI_RMA)
ep_priv->ep.rma = &psmx2_rma_ops;
if (ep_cap & FI_ATOMICS)
ep_priv->ep.atomic = &psmx2_atomic_ops;
ep_priv->caps = ep_cap;
err = psmx2_domain_enable_ep(domain_priv, ep_priv);
if (err)
goto errout_free_ep;
psmx2_domain_acquire(domain_priv);
domain_priv->eps[ep_priv->vlane] = ep_priv;
if (info) {
if (info->tx_attr)
ep_priv->tx_flags = info->tx_attr->op_flags;
if (info->rx_attr)
ep_priv->rx_flags = info->rx_attr->op_flags;
}
psmx2_ep_optimize_ops(ep_priv);
slist_init(&ep_priv->free_context_list);
fastlock_init(&ep_priv->context_lock);
#define PSMX2_FREE_CONTEXT_LIST_SIZE 64
for (i=0; i<PSMX2_FREE_CONTEXT_LIST_SIZE; i++) {
item = calloc(1, sizeof(*item));
if (!item) {
FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL, "out of memory.\n");
exit(-1);
}
slist_insert_tail(&item->list_entry, &ep_priv->free_context_list);
}
*ep = &ep_priv->ep;
return 0;
errout_free_ep:
free(ep_priv);
errout_free_vlane:
psmx2_free_vlane(domain_priv, vlane);
errout:
return err;
}
int sock_av_open(struct fid_domain *domain, struct fi_av_attr *attr,
struct fid_av **av, void *context)
{
int ret = 0;
struct sock_domain *dom;
struct sock_av *_av;
size_t table_sz;
if (!attr || sock_verify_av_attr(attr))
return -FI_EINVAL;
if (attr->type == FI_AV_UNSPEC)
attr->type = FI_AV_TABLE;
dom = container_of(domain, struct sock_domain, dom_fid);
if (dom->attr.av_type != FI_AV_UNSPEC &&
dom->attr.av_type != attr->type)
return -FI_EINVAL;
_av = calloc(1, sizeof(*_av));
if (!_av)
return -FI_ENOMEM;
_av->attr = *attr;
_av->attr.count = (attr->count) ? attr->count : sock_av_def_sz;
table_sz = SOCK_AV_TABLE_SZ(_av->attr.count, attr->name);
if (attr->name) {
ret = ofi_shm_map(&_av->shm, attr->name, table_sz,
attr->flags & FI_READ, (void**)&_av->table_hdr);
if (ret || _av->table_hdr == MAP_FAILED) {
SOCK_LOG_ERROR("map failed\n");
ret = -FI_EINVAL;
goto err;
}
_av->idx_arr = (uint64_t *)(_av->table_hdr + 1);
_av->attr.map_addr = _av->idx_arr;
attr->map_addr = _av->attr.map_addr;
SOCK_LOG_DBG("Updating map_addr: %p\n", _av->attr.map_addr);
if (attr->flags & FI_READ) {
if (_av->table_hdr->size != _av->attr.count) {
ret = -FI_EINVAL;
goto err2;
}
} else {
_av->table_hdr->size = _av->attr.count;
_av->table_hdr->stored = 0;
}
_av->shared = 1;
} else {
_av->table_hdr = calloc(1, table_sz);
if (!_av->table_hdr) {
ret = -FI_ENOMEM;
goto err;
}
_av->table_hdr->size = _av->attr.count;
_av->table_hdr->req_sz = attr->count;
}
sock_update_av_table(_av, _av->attr.count);
_av->av_fid.fid.fclass = FI_CLASS_AV;
_av->av_fid.fid.context = context;
_av->av_fid.fid.ops = &sock_av_fi_ops;
switch (attr->type) {
case FI_AV_MAP:
_av->av_fid.ops = &sock_am_ops;
break;
case FI_AV_TABLE:
_av->av_fid.ops = &sock_at_ops;
break;
default:
ret = -FI_EINVAL;
goto err2;
}
atomic_initialize(&_av->ref, 0);
atomic_inc(&dom->ref);
_av->domain = dom;
switch (dom->info.addr_format) {
case FI_SOCKADDR_IN:
_av->addrlen = sizeof(struct sockaddr_in);
break;
default:
SOCK_LOG_ERROR("Invalid address format: only IPv4 supported\n");
ret = -FI_EINVAL;
goto err2;
}
_av->rx_ctx_bits = attr->rx_ctx_bits;
_av->mask = attr->rx_ctx_bits ?
((uint64_t)1 << (64 - attr->rx_ctx_bits)) - 1 : ~0;
*av = &_av->av_fid;
return 0;
err2:
if(attr->name) {
ofi_shm_unmap(&_av->shm);
} else {
if(_av->table_hdr && _av->table_hdr != MAP_FAILED)
free(_av->table_hdr);
}
err:
free(_av);
return ret;
}
int
usdf_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct usdf_fabric *fp;
struct usdf_domain *udp;
struct sockaddr_in *sin;
size_t addrlen;
int ret;
#if ENABLE_DEBUG
char requested[INET_ADDRSTRLEN], actual[INET_ADDRSTRLEN];
#endif
USDF_TRACE_SYS(DOMAIN, "\n");
if (info->domain_attr != NULL) {
switch (info->domain_attr->mr_mode) {
case FI_MR_UNSPEC:
case FI_MR_BASIC:
break;
default:
/* the caller ignored our fi_getinfo results */
USDF_WARN_SYS(DOMAIN, "MR mode (%d) not supported\n",
info->domain_attr->mr_mode);
return -FI_ENODATA;
}
}
udp = calloc(1, sizeof *udp);
if (udp == NULL) {
USDF_DBG("unable to alloc mem for domain\n");
ret = -FI_ENOMEM;
goto fail;
}
fp = fab_fidtou(fabric);
USDF_DBG("uda_devname=%s\n", fp->fab_dev_attrs->uda_devname);
/*
* Make sure address format is good and matches this fabric
*/
switch (info->addr_format) {
case FI_SOCKADDR:
addrlen = sizeof(struct sockaddr);
break;
case FI_SOCKADDR_IN:
addrlen = sizeof(struct sockaddr_in);
break;
default:
ret = -FI_EINVAL;
goto fail;
}
sin = info->src_addr;
if (info->src_addrlen != addrlen || sin->sin_family != AF_INET ||
sin->sin_addr.s_addr != fp->fab_dev_attrs->uda_ipaddr_be) {
USDF_DBG_SYS(DOMAIN, "requested src_addr (%s) != fabric addr (%s)\n",
inet_ntop(AF_INET, &sin->sin_addr.s_addr,
requested, sizeof(requested)),
inet_ntop(AF_INET, &fp->fab_dev_attrs->uda_ipaddr_be,
actual, sizeof(actual)));
ret = -FI_EINVAL;
goto fail;
}
ret = usd_open(fp->fab_dev_attrs->uda_devname, &udp->dom_dev);
if (ret != 0) {
goto fail;
}
udp->dom_fid.fid.fclass = FI_CLASS_DOMAIN;
udp->dom_fid.fid.context = context;
udp->dom_fid.fid.ops = &usdf_fid_ops;
udp->dom_fid.ops = &usdf_domain_ops;
udp->dom_fid.mr = &usdf_domain_mr_ops;
ret = pthread_spin_init(&udp->dom_progress_lock,
PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&udp->dom_tx_ready);
TAILQ_INIT(&udp->dom_hcq_list);
udp->dom_info = fi_dupinfo(info);
if (udp->dom_info == NULL) {
ret = -FI_ENOMEM;
goto fail;
}
if (udp->dom_info->dest_addr != NULL) {
free(udp->dom_info->dest_addr);
udp->dom_info->dest_addr = NULL;
}
ret = usdf_dom_rdc_alloc_data(udp);
if (ret != 0) {
goto fail;
}
udp->dom_fabric = fp;
LIST_INSERT_HEAD(&fp->fab_domain_list, udp, dom_link);
atomic_initialize(&udp->dom_refcnt, 0);
atomic_inc(&fp->fab_refcnt);
*domain = &udp->dom_fid;
return 0;
fail:
if (udp != NULL) {
if (udp->dom_info != NULL) {
fi_freeinfo(udp->dom_info);
}
if (udp->dom_dev != NULL) {
usd_close(udp->dom_dev);
}
usdf_dom_rdc_free_data(udp);
free(udp);
}
return ret;
}
int psmx2_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
struct fid_cntr **cntr, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_cntr *cntr_priv;
struct fid_wait *wait = NULL;
struct fi_wait_attr wait_attr;
int wait_is_local = 0;
int events;
uint64_t flags;
int err;
events = FI_CNTR_EVENTS_COMP;
flags = 0;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid);
switch (attr->events) {
case FI_CNTR_EVENTS_COMP:
events = attr->events;
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CQ,
"attr->events=%d, supported=%d\n",
attr->events, FI_CNTR_EVENTS_COMP);
return -FI_EINVAL;
}
switch (attr->wait_obj) {
case FI_WAIT_NONE:
break;
case FI_WAIT_SET:
if (!attr->wait_set) {
FI_INFO(&psmx2_prov, FI_LOG_CQ,
"FI_WAIT_SET is specified but attr->wait_set is NULL\n");
return -FI_EINVAL;
}
wait = attr->wait_set;
break;
case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
case FI_WAIT_MUTEX_COND:
wait_attr.wait_obj = attr->wait_obj;
wait_attr.flags = 0;
err = fi_wait_open(&domain_priv->fabric->util_fabric.fabric_fid,
&wait_attr, (struct fid_wait **)&wait);
if (err)
return err;
wait_is_local = 1;
break;
default:
FI_INFO(&psmx2_prov, FI_LOG_CQ,
"attr->wait_obj=%d, supported=%d...%d\n",
attr->wait_obj, FI_WAIT_NONE, FI_WAIT_MUTEX_COND);
return -FI_EINVAL;
}
cntr_priv = (struct psmx2_fid_cntr *) calloc(1, sizeof *cntr_priv);
if (!cntr_priv) {
err = -FI_ENOMEM;
goto fail;
}
cntr_priv->domain = domain_priv;
cntr_priv->events = events;
if (wait)
cntr_priv->wait = container_of(wait, struct util_wait, wait_fid);
cntr_priv->wait_is_local = wait_is_local;
cntr_priv->flags = flags;
cntr_priv->cntr.fid.fclass = FI_CLASS_CNTR;
cntr_priv->cntr.fid.context = context;
cntr_priv->cntr.fid.ops = &psmx2_fi_ops;
cntr_priv->cntr.ops = &psmx2_cntr_ops;
atomic_initialize(&cntr_priv->counter, 0);
atomic_initialize(&cntr_priv->error_counter, 0);
pthread_mutex_init(&cntr_priv->trigger_lock, NULL);
if (wait)
fi_poll_add(&cntr_priv->wait->pollset->poll_fid,
&cntr_priv->cntr.fid, 0);
psmx2_domain_acquire(domain_priv);
*cntr = &cntr_priv->cntr;
return 0;
fail:
if (wait && wait_is_local)
fi_close(&wait->fid);
return err;
}
int sock_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr,
struct fid_cq **cq, void *context)
{
struct sock_domain *sock_dom;
struct sock_cq *sock_cq;
struct fi_wait_attr wait_attr;
struct sock_fid_list *list_entry;
struct sock_wait *wait;
int ret;
sock_dom = container_of(domain, struct sock_domain, dom_fid);
ret = sock_cq_verify_attr(attr);
if (ret)
return ret;
sock_cq = calloc(1, sizeof(*sock_cq));
if (!sock_cq)
return -FI_ENOMEM;
atomic_initialize(&sock_cq->ref, 0);
sock_cq->cq_fid.fid.fclass = FI_CLASS_CQ;
sock_cq->cq_fid.fid.context = context;
sock_cq->cq_fid.fid.ops = &sock_cq_fi_ops;
sock_cq->cq_fid.ops = &sock_cq_ops;
if (attr == NULL)
sock_cq->attr = _sock_cq_def_attr;
else {
sock_cq->attr = *attr;
if (attr->size == 0)
sock_cq->attr.size = _sock_cq_def_attr.size;
}
sock_cq->domain = sock_dom;
sock_cq->cq_entry_size = sock_cq_entry_size(sock_cq);
sock_cq_set_report_fn(sock_cq);
dlist_init(&sock_cq->tx_list);
dlist_init(&sock_cq->rx_list);
dlist_init(&sock_cq->ep_list);
dlist_init(&sock_cq->overflow_list);
if ((ret = rbfdinit(&sock_cq->cq_rbfd, sock_cq->attr.size *
sock_cq->cq_entry_size)))
goto err1;
if ((ret = rbinit(&sock_cq->addr_rb,
sock_cq->attr.size * sizeof(fi_addr_t))))
goto err2;
if ((ret = rbinit(&sock_cq->cqerr_rb, sock_cq->attr.size *
sizeof(struct fi_cq_err_entry))))
goto err3;
fastlock_init(&sock_cq->lock);
switch (sock_cq->attr.wait_obj) {
case FI_WAIT_NONE:
case FI_WAIT_UNSPEC:
case FI_WAIT_FD:
break;
case FI_WAIT_MUTEX_COND:
wait_attr.flags = 0;
wait_attr.wait_obj = FI_WAIT_MUTEX_COND;
ret = sock_wait_open(&sock_dom->fab->fab_fid, &wait_attr,
&sock_cq->waitset);
if (ret) {
ret = -FI_EINVAL;
goto err4;
}
sock_cq->signal = 1;
break;
case FI_WAIT_SET:
if (!attr) {
ret = -FI_EINVAL;
goto err4;
}
sock_cq->waitset = attr->wait_set;
sock_cq->signal = 1;
wait = container_of(attr->wait_set, struct sock_wait, wait_fid);
list_entry = calloc(1, sizeof(*list_entry));
dlist_init(&list_entry->entry);
list_entry->fid = &sock_cq->cq_fid.fid;
dlist_insert_after(&list_entry->entry, &wait->fid_list);
break;
default:
break;
}
*cq = &sock_cq->cq_fid;
atomic_inc(&sock_dom->ref);
fastlock_init(&sock_cq->list_lock);
return 0;
err4:
rbfree(&sock_cq->cqerr_rb);
err3:
rbfree(&sock_cq->addr_rb);
err2:
rbfdfree(&sock_cq->cq_rbfd);
err1:
free(sock_cq);
return ret;
}
int
usdf_pep_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_pep **pep_o, void *context)
{
struct usdf_pep *pep;
struct usdf_fabric *fp;
int ret;
int optval;
USDF_TRACE_SYS(EP_CTRL, "\n");
if (info->ep_attr->type != FI_EP_MSG) {
return -FI_ENODEV;
}
if ((info->caps & ~USDF_MSG_CAPS) != 0) {
return -FI_EBADF;
}
fp = fab_ftou(fabric);
pep = calloc(1, sizeof(*pep));
if (pep == NULL) {
return -FI_ENOMEM;
}
pep->pep_fid.fid.fclass = FI_CLASS_PEP;
pep->pep_fid.fid.context = context;
pep->pep_fid.fid.ops = &usdf_pep_ops;
pep->pep_fid.ops = &usdf_pep_base_ops;
pep->pep_fid.cm = &usdf_pep_cm_ops;
pep->pep_fabric = fp;
pep->pep_sock = socket(AF_INET, SOCK_STREAM, 0);
if (pep->pep_sock == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_GETFL, 0);
if (ret == -1) {
ret = -errno;
goto fail;
}
ret = fcntl(pep->pep_sock, F_SETFL, ret | O_NONBLOCK);
if (ret == -1) {
ret = -errno;
goto fail;
}
/* set SO_REUSEADDR to prevent annoying "Address already in use" errors
* on successive runs of programs listening on a well known port */
optval = 1;
ret = setsockopt(pep->pep_sock, SOL_SOCKET, SO_REUSEADDR, &optval,
sizeof(optval));
if (ret == -1) {
ret = -errno;
goto fail;
}
ret = bind(pep->pep_sock, (struct sockaddr *)info->src_addr,
info->src_addrlen);
if (ret == -1) {
ret = -errno;
goto fail;
}
/* initialize connreq freelist */
ret = pthread_spin_init(&pep->pep_cr_lock, PTHREAD_PROCESS_PRIVATE);
if (ret != 0) {
ret = -ret;
goto fail;
}
TAILQ_INIT(&pep->pep_cr_free);
TAILQ_INIT(&pep->pep_cr_pending);
pep->pep_backlog = 10;
ret = usdf_pep_grow_backlog(pep);
if (ret != 0) {
goto fail;
}
atomic_initialize(&pep->pep_refcnt, 0);
atomic_inc(&fp->fab_refcnt);
*pep_o = pep_utof(pep);
return 0;
fail:
if (pep != NULL) {
usdf_pep_free_cr_lists(pep);
if (pep->pep_sock != -1) {
close(pep->pep_sock);
}
free(pep);
}
return ret;
}
