static int map_range_append_(
struct dlist_entry_t *ranges,
struct map_t_ *maps,
const size_t map_count,
size_t *i)
{
size_t idx = *i;
if (idx >= map_count) {
return 0;
}
uint32_t from_code_max = 0;
uint32_t to_code_max = 0;
struct map_range_t_ range = {
.start_map_idx = idx,
.end_map_idx = idx,
.type = map_type_(&maps[idx])
};
dlist_init(&range.list);
do {
if (from_code_max < maps[idx].from) {
from_code_max = maps[idx].from;
}
if (to_code_max < maps[idx].to) {
to_code_max = maps[idx].to;
}
range.end_map_idx = idx++;
} while (
idx < map_count &&
maps[range.end_map_idx].from + 1 == maps[idx].from &&
map_type_(&maps[idx]) == range.type);
range.end_map_idx = idx;
struct map_range_t_ *r = malloc(sizeof(*r));
if (r == NULL) {
return -1;
}
*r = range;
r->from_code_width = map_code_width_(from_code_max);
r->to_code_width = map_code_width_(to_code_max);
dlist_insert_before(ranges, &r->list);
*i = idx;
return 1;
}
int psmx2_stx_ctx(struct fid_domain *domain, struct fi_tx_attr *attr,
struct fid_stx **stx, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_trx_ctxt *trx_ctxt;
struct psmx2_fid_stx *stx_priv;
int err = -FI_EINVAL;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
stx_priv = (struct psmx2_fid_stx *) calloc(1, sizeof *stx_priv);
if (!stx_priv) {
err = -FI_ENOMEM;
goto errout;
}
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, NULL/*src_addr*/, 0);
if (!trx_ctxt) {
err = -FI_ENOMEM;
goto errout_free_stx;
}
psmx2_domain_acquire(domain_priv);
stx_priv->stx.fid.fclass = FI_CLASS_STX_CTX;
stx_priv->stx.fid.context = context;
stx_priv->stx.fid.ops = &psmx2_fi_ops_stx;
stx_priv->stx.ops = &psmx2_stx_ops;
stx_priv->domain = domain_priv;
stx_priv->tx = trx_ctxt;
ofi_atomic_initialize32(&stx_priv->ref, 0);
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
dlist_insert_before(&trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
*stx = &stx_priv->stx;
return 0;
errout_free_stx:
free(stx_priv);
errout:
return err;
}
int psmx2_sep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **sep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_sep *sep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt;
size_t ctxt_cnt = 1;
size_t ctxt_size;
int err = -FI_EINVAL;
int i;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr) {
if (info->ep_attr->tx_ctx_cnt > psmx2_env.sep_trx_ctxt) {
FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL,
"tx_ctx_cnt %"PRIu64" exceed limit %d.\n",
info->ep_attr->tx_ctx_cnt,
psmx2_env.sep_trx_ctxt);
goto errout;
}
if (info->ep_attr->rx_ctx_cnt > psmx2_env.sep_trx_ctxt) {
FI_WARN(&psmx2_prov, FI_LOG_EP_CTRL,
"rx_ctx_cnt %"PRIu64" exceed limit %d.\n",
info->ep_attr->rx_ctx_cnt,
psmx2_env.sep_trx_ctxt);
goto errout;
}
ctxt_cnt = info->ep_attr->tx_ctx_cnt;
if (ctxt_cnt < info->ep_attr->rx_ctx_cnt)
ctxt_cnt = info->ep_attr->rx_ctx_cnt;
if (ctxt_cnt == 0) {
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL,
"tx_ctx_cnt and rx_ctx_cnt are 0, use 1.\n");
ctxt_cnt = 1;
}
}
ctxt_size = ctxt_cnt * sizeof(struct psmx2_sep_ctxt);
sep_priv = (struct psmx2_fid_sep *) calloc(1, sizeof(*sep_priv) + ctxt_size);
if (!sep_priv) {
err = -FI_ENOMEM;
goto errout;
}
sep_priv->ep.fid.fclass = FI_CLASS_SEP;
sep_priv->ep.fid.context = context;
sep_priv->ep.fid.ops = &psmx2_fi_ops_sep;
sep_priv->ep.ops = &psmx2_sep_ops;
sep_priv->ep.cm = &psmx2_cm_ops;
sep_priv->domain = domain_priv;
sep_priv->ctxt_cnt = ctxt_cnt;
ofi_atomic_initialize32(&sep_priv->ref, 0);
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
for (i = 0; i < ctxt_cnt; i++) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, i);
if (!trx_ctxt) {
err = -FI_ENOMEM;
goto errout_free_ctxt;
}
sep_priv->ctxts[i].trx_ctxt = trx_ctxt;
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
/* override the ops so the fid can't be closed individually */
ep_priv->ep.fid.ops = &psmx2_fi_ops_sep_ctxt;
trx_ctxt->ep = ep_priv;
sep_priv->ctxts[i].ep = ep_priv;
}
sep_priv->type = PSMX2_EP_SCALABLE;
sep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
sep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (sep_priv->service == PSMX2_ANY_SERVICE)
sep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)sep_priv & 0xFFFF);
sep_priv->id = ofi_atomic_inc32(&domain_priv->sep_cnt);
psmx2_lock(&domain_priv->sep_lock, 1);
dlist_insert_before(&sep_priv->entry, &domain_priv->sep_list);
psmx2_unlock(&domain_priv->sep_lock, 1);
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
for (i = 0; i< ctxt_cnt; i++) {
dlist_insert_before(&sep_priv->ctxts[i].trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
}
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = sep_priv->ctxts[0].trx_ctxt->psm2_epid;
ep_name.sep_id = sep_priv->id;
ep_name.type = sep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&sep_priv->service, &ep_name);
psmx2_domain_acquire(domain_priv);
*sep = &sep_priv->ep;
/* Make sure the AM handler is installed to answer SEP query */
psmx2_am_init(sep_priv->ctxts[0].trx_ctxt);
return 0;
errout_free_ctxt:
while (i) {
if (sep_priv->ctxts[i].ep)
psmx2_ep_close_internal(sep_priv->ctxts[i].ep);
if (sep_priv->ctxts[i].trx_ctxt)
psmx2_trx_ctxt_free(sep_priv->ctxts[i].trx_ctxt);
i--;
}
free(sep_priv);
errout:
return err;
}
int psmx2_ep_open(struct fid_domain *domain, struct fi_info *info,
struct fid_ep **ep, void *context)
{
struct psmx2_fid_domain *domain_priv;
struct psmx2_fid_ep *ep_priv;
struct psmx2_ep_name ep_name;
struct psmx2_ep_name *src_addr;
struct psmx2_trx_ctxt *trx_ctxt = NULL;
int err = -FI_EINVAL;
int alloc_trx_ctxt = 1;
domain_priv = container_of(domain, struct psmx2_fid_domain,
util_domain.domain_fid.fid);
if (!domain_priv)
goto errout;
if (info && info->ep_attr && info->ep_attr->rx_ctx_cnt == FI_SHARED_CONTEXT)
return -FI_ENOSYS;
if (info && info->ep_attr && info->ep_attr->tx_ctx_cnt == FI_SHARED_CONTEXT &&
!ofi_recv_allowed(info->caps) && !ofi_rma_target_allowed(info->caps)) {
alloc_trx_ctxt = 0;
FI_INFO(&psmx2_prov, FI_LOG_EP_CTRL, "Tx only endpoint with STX context.\n");
}
src_addr = NULL;
if (info && info->src_addr) {
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
else
src_addr = info->src_addr;
}
if (alloc_trx_ctxt) {
trx_ctxt = psmx2_trx_ctxt_alloc(domain_priv, src_addr, 0);
if (!trx_ctxt)
goto errout;
}
err = psmx2_ep_open_internal(domain_priv, info, &ep_priv, context,
trx_ctxt);
if (err)
goto errout_free_ctxt;
ep_priv->type = PSMX2_EP_REGULAR;
ep_priv->service = PSMX2_ANY_SERVICE;
if (src_addr) {
ep_priv->service = src_addr->service;
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
}
if (ep_priv->service == PSMX2_ANY_SERVICE)
ep_priv->service = ((getpid() & 0x7FFF) << 16) +
((uintptr_t)ep_priv & 0xFFFF);
if (alloc_trx_ctxt) {
trx_ctxt->ep = ep_priv;
psmx2_lock(&domain_priv->trx_ctxt_lock, 1);
dlist_insert_before(&trx_ctxt->entry,
&domain_priv->trx_ctxt_list);
psmx2_unlock(&domain_priv->trx_ctxt_lock, 1);
ep_name.epid = trx_ctxt->psm2_epid;
ep_name.type = ep_priv->type;
ofi_ns_add_local_name(&domain_priv->fabric->name_server,
&ep_priv->service, &ep_name);
}
*ep = &ep_priv->ep;
return 0;
errout_free_ctxt:
psmx2_trx_ctxt_free(trx_ctxt);
errout:
return err;
}
static int map_range_merge_(
const struct map_t_ *const maps,
struct dlist_entry_t *clusters,
struct dlist_entry_t *ranges)
{
while (!dlist_is_empty(ranges)) {
struct map_range_t_ *next_range = dlist_entry(
ranges->next, struct map_range_t_, list);
struct map_cluster_t_ *last_cluster = dlist_is_empty(clusters) ?
NULL : dlist_entry(clusters->prev, struct map_cluster_t_, list);
struct map_range_t_ *last_clustered = (last_cluster == NULL) ?
NULL : dlist_entry(
last_cluster->ranges.prev,
struct map_range_t_, list);
if ( dlist_is_empty(clusters) ||
map_range_is_large_equal_(next_range) ||
!map_range_can_be_merged_(maps, last_cluster, next_range))
{
if ((last_cluster = malloc(sizeof(*last_cluster))) == NULL) {
return -1;
}
last_cluster->illegal_code = ILLEGAL_NOT_SET;
last_cluster->illegal_code_width = 0;
dlist_init(&last_cluster->ranges);
dlist_init(&last_cluster->list);
dlist_insert_before(clusters, &last_cluster->list);
}
dlist_remove(&next_range->list);
dlist_insert_before(&last_cluster->ranges, &next_range->list);
if (last_clustered != NULL) {
/* do not decrease code widths with merged range */
if (next_range->to_code_width <
last_clustered->to_code_width)
{
next_range->to_code_width =
last_clustered->to_code_width;
}
if (next_range->from_code_width <
last_clustered->from_code_width)
{
next_range->from_code_width =
last_clustered->from_code_width;
}
}
}
/* find the maximum code width for a whole cluster */
struct map_cluster_t_ *c;
dlist_foreach_entry(c, struct map_cluster_t_, list, clusters) {
int to_code_width = 0;
struct map_range_t_ *r;
dlist_foreach_entry(r, struct map_range_t_, list, &c->ranges) {
if (to_code_width < r->to_code_width) {
to_code_width = r->to_code_width;
}
}
dlist_foreach_entry(r, struct map_range_t_, list, &c->ranges) {
r->to_code_width = to_code_width;
}
c->illegal_code_width = to_code_width;
}
int psmx2_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct psmx2_fid_fabric *fabric_priv;
struct psmx2_fid_domain *domain_priv;
struct psmx2_ep_name *src_addr = info->src_addr;
int mr_mode = (info->domain_attr->mr_mode & FI_MR_BASIC) ? FI_MR_BASIC : 0;
int err;
FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "\n");
fabric_priv = container_of(fabric, struct psmx2_fid_fabric,
util_fabric.fabric_fid);
if (!info->domain_attr->name ||
strcmp(info->domain_attr->name, PSMX2_DOMAIN_NAME)) {
err = -FI_EINVAL;
goto err_out;
}
domain_priv = (struct psmx2_fid_domain *) calloc(1, sizeof *domain_priv);
if (!domain_priv) {
err = -FI_ENOMEM;
goto err_out;
}
err = ofi_domain_init(fabric, info, &domain_priv->util_domain, context);
if (err)
goto err_out_free_domain;
/* fclass & context are set in ofi_domain_init */
domain_priv->util_domain.domain_fid.fid.ops = &psmx2_fi_ops;
domain_priv->util_domain.domain_fid.ops = &psmx2_domain_ops;
domain_priv->util_domain.domain_fid.mr = &psmx2_mr_ops;
domain_priv->mr_mode = mr_mode;
domain_priv->mode = info->mode;
domain_priv->caps = PSMX2_CAPS | PSMX2_DOM_CAPS;
domain_priv->fabric = fabric_priv;
domain_priv->progress_thread_enabled =
(info->domain_attr->data_progress == FI_PROGRESS_AUTO);
domain_priv->addr_format = info->addr_format;
if (info->addr_format == FI_ADDR_STR)
src_addr = psmx2_string_to_ep_name(info->src_addr);
err = psmx2_domain_init(domain_priv, src_addr);
if (info->addr_format == FI_ADDR_STR)
free(src_addr);
if (err)
goto err_out_close_domain;
psmx2_fabric_acquire(fabric_priv);
psmx2_lock(&fabric_priv->domain_lock, 1);
dlist_insert_before(&domain_priv->entry, &fabric_priv->domain_list);
psmx2_unlock(&fabric_priv->domain_lock, 1);
*domain = &domain_priv->util_domain.domain_fid;
return 0;
err_out_close_domain:
ofi_domain_close(&domain_priv->util_domain);
err_out_free_domain:
free(domain_priv);
err_out:
return err;
}
static int psmx2_domain_init(struct psmx2_fid_domain *domain,
struct psmx2_src_name *src_addr)
{
int err;
psmx2_am_global_init();
psmx2_atomic_global_init();
domain->base_trx_ctxt = psmx2_trx_ctxt_alloc(domain, src_addr, -1);
if (!domain->base_trx_ctxt)
return -FI_ENODEV;
err = fastlock_init(&domain->mr_lock);
if (err) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"fastlock_init(mr_lock) returns %d\n", err);
goto err_out_free_trx_ctxt;
}
domain->mr_map = rbtNew(&psmx2_key_compare);
if (!domain->mr_map) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"rbtNew failed\n");
goto err_out_destroy_mr_lock;
}
domain->mr_reserved_key = 1;
err = fastlock_init(&domain->vl_lock);
if (err) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"fastlock_init(vl_lock) returns %d\n", err);
goto err_out_delete_mr_map;
}
memset(domain->vl_map, 0, sizeof(domain->vl_map));
domain->vl_alloc = 0;
ofi_atomic_initialize32(&domain->sep_cnt, 0);
fastlock_init(&domain->sep_lock);
dlist_init(&domain->sep_list);
dlist_init(&domain->trx_ctxt_list);
fastlock_init(&domain->trx_ctxt_lock);
dlist_insert_before(&domain->base_trx_ctxt->entry, &domain->trx_ctxt_list);
/* Set active domain before psmx2_domain_enable_ep() installs the
* AM handlers to ensure that psmx2_active_fabric->active_domain
* is always non-NULL inside the handlers. Notice that the vlaue
* active_domain becomes NULL again only when the domain is closed.
* At that time the AM handlers are gone with the PSM endpoint.
*/
domain->fabric->active_domain = domain;
if (psmx2_domain_enable_ep(domain, NULL) < 0)
goto err_out_reset_active_domain;
if (domain->progress_thread_enabled)
psmx2_domain_start_progress(domain);
psmx2_am_init(domain->base_trx_ctxt);
return 0;
err_out_reset_active_domain:
domain->fabric->active_domain = NULL;
fastlock_destroy(&domain->vl_lock);
err_out_delete_mr_map:
rbtDelete(domain->mr_map);
err_out_destroy_mr_lock:
fastlock_destroy(&domain->mr_lock);
err_out_free_trx_ctxt:
psmx2_trx_ctxt_free(domain->base_trx_ctxt);
return err;
}
static int __gnix_vc_hndl_conn_req(struct gnix_cm_nic *cm_nic,
char *msg_buffer,
struct gnix_address src_cm_nic_addr)
{
int ret = FI_SUCCESS;
gni_return_t __attribute__((unused)) status;
struct gnix_fid_ep *ep = NULL;
gnix_ht_key_t *key_ptr;
struct gnix_av_addr_entry entry;
struct gnix_address src_addr, target_addr;
struct gnix_vc *vc = NULL;
struct gnix_vc *vc_try = NULL;
struct gnix_work_req *work_req;
int src_vc_id;
gni_smsg_attr_t src_smsg_attr;
uint64_t src_vc_ptr;
struct wq_hndl_conn_req *data = NULL;
ssize_t __attribute__((unused)) len;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
/*
* unpack the message
*/
__gnix_vc_unpack_conn_req(msg_buffer,
&target_addr,
&src_addr,
&src_vc_id,
&src_vc_ptr,
&src_smsg_attr);
GNIX_DEBUG(FI_LOG_EP_CTRL,
"conn req rx: (From Aries addr 0x%x Id %d to Aries 0x%x Id %d src vc 0x%lx )\n",
src_addr.device_addr,
src_addr.cdm_id,
target_addr.device_addr,
target_addr.cdm_id,
src_vc_ptr);
/*
* lookup the ep from the addr_to_ep_ht using the target_addr
* in the datagram
*/
key_ptr = (gnix_ht_key_t *)&target_addr;
ep = (struct gnix_fid_ep *)_gnix_ht_lookup(cm_nic->addr_to_ep_ht,
*key_ptr);
if (ep == NULL) {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_ht_lookup addr_to_ep failed\n");
ret = -FI_ENOENT;
goto err;
}
/*
* look to see if there is a VC already for the
* address of the connecting EP.
*/
key_ptr = (gnix_ht_key_t *)&src_addr;
fastlock_acquire(&ep->vc_ht_lock);
vc = (struct gnix_vc *)_gnix_ht_lookup(ep->vc_ht,
*key_ptr);
/*
* if there is no corresponding vc in the hash,
* or there is an entry and its not in connecting state
* go down the conn req ack route.
*/
if ((vc == NULL) ||
(vc->conn_state == GNIX_VC_CONN_NONE)) {
if (vc == NULL) {
entry.gnix_addr = src_addr;
entry.cm_nic_cdm_id = src_cm_nic_addr.cdm_id;
ret = _gnix_vc_alloc(ep,
&entry,
&vc_try);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_vc_alloc returned %s\n",
fi_strerror(-ret));
goto err;
}
vc_try->conn_state = GNIX_VC_CONNECTING;
ret = _gnix_ht_insert(ep->vc_ht,
*key_ptr,
vc_try);
if (likely(ret == FI_SUCCESS)) {
vc = vc_try;
vc->modes |= GNIX_VC_MODE_IN_HT;
} else if (ret == -FI_ENOSPC) {
_gnix_vc_destroy(vc_try);
} else {
GNIX_WARN(FI_LOG_EP_DATA,
"_gnix_ht_insert returned %s\n",
fi_strerror(-ret));
goto err;
}
} else
vc->conn_state = GNIX_VC_CONNECTING;
/*
* prepare a work request to
* initiate an request response
*/
work_req = calloc(1, sizeof(*work_req));
if (work_req == NULL) {
ret = -FI_ENOMEM;
goto err;
}
data = calloc(1, sizeof(struct wq_hndl_conn_req));
if (data == NULL) {
ret = -FI_ENOMEM;
goto err;
}
memcpy(&data->src_smsg_attr,
&src_smsg_attr,
sizeof(src_smsg_attr));
data->vc = vc;
data->src_vc_id = src_vc_id;
data->src_vc_ptr = src_vc_ptr;
work_req->progress_fn = __gnix_vc_conn_ack_prog_fn;
work_req->data = data;
work_req->completer_fn = __gnix_vc_conn_ack_comp_fn;
work_req->completer_data = data;
/*
* add the work request to the tail of the
* cm_nic's work queue, progress the cm_nic.
*/
fastlock_acquire(&cm_nic->wq_lock);
dlist_insert_before(&work_req->list, &cm_nic->cm_nic_wq);
fastlock_release(&cm_nic->wq_lock);
fastlock_release(&ep->vc_ht_lock);
_gnix_vc_schedule(vc);
ret = _gnix_cm_nic_progress(cm_nic);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_cm_nic_progress returned %s\n",
fi_strerror(-ret));
} else {
/*
* we can only be in connecting state if we
* reach here. We have all the informatinon,
* and the other side will get the information
* at some point, so go ahead and build SMSG connection.
*/
if (vc->conn_state != GNIX_VC_CONNECTING) {
GNIX_WARN(FI_LOG_EP_CTRL,
"vc %p not in connecting state nor in cm wq\n",
vc, vc->conn_state);
ret = -FI_EINVAL;
goto err;
}
ret = __gnix_vc_smsg_init(vc, src_vc_id,
&src_smsg_attr);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_vc_smsg_init returned %s\n",
fi_strerror(-ret));
goto err;
}
vc->conn_state = GNIX_VC_CONNECTED;
GNIX_DEBUG(FI_LOG_EP_CTRL, "moving vc %p state to connected\n",
vc);
fastlock_release(&ep->vc_ht_lock);
ret = _gnix_vc_schedule(vc);
ret = _gnix_cm_nic_progress(cm_nic);
if (ret != FI_SUCCESS)
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_cm_nic_progress returned %s\n",
fi_strerror(-ret));
}
err:
return ret;
}
int _gnix_vc_connect(struct gnix_vc *vc)
{
int ret = FI_SUCCESS;
struct gnix_fid_ep *ep = NULL;
struct gnix_cm_nic *cm_nic = NULL;
struct gnix_work_req *work_req;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
/*
* can happen that we are already connecting, or
* are connected
*/
if ((vc->conn_state == GNIX_VC_CONNECTING) ||
(vc->conn_state == GNIX_VC_CONNECTED)) {
return FI_SUCCESS;
}
ep = vc->ep;
if (ep == NULL)
return -FI_EINVAL;
cm_nic = ep->cm_nic;
if (cm_nic == NULL)
return -FI_EINVAL;
/*
* only endpoints of type FI_EP_RDM use this
* connection method
*/
if (ep->type != FI_EP_RDM)
return -FI_EINVAL;
/*
* have to do something special for
* connect to same cm_nic
*/
if (!memcmp(&vc->peer_cm_nic_addr,
&cm_nic->my_name.gnix_addr,
sizeof(struct gnix_address))) {
return __gnix_vc_connect_to_same_cm_nic(vc);
}
/*
* allocate a work request and try to
* run the progress function once. If it
* doesn't succeed, put it on the cm_nic work queue.
*/
work_req = calloc(1, sizeof(*work_req));
if (work_req == NULL)
return -FI_ENOMEM;
work_req->progress_fn = __gnix_vc_conn_req_prog_fn;
work_req->data = vc;
work_req->completer_fn = __gnix_vc_conn_req_comp_fn;
work_req->completer_data = vc;
/*
* add the work request to the tail of the
* cm_nic's work queue, progress the cm_nic.
*/
fastlock_acquire(&cm_nic->wq_lock);
dlist_insert_before(&work_req->list, &cm_nic->cm_nic_wq);
fastlock_release(&cm_nic->wq_lock);
ret = _gnix_vc_schedule(vc);
return ret;
}
int _gnix_cm_nic_send(struct gnix_cm_nic *cm_nic,
char *sbuf, size_t len,
struct gnix_address target_addr)
{
int ret = FI_SUCCESS;
struct gnix_datagram *dgram = NULL;
ssize_t __attribute__((unused)) plen;
uint8_t tag;
struct gnix_work_req *work_req;
GNIX_TRACE(FI_LOG_EP_CTRL, "\n");
if ((cm_nic == NULL) || (sbuf == NULL))
return -FI_EINVAL;
if (len > GNI_DATAGRAM_MAXSIZE)
return -FI_ENOSPC;
ret = _gnix_dgram_alloc(cm_nic->dgram_hndl,
GNIX_DGRAM_BND,
&dgram);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_dgram_alloc returned %s\n",
fi_strerror(-ret));
goto exit;
}
dgram->target_addr = target_addr;
dgram->callback_fn = __process_datagram;
dgram->cache = cm_nic;
tag = GNIX_CM_NIC_BND_TAG;
__dgram_set_tag(dgram, tag);
plen = _gnix_dgram_pack_buf(dgram, GNIX_DGRAM_IN_BUF,
sbuf, len);
assert (plen == len);
/* If connecting with the same CM NIC, skip datagram exchange. The
* caller could be holding an endpoint lock, so schedule connection
* completion for later. */
if (GNIX_ADDR_EQUAL(target_addr, cm_nic->my_name.gnix_addr)) {
char tmp_buf[GNIX_CM_NIC_MAX_MSG_SIZE];
/* Pack output buffer with input data. */
_gnix_dgram_unpack_buf(dgram, GNIX_DGRAM_IN_BUF, tmp_buf,
GNIX_CM_NIC_MAX_MSG_SIZE);
_gnix_dgram_pack_buf(dgram, GNIX_DGRAM_OUT_BUF, tmp_buf,
GNIX_CM_NIC_MAX_MSG_SIZE);
work_req = calloc(1, sizeof(*work_req));
if (work_req == NULL) {
_gnix_dgram_free(dgram);
return -FI_ENOMEM;
}
work_req->progress_fn = __gnix_cm_nic_intra_progress_fn;
work_req->data = dgram;
work_req->completer_fn = NULL;
fastlock_acquire(&cm_nic->wq_lock);
dlist_insert_before(&work_req->list, &cm_nic->cm_nic_wq);
fastlock_release(&cm_nic->wq_lock);
GNIX_INFO(FI_LOG_EP_CTRL, "Initiated intra-CM NIC connect\n");
} else {
ret = _gnix_dgram_bnd_post(dgram);
if (ret == -FI_EBUSY) {
ret = -FI_EAGAIN;
_gnix_dgram_free(dgram);
}
}
exit:
return ret;
}
int _gnix_cm_nic_progress(void *arg)
{
struct gnix_cm_nic *cm_nic = (struct gnix_cm_nic *)arg;
int ret = FI_SUCCESS;
int complete;
struct gnix_work_req *p = NULL;
/*
* if we're doing FI_PROGRESS_MANUAL,
* see what's going on inside kgni's datagram
* box...
*/
if (cm_nic->ctrl_progress == FI_PROGRESS_MANUAL) {
++cm_nic->poll_cnt;
if (((cm_nic->poll_cnt % 512) == 0) ||
!dlist_empty(&cm_nic->cm_nic_wq)) {
ret = _gnix_dgram_poll(cm_nic->dgram_hndl,
GNIX_DGRAM_NOBLOCK);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"_gnix_dgram_poll returned %s\n",
fi_strerror(-ret));
goto err;
}
}
}
/*
* do a quick check if queue doesn't have anything yet,
* don't need this to be atomic
*/
check_again:
if (dlist_empty(&cm_nic->cm_nic_wq))
return ret;
/*
* okay, stuff to do, lock work queue,
* dequeue head, unlock, process work element,
* if it doesn't compete, put back at the tail
* of the queue.
*/
fastlock_acquire(&cm_nic->wq_lock);
p = dlist_first_entry(&cm_nic->cm_nic_wq, struct gnix_work_req,
list);
if (p == NULL) {
fastlock_release(&cm_nic->wq_lock);
return ret;
}
dlist_remove_init(&p->list);
fastlock_release(&cm_nic->wq_lock);
assert(p->progress_fn);
ret = p->progress_fn(p->data, &complete);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"dgram prog fn returned %s\n",
fi_strerror(-ret));
}
if (complete == 1) {
if (p->completer_fn) {
ret = p->completer_fn(p->completer_data);
free(p);
if (ret != FI_SUCCESS) {
GNIX_WARN(FI_LOG_EP_CTRL,
"dgram completer fn returned %s\n",
fi_strerror(-ret));
goto err;
}
} else {
free(p);
}
goto check_again;
} else {
fastlock_acquire(&cm_nic->wq_lock);
dlist_insert_before(&p->list, &cm_nic->cm_nic_wq);
fastlock_release(&cm_nic->wq_lock);
}
err:
return ret;
}
