static ssize_t psmx2_tagged_peek_generic(struct fid_ep *ep,
void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_cq_event *event;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t req;
psm2_mq_status2_t psm2_status;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr);
vlane = 0;
} else if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = -1;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~PSMX2_SRC_BITS;
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
if (flags & (FI_CLAIM | FI_DISCARD))
err = psm2_mq_improbe2(ep_priv->trx_ctxt->psm2_mq,
psm2_epaddr, &psm2_tag,
&psm2_tagsel, &req, &psm2_status);
else
err = psm2_mq_iprobe2(ep_priv->trx_ctxt->psm2_mq,
psm2_epaddr, &psm2_tag, &psm2_tagsel,
&psm2_status);
switch (err) {
case PSM2_OK:
if (ep_priv->recv_cq) {
if (flags & FI_CLAIM) {
if (context)
PSMX2_CTXT_REQ((struct fi_context *)context) = req;
} else if (flags & FI_DISCARD) {
if (!psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0,
NULL, 0, req, &req))
psm2_mq_wait2(&req, NULL);
}
tag = PSMX2_GET_TAG64(psm2_status.msg_tag);
len = psm2_status.msg_length;
event = psmx2_cq_create_event(
ep_priv->recv_cq,
context, /* op_context */
NULL, /* buf */
flags|FI_RECV|FI_TAGGED,/* flags */
len, /* len */
0, /* data */
tag, /* tag */
len, /* olen */
0); /* err */
if (!event)
return -FI_ENOMEM;
vlane = PSMX2_TAG32_GET_SRC(psm2_status.msg_tag.tag2);
event->source_is_valid = 1;
event->source = PSMX2_EP_TO_ADDR(psm2_status.msg_peer, vlane);
event->source_av = ep_priv->av;
psmx2_cq_enqueue_event(ep_priv->recv_cq, event);
}
return 0;
case PSM2_MQ_NO_COMPLETIONS:
if (ep_priv->recv_cq) {
event = psmx2_cq_create_event(
ep_priv->recv_cq,
context, /* op_context */
NULL, /* buf */
flags|FI_RECV|FI_TAGGED,/* flags */
len, /* len */
0, /* data */
tag, /* tag */
len, /* olen */
-FI_ENOMSG); /* err */
if (!event)
return -FI_ENOMEM;
event->source = 0;
psmx2_cq_enqueue_event(ep_priv->recv_cq, event);
}
return 0;
default:
return psmx2_errno(err);
}
}
static int psmx2_av_insert(struct fid_av *av, const void *addr,
size_t count, fi_addr_t *fi_addr,
uint64_t flags, void *context)
{
struct psmx2_fid_av *av_priv;
psm2_epid_t *epids;
uint8_t *vlanes;
psm2_epaddr_t *epaddrs;
psm2_error_t *errors;
int *mask;
const struct psmx2_ep_name *names = addr;
int error_count;
int i;
if (count && !addr) {
FI_INFO(&psmx2_prov, FI_LOG_AV,
"the input address array is NULL.\n");
return -FI_EINVAL;
}
av_priv = container_of(av, struct psmx2_fid_av, av);
if ((av_priv->flags & FI_EVENT) && !av_priv->eq)
return -FI_ENOEQ;
if (psmx2_av_check_table_size(av_priv, count))
return -FI_ENOMEM;
epids = av_priv->epids + av_priv->last;
epaddrs = av_priv->epaddrs + av_priv->last;
vlanes = av_priv->vlanes + av_priv->last;
for (i=0; i<count; i++) {
epids[i] = names[i].epid;
vlanes[i] = names[i].vlane;
}
errors = (psm2_error_t *) calloc(count, sizeof *errors);
mask = (int *) calloc(count, sizeof *mask);
if (!errors || !mask) {
free(mask);
free(errors);
return -FI_ENOMEM;
}
error_count = psmx2_av_connet_eps(av_priv, count, epids, mask,
errors, epaddrs, context);
free(mask);
free(errors);
if (fi_addr) {
for (i=0; i<count; i++) {
if (epaddrs[i] == (void *)FI_ADDR_NOTAVAIL)
fi_addr[i] = FI_ADDR_NOTAVAIL;
else if (av_priv->type == FI_AV_TABLE)
fi_addr[i] = av_priv->last + i;
else
fi_addr[i] = PSMX2_EP_TO_ADDR(epaddrs[i], vlanes[i]);
}
}
if (av_priv->type == FI_AV_TABLE)
av_priv->last += count;
if (!(av_priv->flags & FI_EVENT))
return count - error_count;
psmx2_av_post_completion(av_priv, context, count - error_count, 0);
return 0;
}
ssize_t psmx2_tagged_recv_generic(struct fid_ep *ep, void *buf,
size_t len, void *desc,
fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_PEEK)
return psmx2_tagged_peek_generic(ep, buf, len, desc,
src_addr, tag, ignore,
context, flags);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_TRECV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->trecv.ep = ep;
trigger->trecv.buf = buf;
trigger->trecv.len = len;
trigger->trecv.desc = desc;
trigger->trecv.src_addr = src_addr;
trigger->trecv.tag = tag;
trigger->trecv.ignore = ignore;
trigger->trecv.context = context;
trigger->trecv.flags = flags & ~FI_TRIGGER;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if (flags & FI_CLAIM) {
if (!context)
return -FI_EINVAL;
if (flags & FI_DISCARD) {
psm2_mq_status2_t psm2_status;
struct psmx2_cq_event *event;
fi_context = context;
psm2_req = PSMX2_CTXT_REQ(fi_context);
err = psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0,
NULL, 0, context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
psm2_mq_wait2(&psm2_req, &psm2_status);
if (ep_priv->recv_cq &&
(!ep_priv->recv_selective_completion || (flags & FI_COMPLETION))) {
tag = PSMX2_GET_TAG64(psm2_status.msg_tag);
event = psmx2_cq_create_event(
ep_priv->recv_cq,
context, /* op_context */
NULL, /* buf */
flags|FI_RECV|FI_TAGGED,/* flags */
0, /* len */
0, /* data */
tag, /* tag */
0, /* olen */
0); /* err */
if (!event)
return -FI_ENOMEM;
vlane = PSMX2_TAG32_GET_SRC(psm2_status.msg_tag.tag2);
event->source_is_valid = 1;
event->source = PSMX2_EP_TO_ADDR(psm2_status.msg_peer, vlane);
event->source_av = ep_priv->av;
psmx2_cq_enqueue_event(ep_priv->recv_cq, event);
}
if (ep_priv->recv_cntr)
psmx2_cntr_inc(ep_priv->recv_cntr);
return 0;
}
fi_context = context;
psm2_req = PSMX2_CTXT_REQ(fi_context);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
err = psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0,
buf, len, context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION)) {
fi_context = psmx2_ep_get_op_context(ep_priv);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
} else {
if (!context)
return -FI_EINVAL;
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_SIZE(fi_context) = len;
}
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr);
vlane = 0;
} else if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = ~PSMX2_IOV_BIT;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
err = psm2_mq_irecv2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (fi_context == context)
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
