static ssize_t psmx_ep_cancel(fid_t fid, void *context)
{
struct psmx_fid_ep *ep;
psm_mq_status_t status;
struct fi_context *fi_context = context;
uint64_t flags;
struct psmx_cq_event *event;
int err;
ep = container_of(fid, struct psmx_fid_ep, ep.fid);
if (!ep->domain)
return -FI_EBADF;
if (!fi_context)
return -FI_EINVAL;
switch (PSMX_CTXT_TYPE(fi_context)) {
case PSMX_TRECV_CONTEXT:
flags = FI_RECV | FI_TAGGED;
break;
case PSMX_RECV_CONTEXT:
case PSMX_MULTI_RECV_CONTEXT:
flags = FI_RECV | FI_MSG;
break;
default:
return -FI_EOPNOTSUPP;
}
err = psm_mq_cancel((psm_mq_req_t *)&PSMX_CTXT_REQ(fi_context));
if (err == PSM_OK) {
err = psm_mq_test((psm_mq_req_t *)&PSMX_CTXT_REQ(fi_context), &status);
if (err == PSM_OK && ep->recv_cq) {
event = psmx_cq_create_event(
ep->recv_cq,
status.context,
NULL, /* buf */
flags,
0, /* len */
0, /* data */
0, /* tag */
0 /* olen */,
-FI_ECANCELED);
if (event)
psmx_cq_enqueue_event(ep->recv_cq, event);
else
return -FI_ENOMEM;
}
}
return psmx_errno(err);
}
ssize_t psmx_tagged_send_no_flag_av_map(struct fid_ep *ep, const void *buf,
size_t len, void *desc,
fi_addr_t dest_addr, uint64_t tag,
void *context)
{
struct psmx_fid_ep *ep_priv;
psm_epaddr_t psm_epaddr;
psm_mq_req_t psm_req;
uint64_t psm_tag;
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
psm_epaddr = (psm_epaddr_t) dest_addr;
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
fi_context = context;
PSMX_CTXT_TYPE(fi_context) = PSMX_TSEND_CONTEXT;
PSMX_CTXT_USER(fi_context) = (void *)buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
err = psm_mq_isend(ep_priv->domain->psm_mq, psm_epaddr, 0,
psm_tag, buf, len, (void*)fi_context, &psm_req);
if (err != PSM_OK)
return psmx_errno(err);
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
ssize_t psmx_tagged_recv_no_flag_av_table(struct fid_ep *ep, void *buf,
size_t len, void *desc,
fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context)
{
struct psmx_fid_ep *ep_priv;
psm_mq_req_t psm_req;
uint64_t psm_tag, psm_tagsel;
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
psm_tagsel = (~ignore) | ep_priv->domain->reserved_tag_bits;
fi_context = context;
PSMX_CTXT_TYPE(fi_context) = PSMX_TRECV_CONTEXT;
PSMX_CTXT_USER(fi_context) = buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
err = psm_mq_irecv(ep_priv->domain->psm_mq,
psm_tag, psm_tagsel, 0, /* flags */
buf, len, (void *)fi_context, &psm_req);
if (err != PSM_OK)
return psmx_errno(err);
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
ssize_t psmx_tagged_recv_no_flag_av_table(struct fid_ep *ep, void *buf,
size_t len, void *desc,
fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context)
{
struct psmx_fid_ep *ep_priv;
psm_mq_req_t psm_req;
uint64_t psm_tag, psm_tagsel;
#if (PSM_VERNO_MAJOR >= 2)
psm_mq_tag_t psm_tag2, psm_tagsel2;
struct psmx_fid_av *av;
psm_epaddr_t psm_epaddr;
size_t idx;
#endif
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
psm_tagsel = (~ignore) | ep_priv->domain->reserved_tag_bits;
fi_context = context;
PSMX_CTXT_TYPE(fi_context) = PSMX_TRECV_CONTEXT;
PSMX_CTXT_USER(fi_context) = buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
#if (PSM_VERNO_MAJOR >= 2)
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm_epaddr = av->psm_epaddrs[idx];
}
else {
psm_epaddr = NULL;
}
PSMX_SET_TAG(psm_tag2, psm_tag, 0);
PSMX_SET_TAG(psm_tagsel2, psm_tagsel, 0);
err = psm_mq_irecv2(ep_priv->domain->psm_mq,
psm_epaddr,
&psm_tag2, &psm_tagsel2, 0, /* flags */
buf, len, (void *)fi_context, &psm_req);
#else
err = psm_mq_irecv(ep_priv->domain->psm_mq,
psm_tag, psm_tagsel, 0, /* flags */
buf, len, (void *)fi_context, &psm_req);
#endif
if (err != PSM_OK)
return psmx_errno(err);
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
static ssize_t psmx_ep_cancel(fid_t fid, void *context)
{
struct psmx_fid_ep *fid_ep;
psm_mq_status_t status;
struct fi_context *fi_context = context;
int err;
fid_ep = container_of(fid, struct psmx_fid_ep, ep.fid);
if (!fid_ep->domain)
return -EBADF;
if (!fi_context)
return -EINVAL;
err = psm_mq_cancel((psm_mq_req_t *)&PSMX_CTXT_REQ(fi_context));
if (err == PSM_OK)
err = psm_mq_test((psm_mq_req_t *)&PSMX_CTXT_REQ(fi_context), &status);
return psmx_errno(err);
}
ssize_t psmx_tagged_send_no_flag_av_table(struct fid_ep *ep, const void *buf,
size_t len, void *desc,
fi_addr_t dest_addr, uint64_t tag,
void *context)
{
struct psmx_fid_ep *ep_priv;
struct psmx_fid_av *av;
psm_epaddr_t psm_epaddr;
psm_mq_req_t psm_req;
uint64_t psm_tag;
#if (PSM_VERNO_MAJOR >= 2)
psm_mq_tag_t psm_tag2;
#endif
struct fi_context *fi_context;
int err;
size_t idx;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
av = ep_priv->av;
idx = (size_t)dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm_epaddr = av->psm_epaddrs[idx];
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
#if (PSM_VERNO_MAJOR >= 2)
PSMX_SET_TAG(psm_tag2, psm_tag, 0);
#endif
fi_context = context;
PSMX_CTXT_TYPE(fi_context) = PSMX_TSEND_CONTEXT;
PSMX_CTXT_USER(fi_context) = (void *)buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
#if (PSM_VERNO_MAJOR >= 2)
err = psm_mq_isend2(ep_priv->domain->psm_mq, psm_epaddr, 0,
&psm_tag2, buf, len, (void*)fi_context, &psm_req);
#else
err = psm_mq_isend(ep_priv->domain->psm_mq, psm_epaddr, 0,
psm_tag, buf, len, (void*)fi_context, &psm_req);
#endif
if (err != PSM_OK)
return psmx_errno(err);
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
int psmx_eq_poll_mq(struct psmx_fid_eq *eq, struct psmx_fid_domain *domain_if_null_eq)
{
psm_mq_req_t psm_req;
psm_mq_status_t psm_status;
struct fi_context *fi_context;
struct psmx_fid_domain *domain;
struct psmx_fid_ep *tmp_ep;
struct psmx_fid_eq *tmp_eq;
struct psmx_fid_cntr *tmp_cntr;
struct psmx_event *event;
int multi_recv;
int err;
if (eq)
domain = eq->domain;
else
domain = domain_if_null_eq;
while (1) {
err = psm_mq_ipeek(domain->psm_mq, &psm_req, NULL);
if (err == PSM_OK) {
err = psm_mq_test(&psm_req, &psm_status);
fi_context = psm_status.context;
tmp_ep = PSMX_CTXT_EP(fi_context);
tmp_eq = NULL;
tmp_cntr = NULL;
multi_recv = 0;
switch (PSMX_CTXT_TYPE(fi_context)) {
case PSMX_NOCOMP_SEND_CONTEXT:
tmp_ep->pending_sends--;
if (!tmp_ep->send_cntr_event_flag)
tmp_cntr = tmp_ep->send_cntr;
break;
case PSMX_NOCOMP_RECV_CONTEXT:
if (!tmp_ep->recv_cntr_event_flag)
tmp_cntr = tmp_ep->recv_cntr;
break;
case PSMX_NOCOMP_WRITE_CONTEXT:
tmp_ep->pending_writes--;
if (!tmp_ep->write_cntr_event_flag)
tmp_cntr = tmp_ep->write_cntr;
break;
case PSMX_NOCOMP_READ_CONTEXT:
tmp_ep->pending_reads--;
if (!tmp_ep->read_cntr_event_flag)
tmp_cntr = tmp_ep->read_cntr;
break;
case PSMX_INJECT_CONTEXT:
tmp_ep->pending_sends--;
if (!tmp_ep->send_cntr_event_flag)
tmp_cntr = tmp_ep->send_cntr;
free(fi_context);
break;
case PSMX_INJECT_WRITE_CONTEXT:
tmp_ep->pending_writes--;
if (!tmp_ep->write_cntr_event_flag)
tmp_cntr = tmp_ep->write_cntr;
free(fi_context);
break;
case PSMX_SEND_CONTEXT:
tmp_ep->pending_sends--;
tmp_eq = tmp_ep->send_eq;
tmp_cntr = tmp_ep->send_cntr;
break;
case PSMX_RECV_CONTEXT:
tmp_eq = tmp_ep->recv_eq;
tmp_cntr = tmp_ep->recv_cntr;
break;
case PSMX_MULTI_RECV_CONTEXT:
multi_recv = 1;
tmp_eq = tmp_ep->recv_eq;
tmp_cntr = tmp_ep->recv_cntr;
break;
case PSMX_READ_CONTEXT:
tmp_ep->pending_reads--;
tmp_eq = tmp_ep->send_eq;
tmp_cntr = tmp_ep->read_cntr;
break;
case PSMX_WRITE_CONTEXT:
tmp_ep->pending_writes--;
tmp_eq = tmp_ep->send_eq;
tmp_cntr = tmp_ep->write_cntr;
break;
}
if (tmp_eq) {
event = psmx_eq_create_event_from_status(tmp_eq, &psm_status);
if (!event)
return -ENOMEM;
psmx_eq_enqueue_event(tmp_eq, event);
}
if (tmp_cntr)
tmp_cntr->cntr.ops->add(&tmp_cntr->cntr, 1);
if (multi_recv) {
struct psmx_multi_recv *req;
psm_mq_req_t psm_req;
req = PSMX_CTXT_USER(fi_context);
req->offset += psm_status.nbytes;
if (req->offset + req->min_buf_size <= req->len) {
err = psm_mq_irecv(tmp_ep->domain->psm_mq,
req->tag, req->tagsel, req->flag,
req->buf + req->offset,
req->len - req->offset,
(void *)fi_context, &psm_req);
if (err != PSM_OK)
return psmx_errno(err);
PSMX_CTXT_REQ(fi_context) = psm_req;
}
else {
if (tmp_eq) {
event = psmx_eq_create_event(
tmp_eq,
req->context,
req->buf,
FI_MULTI_RECV,
req->len,
req->len - req->offset, /* data */
0, /* tag */
0, /* olen */
0); /* err */
if (!event)
return -ENOMEM;
psmx_eq_enqueue_event(tmp_eq, event);
}
free(req);
}
}
if (!eq || tmp_eq == eq)
return 1;
}
else if (err == PSM_MQ_NO_COMPLETIONS) {
return 0;
}
else {
return psmx_errno(err);
}
}
}
ssize_t _psmx_tagged_send(struct fid_ep *ep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr, uint64_t tag,
void *context, uint64_t flags)
#endif
{
struct psmx_fid_ep *ep_priv;
struct psmx_fid_av *av;
psm_epaddr_t psm_epaddr;
psm_mq_req_t psm_req;
uint64_t psm_tag;
#if (PSM_VERNO_MAJOR >= 2)
psm_mq_tag_t psm_tag2;
#endif
struct fi_context *fi_context;
int err;
size_t idx;
int no_completion = 0;
struct psmx_cq_event *event;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX_TRIGGERED_TSEND;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->tsend.ep = ep;
trigger->tsend.buf = buf;
trigger->tsend.len = len;
trigger->tsend.desc = desc;
trigger->tsend.dest_addr = dest_addr;
trigger->tsend.tag = tag;
trigger->tsend.context = context;
trigger->tsend.flags = flags & ~FI_TRIGGER;
#if (PSM_VERNO_MAJOR >= 2)
trigger->tsend.data = data;
#endif
psmx_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if (tag & ep_priv->domain->reserved_tag_bits) {
FI_WARN(&psmx_prov, FI_LOG_EP_DATA,
"using reserved tag bits."
"tag=%lx. reserved_bits=%lx.\n", tag,
ep_priv->domain->reserved_tag_bits);
}
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = (size_t)dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm_epaddr = av->psm_epaddrs[idx];
}
else {
psm_epaddr = (psm_epaddr_t) dest_addr;
}
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
#if (PSM_VERNO_MAJOR >= 2)
PSMX_SET_TAG(psm_tag2, psm_tag, data);
#endif
if ((flags & PSMX_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION)))
no_completion = 1;
if (flags & FI_INJECT) {
if (len > PSMX_INJECT_SIZE)
return -FI_EMSGSIZE;
#if (PSM_VERNO_MAJOR >= 2)
err = psm_mq_send2(ep_priv->domain->psm_mq, psm_epaddr, 0,
&psm_tag2, buf, len);
#else
err = psm_mq_send(ep_priv->domain->psm_mq, psm_epaddr, 0,
psm_tag, buf, len);
#endif
if (err != PSM_OK)
return psmx_errno(err);
if (ep_priv->send_cntr)
psmx_cntr_inc(ep_priv->send_cntr);
if (ep_priv->send_cq && !no_completion) {
event = psmx_cq_create_event(
ep_priv->send_cq,
context, (void *)buf, flags, len,
#if (PSM_VERNO_MAJOR >= 2)
(uint64_t) data, psm_tag,
#else
0 /* data */, psm_tag,
#endif
0 /* olen */,
0 /* err */);
if (event)
psmx_cq_enqueue_event(ep_priv->send_cq, event);
else
return -FI_ENOMEM;
}
return 0;
}
if (no_completion && !context) {
fi_context = &ep_priv->nocomp_send_context;
}
else {
if (!context)
return -FI_EINVAL;
fi_context = context;
PSMX_CTXT_TYPE(fi_context) = PSMX_TSEND_CONTEXT;
PSMX_CTXT_USER(fi_context) = (void *)buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
}
#if (PSM_VERNO_MAJOR >= 2)
err = psm_mq_isend2(ep_priv->domain->psm_mq, psm_epaddr, 0,
&psm_tag2, buf, len, (void*)fi_context, &psm_req);
#else
err = psm_mq_isend(ep_priv->domain->psm_mq, psm_epaddr, 0,
psm_tag, buf, len, (void*)fi_context, &psm_req);
#endif
if (err != PSM_OK)
return psmx_errno(err);
if (fi_context == context)
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
ssize_t _psmx_tagged_peek(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx_fid_ep *ep_priv;
#if (PSM_VERNO_MAJOR >= 2)
psm_mq_status2_t psm_status2;
psm_mq_tag_t psm_tag2, psm_tagsel2;
psm_mq_req_t req;
struct psmx_fid_av *av;
size_t idx;
psm_epaddr_t psm_src_addr;
#else
psm_mq_status_t psm_status;
#endif
uint64_t psm_tag, psm_tagsel;
struct psmx_cq_event *event;
int err;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
if (tag & ep_priv->domain->reserved_tag_bits) {
FI_WARN(&psmx_prov, FI_LOG_EP_DATA,
"using reserved tag bits."
"tag=%lx. reserved_bits=%lx.\n", tag,
ep_priv->domain->reserved_tag_bits);
}
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
psm_tagsel = (~ignore) | ep_priv->domain->reserved_tag_bits;
#if (PSM_VERNO_MAJOR >= 2)
if (src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm_src_addr = av->psm_epaddrs[idx];
}
else {
psm_src_addr = (psm_epaddr_t)src_addr;
}
}
else {
psm_src_addr = NULL;
}
PSMX_SET_TAG(psm_tag2, psm_tag, 0);
PSMX_SET_TAG(psm_tagsel2, psm_tagsel, 0);
if (flags & (FI_CLAIM | FI_DISCARD))
err = psm_mq_improbe2(ep_priv->domain->psm_mq,
psm_src_addr,
&psm_tag2, &psm_tagsel2,
&req, &psm_status2);
else
err = psm_mq_iprobe2(ep_priv->domain->psm_mq,
psm_src_addr,
&psm_tag2, &psm_tagsel2,
&psm_status2);
#else
if (flags & (FI_CLAIM | FI_DISCARD))
return -FI_EOPNOTSUPP;
err = psm_mq_iprobe(ep_priv->domain->psm_mq, psm_tag, psm_tagsel,
&psm_status);
#endif
switch (err) {
case PSM_OK:
if (ep_priv->recv_cq) {
#if (PSM_VERNO_MAJOR >= 2)
if ((flags & FI_CLAIM) && context)
PSMX_CTXT_REQ((struct fi_context *)context) = req;
tag = psm_status2.msg_tag.tag0 | (((uint64_t)psm_status2.msg_tag.tag1) << 32);
len = psm_status2.msg_length;
src_addr = (fi_addr_t)psm_status2.msg_peer;
#else
tag = psm_status.msg_tag;
len = psm_status.msg_length;
src_addr = 0;
#endif
event = psmx_cq_create_event(
ep_priv->recv_cq,
context, /* op_context */
NULL, /* buf */
flags|FI_RECV|FI_TAGGED,/* flags */
len, /* len */
0, /* data */
tag, /* tag */
len, /* olen */
0); /* err */
if (!event)
return -FI_ENOMEM;
event->source = src_addr;
psmx_cq_enqueue_event(ep_priv->recv_cq, event);
}
return 0;
case PSM_MQ_NO_COMPLETIONS:
return -FI_ENOMSG;
default:
return psmx_errno(err);
}
}
ssize_t _psmx_tagged_recv(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx_fid_ep *ep_priv;
psm_mq_req_t psm_req;
uint64_t psm_tag, psm_tagsel;
#if (PSM_VERNO_MAJOR >= 2)
psm_mq_tag_t psm_tag2, psm_tagsel2;
struct psmx_fid_av *av;
size_t idx;
#endif
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
if (flags & FI_PEEK)
return _psmx_tagged_peek(ep, buf, len, desc, src_addr,
tag, ignore, context, flags);
if (flags & FI_TRIGGER) {
struct psmx_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX_TRIGGERED_TRECV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->trecv.ep = ep;
trigger->trecv.buf = buf;
trigger->trecv.len = len;
trigger->trecv.desc = desc;
trigger->trecv.src_addr = src_addr;
trigger->trecv.tag = tag;
trigger->trecv.ignore = ignore;
trigger->trecv.context = context;
trigger->trecv.flags = flags & ~FI_TRIGGER;
psmx_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
#if (PSM_VERNO_MAJOR >= 2)
if (flags & FI_CLAIM) {
if (!context)
return -FI_EINVAL;
/* TODO: handle FI_DISCARD */
fi_context = context;
psm_req = PSMX_CTXT_REQ(fi_context);
PSMX_CTXT_TYPE(fi_context) = PSMX_TRECV_CONTEXT;
PSMX_CTXT_USER(fi_context) = buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
err = psm_mq_imrecv(ep_priv->domain->psm_mq, 0, /*flags*/
buf, len, context, &psm_req);
if (err != PSM_OK)
return psmx_errno(err);
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
#endif
if (tag & ep_priv->domain->reserved_tag_bits) {
FI_WARN(&psmx_prov, FI_LOG_EP_DATA,
"using reserved tag bits."
"tag=%lx. reserved_bits=%lx.\n", tag,
ep_priv->domain->reserved_tag_bits);
}
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
psm_tagsel = (~ignore) | ep_priv->domain->reserved_tag_bits;
if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION) && !context) {
fi_context = &ep_priv->nocomp_recv_context;
}
else {
if (!context)
return -FI_EINVAL;
fi_context = context;
PSMX_CTXT_TYPE(fi_context) = PSMX_TRECV_CONTEXT;
PSMX_CTXT_USER(fi_context) = buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
}
#if (PSM_VERNO_MAJOR >= 2)
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
src_addr = (fi_addr_t)av->psm_epaddrs[idx];
}
}
else {
src_addr = 0;
}
PSMX_SET_TAG(psm_tag2, psm_tag, 0);
PSMX_SET_TAG(psm_tagsel2, psm_tagsel, 0);
err = psm_mq_irecv2(ep_priv->domain->psm_mq,
(psm_epaddr_t)src_addr,
&psm_tag2, &psm_tagsel2, 0, /* flags */
buf, len, (void *)fi_context, &psm_req);
#else
err = psm_mq_irecv(ep_priv->domain->psm_mq,
psm_tag, psm_tagsel, 0, /* flags */
buf, len, (void *)fi_context, &psm_req);
#endif
if (err != PSM_OK)
return psmx_errno(err);
if (fi_context == context)
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
ssize_t _psmx_tagged_recv(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx_fid_ep *ep_priv;
psm_mq_req_t psm_req;
uint64_t psm_tag, psm_tagsel;
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx_fid_ep, ep);
if (flags & FI_PEEK)
return _psmx_tagged_peek(ep, buf, len, desc, src_addr,
tag, ignore, context, flags);
if (flags & FI_TRIGGER) {
struct psmx_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX_TRIGGERED_TRECV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->trecv.ep = ep;
trigger->trecv.buf = buf;
trigger->trecv.len = len;
trigger->trecv.desc = desc;
trigger->trecv.src_addr = src_addr;
trigger->trecv.tag = tag;
trigger->trecv.ignore = ignore;
trigger->trecv.context = context;
trigger->trecv.flags = flags & ~FI_TRIGGER;
psmx_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if (tag & ep_priv->domain->reserved_tag_bits) {
FI_WARN(&psmx_prov, FI_LOG_EP_DATA,
"using reserved tag bits."
"tag=%lx. reserved_bits=%lx.\n", tag,
ep_priv->domain->reserved_tag_bits);
}
psm_tag = tag & (~ep_priv->domain->reserved_tag_bits);
psm_tagsel = (~ignore) | ep_priv->domain->reserved_tag_bits;
if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION) && !context) {
fi_context = &ep_priv->nocomp_recv_context;
}
else {
if (!context)
return -FI_EINVAL;
fi_context = context;
PSMX_CTXT_TYPE(fi_context) = PSMX_TRECV_CONTEXT;
PSMX_CTXT_USER(fi_context) = buf;
PSMX_CTXT_EP(fi_context) = ep_priv;
}
err = psm_mq_irecv(ep_priv->domain->psm_mq,
psm_tag, psm_tagsel, 0, /* flags */
buf, len, (void *)fi_context, &psm_req);
if (err != PSM_OK)
return psmx_errno(err);
if (fi_context == context)
PSMX_CTXT_REQ(fi_context) = psm_req;
return 0;
}
