/* op_flags=0, FI_SELECTIVE_COMPLETION set, FI_AV_MAP */
static ssize_t
psmx2_tagged_send_no_event_av_map(struct fid_ep *ep, const void *buf,
size_t len, void *desc,
fi_addr_t dest_addr, uint64_t tag,
void *context)
{
struct psmx2_fid_ep *ep_priv;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
vlane = PSMX2_ADDR_TO_VL(dest_addr);
tag32 = PSMX2_TAG32(0, ep_priv->vlane, vlane);
PSMX2_SET_TAG(psm2_tag, tag, tag32);
fi_context = &ep_priv->nocomp_send_context;
err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, 0,
&psm2_tag, buf, len, (void*)fi_context,
&psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
return 0;
}
int psmx2_am_process_rma(struct psmx2_fid_domain *domain,
struct psmx2_am_request *req)
{
int err;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32;
if ((req->op & PSMX2_AM_OP_MASK) == PSMX2_AM_REQ_WRITE_LONG) {
tag32 = PSMX2_TAG32(PSMX2_RMA_BIT, req->write.peer_vl, req->write.vl);
PSMX2_SET_TAG(psm2_tag, (uint64_t)req->write.context, tag32);
PSMX2_SET_TAG(psm2_tagsel, -1ULL, -1);
err = psm2_mq_irecv2(domain->psm2_mq,
(psm2_epaddr_t)req->write.peer_addr,
&psm2_tag, &psm2_tagsel, 0,
(void *)req->write.addr, req->write.len,
(void *)&req->fi_context, &psm2_req);
} else {
tag32 = PSMX2_TAG32(PSMX2_RMA_BIT, req->read.vl, req->read.peer_vl);
PSMX2_SET_TAG(psm2_tag, (uint64_t)req->read.context, tag32);
err = psm2_mq_isend2(domain->psm2_mq,
(psm2_epaddr_t)req->read.peer_addr,
0, &psm2_tag,
(void *)req->read.addr, req->read.len,
(void *)&req->fi_context, &psm2_req);
}
return psmx2_errno(err);
}
int psmx2_epid_to_epaddr(struct psmx2_fid_domain *domain,
psm2_epid_t epid, psm2_epaddr_t *epaddr)
{
int err;
psm2_error_t errors;
psm2_epconn_t epconn;
struct psmx2_epaddr_context *context;
err = psm2_ep_epid_lookup(epid, &epconn);
if (err == PSM2_OK) {
context = psm2_epaddr_getctxt(epconn.addr);
if (context && context->epid == epid) {
*epaddr = epconn.addr;
return 0;
}
}
err = psm2_ep_connect(domain->psm2_ep, 1, &epid, NULL, &errors,
epaddr, psmx2_conn_timeout(1));
if (err != PSM2_OK)
return psmx2_errno(err);
psmx2_set_epaddr_context(domain,epid,*epaddr);
return 0;
}
int psmx2_am_process_rma(struct psmx2_trx_ctxt *trx_ctxt,
struct psmx2_am_request *req)
{
int err;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
if ((req->op & PSMX2_AM_OP_MASK) == PSMX2_AM_REQ_WRITE_LONG) {
PSMX2_SET_TAG(psm2_tag, (uint64_t)req->write.context, 0,
PSMX2_RMA_TYPE_WRITE);
PSMX2_SET_MASK(psm2_tagsel, PSMX2_MATCH_ALL, PSMX2_RMA_TYPE_MASK);
err = psm2_mq_irecv2(trx_ctxt->psm2_mq,
(psm2_epaddr_t)req->write.peer_addr,
&psm2_tag, &psm2_tagsel, 0,
(void *)req->write.addr, req->write.len,
(void *)&req->fi_context, &psm2_req);
} else {
PSMX2_SET_TAG(psm2_tag, (uint64_t)req->read.context, 0,
PSMX2_RMA_TYPE_READ);
err = psm2_mq_isend2(trx_ctxt->psm2_mq,
(psm2_epaddr_t)req->read.peer_addr,
0, &psm2_tag,
(void *)req->read.addr, req->read.len,
(void *)&req->fi_context, &psm2_req);
}
return psmx2_errno(err);
}
/* op_flags=0, FI_AV_MAP */
static ssize_t
psmx2_tagged_inject_no_flag_av_map(struct fid_ep *ep, const void *buf,
size_t len, fi_addr_t dest_addr,
uint64_t tag)
{
struct psmx2_fid_ep *ep_priv;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
int err;
if (len > PSMX2_INJECT_SIZE)
return -FI_EMSGSIZE;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
vlane = PSMX2_ADDR_TO_VL(dest_addr);
tag32 = PSMX2_TAG32(0, ep_priv->vlane, vlane);
PSMX2_SET_TAG(psm2_tag, tag, tag32);
err = psm2_mq_send2(ep_priv->domain->psm2_mq, psm2_epaddr, 0,
&psm2_tag, buf, len);
if (err != PSM2_OK)
return psmx2_errno(err);
if (ep_priv->send_cntr)
psmx2_cntr_inc(ep_priv->send_cntr);
return 0;
}
/* op_flags=0, FI_SELECTIVE_COMPLETION not set, FI_AV_TABLE */
static ssize_t
psmx2_tagged_recv_no_flag_av_table(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_SIZE(fi_context) = len;
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr);
vlane = 0;
} else {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
}
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = ~PSMX2_IOV_BIT;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
err = psm2_mq_irecv2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
static ssize_t psmx2_ep_cancel(fid_t fid, void *context)
{
struct psmx2_fid_ep *ep;
psm2_mq_status2_t status;
struct fi_context *fi_context = context;
uint64_t flags;
struct psmx2_cq_event *event;
int err;
ep = container_of(fid, struct psmx2_fid_ep, ep.fid);
if (!ep->domain)
return -FI_EBADF;
if (!fi_context)
return -FI_EINVAL;
switch (PSMX2_CTXT_TYPE(fi_context)) {
case PSMX2_TRECV_CONTEXT:
flags = FI_RECV | FI_TAGGED;
break;
case PSMX2_RECV_CONTEXT:
case PSMX2_MULTI_RECV_CONTEXT:
flags = FI_RECV | FI_MSG;
break;
default:
return -FI_EOPNOTSUPP;
}
err = psm2_mq_cancel((psm2_mq_req_t *)&PSMX2_CTXT_REQ(fi_context));
if (err == PSM2_OK) {
err = psm2_mq_test2((psm2_mq_req_t *)&PSMX2_CTXT_REQ(fi_context), &status);
if (err == PSM2_OK && ep->recv_cq) {
event = psmx2_cq_create_event(
ep->recv_cq,
status.context,
NULL, /* buf */
flags,
0, /* len */
0, /* data */
0, /* tag */
0 /* olen */,
-FI_ECANCELED);
if (event)
psmx2_cq_enqueue_event(ep->recv_cq, event);
else
return -FI_ENOMEM;
}
}
return psmx2_errno(err);
}
/* op_flags=0, FI_SELECTIVE_COMPLETION set, FI_AV_TABLE */
static ssize_t
psmx2_tagged_recv_no_event_av_table(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
fi_context = psmx2_ep_get_op_context(ep_priv);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = ~PSMX2_IOV_BIT;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
err = psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)fi_context, &psm2_req);
return psmx2_errno(err);
}
/* op_flags=0, FI_SELECTIVE_COMPLETION set, FI_AV_MAP */
static ssize_t
psmx2_tagged_recv_no_event_av_map(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
fi_context = psmx2_ep_get_op_context(ep_priv);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr);
vlane = 0;
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = ~PSMX2_IOV_BIT;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
err = psm2_mq_irecv2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)fi_context, &psm2_req);
return psmx2_errno(err);
}
/* op_flags=0, FI_SELECTIVE_COMPLETION not set, FI_AV_TABLE */
static ssize_t
psmx2_tagged_send_no_flag_av_table(struct fid_ep *ep, const void *buf,
size_t len, void *desc,
fi_addr_t dest_addr, uint64_t tag,
void *context)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
struct fi_context *fi_context;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(dest_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, dest_addr);
vlane = 0;
} else {
idx = (size_t)dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
}
tag32 = PSMX2_TAG32(0, ep_priv->vlane, vlane);
PSMX2_SET_TAG(psm2_tag, tag, tag32);
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TSEND_CONTEXT;
PSMX2_CTXT_USER(fi_context) = (void *)buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
err = psm2_mq_isend2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr, 0,
&psm2_tag, buf, len, (void*)fi_context,
&psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
/* op_flags=0, FI_SELECTIVE_COMPLETION not set, FI_AV_MAP */
static ssize_t
psmx2_tagged_recv_no_flag_av_map(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context)
{
struct psmx2_fid_ep *ep_priv;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_SIZE(fi_context) = len;
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = ~PSMX2_IOV_BIT;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
err = psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
/* op_flags=0, FI_AV_TABLE */
static ssize_t
psmx2_tagged_inject_no_flag_av_table(struct fid_ep *ep, const void *buf,
size_t len, fi_addr_t dest_addr,
uint64_t tag)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
int err;
size_t idx;
if (len > PSMX2_INJECT_SIZE)
return -FI_EMSGSIZE;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(dest_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, dest_addr);
vlane = 0;
} else {
idx = (size_t)dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
}
tag32 = PSMX2_TAG32(0, ep_priv->vlane, vlane);
PSMX2_SET_TAG(psm2_tag, tag, tag32);
err = psm2_mq_send2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr, 0,
&psm2_tag, buf, len);
if (err != PSM2_OK)
return psmx2_errno(err);
if (ep_priv->send_cntr)
psmx2_cntr_inc(ep_priv->send_cntr);
return 0;
}
/* op_flags=0, FI_SELECTIVE_COMPLETION set, FI_AV_TABLE */
static ssize_t
psmx2_tagged_send_no_event_av_table(struct fid_ep *ep, const void *buf,
size_t len, void *desc,
fi_addr_t dest_addr, uint64_t tag,
void *context)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
struct fi_context *fi_context;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
av = ep_priv->av;
idx = (size_t)dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
tag32 = PSMX2_TAG32(0, ep_priv->vlane, vlane);
PSMX2_SET_TAG(psm2_tag, tag, tag32);
fi_context = &ep_priv->nocomp_send_context;
err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, 0,
&psm2_tag, buf, len, (void*)fi_context,
&psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
return 0;
}
static void psmx2_av_post_completion(struct psmx2_fid_av *av, void *context,
uint64_t data, int prov_errno)
{
if (prov_errno) {
struct fi_eq_err_entry entry;
entry.fid = &av->av.fid;
entry.context = context;
entry.data = data;
entry.err = -psmx2_errno(prov_errno);
entry.prov_errno = prov_errno;
entry.err_data = NULL;
entry.err_data_size = 0;
fi_eq_write(av->eq, FI_AV_COMPLETE, &entry, sizeof(entry),
UTIL_FLAG_ERROR);
} else {
struct fi_eq_entry entry;
entry.fid = &av->av.fid;
entry.context = context;
entry.data = data;
fi_eq_write(av->eq, FI_AV_COMPLETE, &entry, sizeof(entry), 0);
}
}
int psmx2_handle_sendv_req(struct psmx2_fid_ep *ep,
psm2_mq_status2_t *psm2_status,
int multi_recv)
{
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
struct psmx2_sendv_reply *rep;
struct psmx2_multi_recv *recv_req;
struct fi_context *fi_context;
struct fi_context *recv_context;
int i, err;
uint8_t *recv_buf;
size_t recv_len, len;
if (psm2_status->error_code != PSM2_OK)
return psmx2_errno(psm2_status->error_code);
rep = malloc(sizeof(*rep));
if (!rep) {
psm2_status->error_code = PSM2_NO_MEMORY;
return -FI_ENOMEM;
}
recv_context = psm2_status->context;
if (multi_recv) {
recv_req = PSMX2_CTXT_USER(recv_context);
recv_buf = recv_req->buf + recv_req->offset;
recv_len = recv_req->len - recv_req->offset;
rep->multi_recv = 1;
} else {
recv_buf = PSMX2_CTXT_USER(recv_context);
recv_len = PSMX2_CTXT_SIZE(recv_context);
rep->multi_recv = 0;
}
/* assert(psm2_status->nbytes <= PSMX2_IOV_BUF_SIZE */
memcpy(&rep->iov_info, recv_buf, psm2_status->nbytes);
rep->user_context = psm2_status->context;
rep->buf = recv_buf;
rep->no_completion = 0;
rep->iov_done = 0;
rep->bytes_received = 0;
rep->msg_length = 0;
rep->error_code = PSM2_OK;
fi_context = &rep->fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_IOV_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = rep;
PSMX2_CTXT_EP(fi_context) = ep;
/* use the same tag, with IOV bit cleared, and seq_num added */
psm2_tag = psm2_status->msg_tag;
psm2_tag.tag2 &= ~PSMX2_IOV_BIT;
PSMX2_TAG32_SET_SEQ(psm2_tag.tag2, rep->iov_info.seq_num);
rep->comp_flag = (psm2_tag.tag2 & PSMX2_MSG_BIT) ? FI_MSG : FI_TAGGED;
if (psm2_tag.tag2 & PSMX2_IMM_BIT)
rep->comp_flag |= FI_REMOTE_CQ_DATA;
/* match every bit of the tag */
PSMX2_SET_TAG(psm2_tagsel, -1UL, -1);
for (i=0; i<rep->iov_info.count; i++) {
if (recv_len) {
len = MIN(recv_len, rep->iov_info.len[i]);
err = psm2_mq_irecv2(ep->domain->psm2_mq,
psm2_status->msg_peer,
&psm2_tag, &psm2_tagsel,
0/*flag*/, recv_buf, len,
(void *)fi_context, &psm2_req);
if (err) {
psm2_status->error_code = err;
return psmx2_errno(psm2_status->error_code);
}
recv_buf += len;
recv_len -= len;
} else {
/* recv buffer full, pust empty recvs */
err = psm2_mq_irecv2(ep->domain->psm2_mq,
psm2_status->msg_peer,
&psm2_tag, &psm2_tagsel,
0/*flag*/, NULL, 0,
(void *)fi_context, &psm2_req);
if (err) {
psm2_status->error_code = err;
return psmx2_errno(psm2_status->error_code);
}
}
}
return 0;
}
ssize_t psmx2_recv_generic(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr, void *context,
uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
int recv_flag = 0;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_RECV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->recv.ep = ep;
trigger->recv.buf = buf;
trigger->recv.len = len;
trigger->recv.desc = desc;
trigger->recv.src_addr = src_addr;
trigger->recv.context = context;
trigger->recv.flags = flags & ~FI_TRIGGER;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
tag32 = PSMX2_TAG32(PSMX2_MSG_BIT, vlane, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_IMM_BIT);
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(PSMX2_MSG_BIT, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_IMM_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, 0ULL, tag32);
PSMX2_SET_TAG(psm2_tagsel, 0ULL, tagsel32);
if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION)) {
fi_context = psmx2_ep_get_op_context(ep_priv);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
} else {
if (!context)
return -FI_EINVAL;
fi_context = context;
if (flags & FI_MULTI_RECV) {
struct psmx2_multi_recv *req;
req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
req->src_addr = psm2_epaddr;
req->tag = psm2_tag;
req->tagsel = psm2_tagsel;
req->flag = recv_flag;
req->buf = buf;
req->len = len;
req->offset = 0;
req->min_buf_size = ep_priv->min_multi_recv;
req->context = fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_MULTI_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
if (len > PSMX2_MAX_MSG_SIZE)
len = PSMX2_MAX_MSG_SIZE;
} else {
PSMX2_CTXT_TYPE(fi_context) = PSMX2_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
}
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_SIZE(fi_context) = len;
}
err = psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, recv_flag, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (fi_context == context)
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
ssize_t psmx2_sendv_generic(struct fid_ep *ep, const struct iovec *iov,
void *desc, size_t count, fi_addr_t dest_addr,
void *context, uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t tag32, tag32_base;
struct fi_context * fi_context;
int send_flag = 0;
int err;
size_t idx;
int no_completion = 0;
struct psmx2_cq_event *event;
size_t real_count;
size_t len, total_len;
char *p;
uint32_t *q;
int i;
struct psmx2_sendv_request *req;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_SENDV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->sendv.ep = ep;
trigger->sendv.iov = iov;
trigger->sendv.desc = desc;
trigger->sendv.count = count;
trigger->sendv.dest_addr = dest_addr;
trigger->sendv.context = context;
trigger->sendv.flags = flags & ~FI_TRIGGER;
trigger->sendv.data = data;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
total_len = 0;
real_count = 0;
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
total_len += iov[i].iov_len;
real_count++;
}
}
req = malloc(sizeof(*req));
if (!req)
return -FI_ENOMEM;
if (total_len <= PSMX2_IOV_BUF_SIZE) {
req->iov_protocol = PSMX2_IOV_PROTO_PACK;
p = req->buf;
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
memcpy(p, iov[i].iov_base, iov[i].iov_len);
p += iov[i].iov_len;
}
}
tag32_base = PSMX2_MSG_BIT;
len = total_len;
} else {
req->iov_protocol = PSMX2_IOV_PROTO_MULTI;
req->iov_done = 0;
req->iov_info.seq_num = (++ep_priv->iov_seq_num) %
PSMX2_IOV_MAX_SEQ_NUM + 1;
req->iov_info.count = (uint32_t)real_count;
req->iov_info.total_len = (uint32_t)total_len;
q = req->iov_info.len;
for (i=0; i<count; i++) {
if (iov[i].iov_len)
*q++ = (uint32_t)iov[i].iov_len;
}
tag32_base = PSMX2_MSG_BIT | PSMX2_IOV_BIT;
len = (3 + real_count) * sizeof(uint32_t);
}
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = (size_t)dest_addr;
if (idx >= av->last) {
free(req);
return -FI_EINVAL;
}
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
vlane = PSMX2_ADDR_TO_VL(dest_addr);
}
tag32 = PSMX2_TAG32(tag32_base, ep_priv->vlane, vlane);
if (flags & FI_REMOTE_CQ_DATA)
tag32 |= PSMX2_IMM_BIT;
PSMX2_SET_TAG(psm2_tag, data, tag32);
if ((flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION)))
no_completion = 1;
if (flags & FI_INJECT) {
if (len > PSMX2_INJECT_SIZE) {
free(req);
return -FI_EMSGSIZE;
}
err = psm2_mq_send2(ep_priv->domain->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, req->buf, len);
free(req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (ep_priv->send_cntr)
psmx2_cntr_inc(ep_priv->send_cntr);
if (ep_priv->send_cq && !no_completion) {
event = psmx2_cq_create_event(
ep_priv->send_cq,
context, NULL, flags, len,
(uint64_t) data,
0 /* tag */,
0 /* olen */,
0 /* err */);
if (event)
psmx2_cq_enqueue_event(ep_priv->send_cq, event);
else
return -FI_ENOMEM;
}
return 0;
}
req->no_completion = no_completion;
req->user_context = context;
req->comp_flag = FI_MSG;
fi_context = &req->fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_SENDV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
PSMX2_CTXT_EP(fi_context) = ep_priv;
err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, req->buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK) {
free(req);
return psmx2_errno(err);
}
PSMX2_CTXT_REQ(fi_context) = psm2_req;
if (req->iov_protocol == PSMX2_IOV_PROTO_MULTI) {
fi_context = &req->fi_context_iov;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_IOV_SEND_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
PSMX2_CTXT_EP(fi_context) = ep_priv;
tag32 &= ~PSMX2_IOV_BIT;
PSMX2_TAG32_SET_SEQ(tag32, req->iov_info.seq_num);
PSMX2_SET_TAG(psm2_tag, data, tag32);
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
err = psm2_mq_isend2(ep_priv->domain->psm2_mq,
psm2_epaddr, send_flag, &psm2_tag,
iov[i].iov_base, iov[i].iov_len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
}
}
}
return 0;
}
ssize_t psmx2_send_generic(struct fid_ep *ep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr, void *context,
uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t tag32;
struct fi_context * fi_context;
int send_flag = 0;
int err;
size_t idx;
int no_completion = 0;
struct psmx2_cq_event *event;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_SEND;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->send.ep = ep;
trigger->send.buf = buf;
trigger->send.len = len;
trigger->send.desc = desc;
trigger->send.dest_addr = dest_addr;
trigger->send.context = context;
trigger->send.flags = flags & ~FI_TRIGGER;
trigger->send.data = data;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = (size_t)dest_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr);
vlane = PSMX2_ADDR_TO_VL(dest_addr);
}
tag32 = PSMX2_TAG32(PSMX2_MSG_BIT, ep_priv->vlane, vlane);
if (flags & FI_REMOTE_CQ_DATA)
tag32 |= PSMX2_IMM_BIT;
PSMX2_SET_TAG(psm2_tag, data, tag32);
if ((flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION)))
no_completion = 1;
if (flags & FI_INJECT) {
if (len > PSMX2_INJECT_SIZE)
return -FI_EMSGSIZE;
err = psm2_mq_send2(ep_priv->domain->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, buf, len);
if (err != PSM2_OK)
return psmx2_errno(err);
if (ep_priv->send_cntr)
psmx2_cntr_inc(ep_priv->send_cntr);
if (ep_priv->send_cq && !no_completion) {
event = psmx2_cq_create_event(
ep_priv->send_cq,
context, (void *)buf, flags, len,
(uint64_t) data,
0 /* tag */,
0 /* olen */,
0 /* err */);
if (event)
psmx2_cq_enqueue_event(ep_priv->send_cq, event);
else
return -FI_ENOMEM;
}
return 0;
}
if (no_completion && !context) {
fi_context = &ep_priv->nocomp_send_context;
} else {
if (!context)
return -FI_EINVAL;
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_SEND_CONTEXT;
PSMX2_CTXT_USER(fi_context) = (void *)buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
}
err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (fi_context == context)
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
int psmx2_domain_open(struct fid_fabric *fabric, struct fi_info *info,
struct fid_domain **domain, void *context)
{
struct psmx2_fid_fabric *fabric_priv;
struct psmx2_fid_domain *domain_priv;
struct psm2_ep_open_opts opts;
int err;
FI_INFO(&psmx2_prov, FI_LOG_DOMAIN, "\n");
fabric_priv = container_of(fabric, struct psmx2_fid_fabric, fabric);
psmx2_fabric_acquire(fabric_priv);
if (fabric_priv->active_domain) {
psmx2_domain_acquire(fabric_priv->active_domain);
*domain = &fabric_priv->active_domain->domain;
return 0;
}
if (!info->domain_attr->name ||
strcmp(info->domain_attr->name, PSMX2_DOMAIN_NAME)) {
err = -FI_EINVAL;
goto err_out;
}
domain_priv = (struct psmx2_fid_domain *) calloc(1, sizeof *domain_priv);
if (!domain_priv) {
err = -FI_ENOMEM;
goto err_out;
}
domain_priv->domain.fid.fclass = FI_CLASS_DOMAIN;
domain_priv->domain.fid.context = context;
domain_priv->domain.fid.ops = &psmx2_fi_ops;
domain_priv->domain.ops = &psmx2_domain_ops;
domain_priv->domain.mr = &psmx2_mr_ops;
domain_priv->mr_mode = info->domain_attr->mr_mode;
domain_priv->mode = info->mode;
domain_priv->caps = info->caps;
domain_priv->fabric = fabric_priv;
domain_priv->progress_thread_enabled =
(info->domain_attr->data_progress == FI_PROGRESS_AUTO);
psm2_ep_open_opts_get_defaults(&opts);
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"uuid: %s\n", psmx2_uuid_to_string(fabric_priv->uuid));
err = psm2_ep_open(fabric_priv->uuid, &opts,
&domain_priv->psm2_ep, &domain_priv->psm2_epid);
if (err != PSM2_OK) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"psm2_ep_open returns %d, errno=%d\n", err, errno);
err = psmx2_errno(err);
goto err_out_free_domain;
}
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"epid: 0x%016lx\n", domain_priv->psm2_epid);
err = psm2_mq_init(domain_priv->psm2_ep, PSM2_MQ_ORDERMASK_ALL,
NULL, 0, &domain_priv->psm2_mq);
if (err != PSM2_OK) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"psm2_mq_init returns %d, errno=%d\n", err, errno);
err = psmx2_errno(err);
goto err_out_close_ep;
}
err = fastlock_init(&domain_priv->mr_lock);
if (err) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"fastlock_init(mr_lock) returns %d\n", err);
goto err_out_finalize_mq;
}
domain_priv->mr_map = rbtNew(&psmx2_key_compare);
if (!domain_priv->mr_map) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"rbtNew failed\n");
goto err_out_destroy_mr_lock;
}
domain_priv->mr_reserved_key = 1;
err = fastlock_init(&domain_priv->vl_lock);
if (err) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"fastlock_init(vl_lock) returns %d\n", err);
goto err_out_delete_mr_map;
}
memset(domain_priv->vl_map, 0, sizeof(domain_priv->vl_map));
domain_priv->vl_alloc = 0;
err = fastlock_init(&domain_priv->poll_lock);
if (err) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"fastlock_init(poll_lock) returns %d\n", err);
goto err_out_destroy_vl_lock;
}
/* Set active domain before psmx2_domain_enable_ep() installs the
* AM handlers to ensure that psmx2_active_fabric->active_domain
* is always non-NULL inside the handlers. Notice that the vlaue
* active_domain becomes NULL again only when the domain is closed.
* At that time the AM handlers are gone with the PSM endpoint.
*/
fabric_priv->active_domain = domain_priv;
if (psmx2_domain_enable_ep(domain_priv, NULL) < 0)
goto err_out_reset_active_domain;
if (domain_priv->progress_thread_enabled)
psmx2_domain_start_progress(domain_priv);
domain_priv->refcnt = 1;
*domain = &domain_priv->domain;
return 0;
err_out_reset_active_domain:
fabric_priv->active_domain = NULL;
fastlock_destroy(&domain_priv->poll_lock);
err_out_destroy_vl_lock:
fastlock_destroy(&domain_priv->vl_lock);
err_out_delete_mr_map:
rbtDelete(domain_priv->mr_map);
err_out_destroy_mr_lock:
fastlock_destroy(&domain_priv->mr_lock);
err_out_finalize_mq:
psm2_mq_finalize(domain_priv->psm2_mq);
err_out_close_ep:
if (psm2_ep_close(domain_priv->psm2_ep, PSM2_EP_CLOSE_GRACEFUL,
(int64_t) psmx2_env.timeout * 1000000000LL) != PSM2_OK)
psm2_ep_close(domain_priv->psm2_ep, PSM2_EP_CLOSE_FORCE, 0);
err_out_free_domain:
free(domain_priv);
err_out:
psmx2_fabric_release(fabric_priv);
return err;
}
ssize_t psmx2_recv_generic(struct fid_ep *ep, void *buf, size_t len,
void *desc, fi_addr_t src_addr, void *context,
uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
struct fi_context *fi_context;
int recv_flag = 0;
int err;
int enable_completion;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_recv(ep, buf, len, desc, src_addr,
context, flags);
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
assert(av);
psm2_epaddr = psmx2_av_translate_addr(av, ep_priv->rx, src_addr, av->type);
} else {
psm2_epaddr = 0;
}
PSMX2_SET_TAG(psm2_tag, 0ULL, 0, PSMX2_TYPE_MSG);
PSMX2_SET_MASK(psm2_tagsel, PSMX2_MATCH_NONE, PSMX2_TYPE_MASK);
enable_completion = !ep_priv->recv_selective_completion ||
(flags & FI_COMPLETION);
if (enable_completion) {
assert(context);
fi_context = context;
if (flags & FI_MULTI_RECV) {
struct psmx2_multi_recv *req;
req = calloc(1, sizeof(*req));
if (!req)
return -FI_ENOMEM;
req->src_addr = psm2_epaddr;
req->tag = psm2_tag;
req->tagsel = psm2_tagsel;
req->flag = recv_flag;
req->buf = buf;
req->len = len;
req->offset = 0;
req->min_buf_size = ep_priv->min_multi_recv;
req->context = fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_MULTI_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
if (len > PSMX2_MAX_MSG_SIZE)
len = PSMX2_MAX_MSG_SIZE;
} else {
PSMX2_CTXT_TYPE(fi_context) = PSMX2_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
}
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_SIZE(fi_context) = len;
} else {
PSMX2_EP_GET_OP_CONTEXT(ep_priv, fi_context);
#if !PSMX2_USE_REQ_CONTEXT
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
#endif
}
err = psm2_mq_irecv2(ep_priv->rx->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, recv_flag, buf, len,
(void *)fi_context, &psm2_req);
if (OFI_UNLIKELY(err != PSM2_OK))
return psmx2_errno(err);
if (enable_completion) {
PSMX2_CTXT_REQ(fi_context) = psm2_req;
} else {
#if PSMX2_USE_REQ_CONTEXT
PSMX2_REQ_GET_OP_CONTEXT(psm2_req, fi_context);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
#endif
}
return 0;
}
ssize_t psmx2_tagged_recv_generic(struct fid_ep *ep, void *buf,
size_t len, void *desc,
fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_PEEK)
return psmx2_tagged_peek_generic(ep, buf, len, desc,
src_addr, tag, ignore,
context, flags);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_TRECV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->trecv.ep = ep;
trigger->trecv.buf = buf;
trigger->trecv.len = len;
trigger->trecv.desc = desc;
trigger->trecv.src_addr = src_addr;
trigger->trecv.tag = tag;
trigger->trecv.ignore = ignore;
trigger->trecv.context = context;
trigger->trecv.flags = flags & ~FI_TRIGGER;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if (flags & FI_CLAIM) {
if (!context)
return -FI_EINVAL;
if (flags & FI_DISCARD) {
psm2_mq_status2_t psm2_status;
struct psmx2_cq_event *event;
fi_context = context;
psm2_req = PSMX2_CTXT_REQ(fi_context);
err = psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0,
NULL, 0, context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
psm2_mq_wait2(&psm2_req, &psm2_status);
if (ep_priv->recv_cq &&
(!ep_priv->recv_selective_completion || (flags & FI_COMPLETION))) {
tag = PSMX2_GET_TAG64(psm2_status.msg_tag);
event = psmx2_cq_create_event(
ep_priv->recv_cq,
context, /* op_context */
NULL, /* buf */
flags|FI_RECV|FI_TAGGED,/* flags */
0, /* len */
0, /* data */
tag, /* tag */
0, /* olen */
0); /* err */
if (!event)
return -FI_ENOMEM;
vlane = PSMX2_TAG32_GET_SRC(psm2_status.msg_tag.tag2);
event->source_is_valid = 1;
event->source = PSMX2_EP_TO_ADDR(psm2_status.msg_peer, vlane);
event->source_av = ep_priv->av;
psmx2_cq_enqueue_event(ep_priv->recv_cq, event);
}
if (ep_priv->recv_cntr)
psmx2_cntr_inc(ep_priv->recv_cntr);
return 0;
}
fi_context = context;
psm2_req = PSMX2_CTXT_REQ(fi_context);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
err = psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0,
buf, len, context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION)) {
fi_context = psmx2_ep_get_op_context(ep_priv);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
} else {
if (!context)
return -FI_EINVAL;
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_SIZE(fi_context) = len;
}
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr);
vlane = 0;
} else if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = ~PSMX2_IOV_BIT;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
err = psm2_mq_irecv2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (fi_context == context)
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
ssize_t psmx2_tagged_recv_generic(struct fid_ep *ep, void *buf,
size_t len, void *desc,
fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
struct fi_context *fi_context;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_PEEK)
return psmx2_tagged_peek_generic(ep, buf, len, desc,
src_addr, tag, ignore,
context, flags);
if (flags & FI_TRIGGER) {
struct psmx2_trigger *trigger;
struct fi_triggered_context *ctxt = context;
trigger = calloc(1, sizeof(*trigger));
if (!trigger)
return -FI_ENOMEM;
trigger->op = PSMX2_TRIGGERED_TRECV;
trigger->cntr = container_of(ctxt->trigger.threshold.cntr,
struct psmx2_fid_cntr, cntr);
trigger->threshold = ctxt->trigger.threshold.threshold;
trigger->trecv.ep = ep;
trigger->trecv.buf = buf;
trigger->trecv.len = len;
trigger->trecv.desc = desc;
trigger->trecv.src_addr = src_addr;
trigger->trecv.tag = tag;
trigger->trecv.ignore = ignore;
trigger->trecv.context = context;
trigger->trecv.flags = flags & ~FI_TRIGGER;
psmx2_cntr_add_trigger(trigger->cntr, trigger);
return 0;
}
if (flags & FI_CLAIM) {
if (!context)
return -FI_EINVAL;
/* TODO: handle FI_DISCARD */
fi_context = context;
psm2_req = PSMX2_CTXT_REQ(fi_context);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
err = psm2_mq_imrecv(ep_priv->domain->psm2_mq, 0,
buf, len, context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION)) {
fi_context = psmx2_ep_get_op_context(ep_priv);
PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_SIZE(fi_context) = len;
} else {
if (!context)
return -FI_EINVAL;
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_CTXT_SIZE(fi_context) = len;
}
if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = ~PSMX2_IOV_BIT;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS);
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
err = psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr,
&psm2_tag, &psm2_tagsel, 0, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (fi_context == context)
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
ssize_t psmx2_sendv_generic(struct fid_ep *ep, const struct iovec *iov,
void **desc, size_t count, fi_addr_t dest_addr,
void *context, uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
uint32_t msg_flags;
struct fi_context * fi_context;
int send_flag = 0;
int err;
int no_completion = 0;
struct psmx2_cq_event *event;
size_t real_count;
size_t len, total_len;
char *p;
uint32_t *q;
int i, j;
struct psmx2_sendv_request *req;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_sendv(ep, iov, desc, count,
dest_addr, context, flags,
data);
total_len = 0;
real_count = 0;
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
total_len += iov[i].iov_len;
real_count++;
j = i;
}
}
if (real_count == 1)
return psmx2_send_generic(ep, iov[j].iov_base, iov[j].iov_len,
desc ? desc[j] : NULL, dest_addr,
context, flags, data);
req = malloc(sizeof(*req));
if (!req)
return -FI_ENOMEM;
if (total_len <= PSMX2_IOV_BUF_SIZE) {
req->iov_protocol = PSMX2_IOV_PROTO_PACK;
p = req->buf;
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
memcpy(p, iov[i].iov_base, iov[i].iov_len);
p += iov[i].iov_len;
}
}
msg_flags = PSMX2_TYPE_MSG;
len = total_len;
} else {
req->iov_protocol = PSMX2_IOV_PROTO_MULTI;
req->iov_done = 0;
req->iov_info.seq_num = (++ep_priv->iov_seq_num) %
PSMX2_IOV_MAX_SEQ_NUM + 1;
req->iov_info.count = (uint32_t)real_count;
req->iov_info.total_len = (uint32_t)total_len;
q = req->iov_info.len;
for (i=0; i<count; i++) {
if (iov[i].iov_len)
*q++ = (uint32_t)iov[i].iov_len;
}
msg_flags = PSMX2_TYPE_MSG | PSMX2_IOV_BIT;
len = (3 + real_count) * sizeof(uint32_t);
}
av = ep_priv->av;
assert(av);
psm2_epaddr = psmx2_av_translate_addr(av, ep_priv->tx, dest_addr, av->type);
if (flags & FI_REMOTE_CQ_DATA)
msg_flags |= PSMX2_IMM_BIT;
PSMX2_SET_TAG(psm2_tag, 0ULL, data, msg_flags);
if ((flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION)))
no_completion = 1;
if (flags & FI_INJECT) {
if (len > psmx2_env.inject_size) {
free(req);
return -FI_EMSGSIZE;
}
err = psm2_mq_send2(ep_priv->tx->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, req->buf, len);
free(req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (ep_priv->send_cntr)
psmx2_cntr_inc(ep_priv->send_cntr, 0);
if (ep_priv->send_cq && !no_completion) {
event = psmx2_cq_create_event(
ep_priv->send_cq,
context, NULL, flags, len,
(uint64_t) data,
0 /* tag */,
0 /* olen */,
0 /* err */);
if (event)
psmx2_cq_enqueue_event(ep_priv->send_cq, event);
else
return -FI_ENOMEM;
}
return 0;
}
req->no_completion = no_completion;
req->user_context = context;
req->comp_flag = FI_MSG;
fi_context = &req->fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_SENDV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
PSMX2_CTXT_EP(fi_context) = ep_priv;
err = psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, req->buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK) {
free(req);
return psmx2_errno(err);
}
PSMX2_CTXT_REQ(fi_context) = psm2_req;
if (req->iov_protocol == PSMX2_IOV_PROTO_MULTI) {
fi_context = &req->fi_context_iov;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_IOV_SEND_CONTEXT;
PSMX2_CTXT_USER(fi_context) = req;
PSMX2_CTXT_EP(fi_context) = ep_priv;
PSMX2_SET_TAG(psm2_tag, req->iov_info.seq_num, 0, PSMX2_TYPE_IOV_PAYLOAD);
for (i=0; i<count; i++) {
if (iov[i].iov_len) {
err = psm2_mq_isend2(ep_priv->tx->psm2_mq,
psm2_epaddr, send_flag, &psm2_tag,
iov[i].iov_base, iov[i].iov_len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
}
}
}
return 0;
}
static ssize_t psmx2_tagged_peek_generic(struct fid_ep *ep,
void *buf, size_t len,
void *desc, fi_addr_t src_addr,
uint64_t tag, uint64_t ignore,
void *context, uint64_t flags)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
struct psmx2_cq_event *event;
psm2_epaddr_t psm2_epaddr;
uint8_t vlane;
psm2_mq_req_t req;
psm2_mq_status2_t psm2_status;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
uint32_t tag32, tagsel32;
size_t idx;
int err;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (src_addr != FI_ADDR_UNSPEC) {
av = ep_priv->av;
if (av && PSMX2_SEP_ADDR_TEST(src_addr)) {
psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr);
vlane = 0;
} else if (av && av->type == FI_AV_TABLE) {
idx = (size_t)src_addr;
if (idx >= av->last)
return -FI_EINVAL;
psm2_epaddr = av->epaddrs[idx];
vlane = av->vlanes[idx];
} else {
psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr);
vlane = PSMX2_ADDR_TO_VL(src_addr);
}
tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane);
tagsel32 = -1;
} else {
psm2_epaddr = 0;
tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane);
tagsel32 = ~PSMX2_SRC_BITS;
}
PSMX2_SET_TAG(psm2_tag, tag, tag32);
PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32);
if (flags & (FI_CLAIM | FI_DISCARD))
err = psm2_mq_improbe2(ep_priv->trx_ctxt->psm2_mq,
psm2_epaddr, &psm2_tag,
&psm2_tagsel, &req, &psm2_status);
else
err = psm2_mq_iprobe2(ep_priv->trx_ctxt->psm2_mq,
psm2_epaddr, &psm2_tag, &psm2_tagsel,
&psm2_status);
switch (err) {
case PSM2_OK:
if (ep_priv->recv_cq) {
if (flags & FI_CLAIM) {
if (context)
PSMX2_CTXT_REQ((struct fi_context *)context) = req;
} else if (flags & FI_DISCARD) {
if (!psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0,
NULL, 0, req, &req))
psm2_mq_wait2(&req, NULL);
}
tag = PSMX2_GET_TAG64(psm2_status.msg_tag);
len = psm2_status.msg_length;
event = psmx2_cq_create_event(
ep_priv->recv_cq,
context, /* op_context */
NULL, /* buf */
flags|FI_RECV|FI_TAGGED,/* flags */
len, /* len */
0, /* data */
tag, /* tag */
len, /* olen */
0); /* err */
if (!event)
return -FI_ENOMEM;
vlane = PSMX2_TAG32_GET_SRC(psm2_status.msg_tag.tag2);
event->source_is_valid = 1;
event->source = PSMX2_EP_TO_ADDR(psm2_status.msg_peer, vlane);
event->source_av = ep_priv->av;
psmx2_cq_enqueue_event(ep_priv->recv_cq, event);
}
return 0;
case PSM2_MQ_NO_COMPLETIONS:
if (ep_priv->recv_cq) {
event = psmx2_cq_create_event(
ep_priv->recv_cq,
context, /* op_context */
NULL, /* buf */
flags|FI_RECV|FI_TAGGED,/* flags */
len, /* len */
0, /* data */
tag, /* tag */
len, /* olen */
-FI_ENOMSG); /* err */
if (!event)
return -FI_ENOMEM;
event->source = 0;
psmx2_cq_enqueue_event(ep_priv->recv_cq, event);
}
return 0;
default:
return psmx2_errno(err);
}
}
struct psmx2_trx_ctxt *psmx2_trx_ctxt_alloc(struct psmx2_fid_domain *domain,
struct psmx2_src_name *src_addr,
int sep_ctxt_idx)
{
struct psmx2_trx_ctxt *trx_ctxt;
struct psm2_ep_open_opts opts;
int should_retry = 0;
int err;
trx_ctxt = calloc(1, sizeof(*trx_ctxt));
if (!trx_ctxt) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"failed to allocate trx_ctxt.\n");
return NULL;
}
psm2_ep_open_opts_get_defaults(&opts);
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"uuid: %s\n", psmx2_uuid_to_string(domain->fabric->uuid));
if (src_addr) {
opts.unit = src_addr->unit;
opts.port = src_addr->port;
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"ep_open_opts: unit=%d port=%u\n", opts.unit, opts.port);
}
if (opts.unit < 0 && sep_ctxt_idx >= 0) {
should_retry = 1;
opts.unit = sep_ctxt_idx % psmx2_env.num_devunits;
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"sep %d: ep_open_opts: unit=%d\n", sep_ctxt_idx, opts.unit);
}
err = psm2_ep_open(domain->fabric->uuid, &opts,
&trx_ctxt->psm2_ep, &trx_ctxt->psm2_epid);
if (err != PSM2_OK) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"psm2_ep_open returns %d, errno=%d\n", err, errno);
if (!should_retry) {
err = psmx2_errno(err);
goto err_out;
}
/* When round-robin fails, retry w/o explicit assignment */
opts.unit = -1;
err = psm2_ep_open(domain->fabric->uuid, &opts,
&trx_ctxt->psm2_ep, &trx_ctxt->psm2_epid);
if (err != PSM2_OK) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"psm2_ep_open returns %d, errno=%d\n", err, errno);
err = psmx2_errno(err);
goto err_out;
}
}
FI_INFO(&psmx2_prov, FI_LOG_CORE,
"epid: 0x%016lx\n", trx_ctxt->psm2_epid);
err = psm2_mq_init(trx_ctxt->psm2_ep, PSM2_MQ_ORDERMASK_ALL,
NULL, 0, &trx_ctxt->psm2_mq);
if (err != PSM2_OK) {
FI_WARN(&psmx2_prov, FI_LOG_CORE,
"psm2_mq_init returns %d, errno=%d\n", err, errno);
err = psmx2_errno(err);
goto err_out_close_ep;
}
fastlock_init(&trx_ctxt->poll_lock);
fastlock_init(&trx_ctxt->rma_queue.lock);
fastlock_init(&trx_ctxt->trigger_queue.lock);
slist_init(&trx_ctxt->rma_queue.list);
slist_init(&trx_ctxt->trigger_queue.list);
return trx_ctxt;
err_out_close_ep:
if (psm2_ep_close(trx_ctxt->psm2_ep, PSM2_EP_CLOSE_GRACEFUL,
(int64_t) psmx2_env.timeout * 1000000000LL) != PSM2_OK)
psm2_ep_close(trx_ctxt->psm2_ep, PSM2_EP_CLOSE_FORCE, 0);
err_out:
free(trx_ctxt);
return NULL;
}
ssize_t psmx2_send_generic(struct fid_ep *ep, const void *buf, size_t len,
void *desc, fi_addr_t dest_addr, void *context,
uint64_t flags, uint64_t data)
{
struct psmx2_fid_ep *ep_priv;
struct psmx2_fid_av *av;
psm2_epaddr_t psm2_epaddr;
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag;
struct fi_context * fi_context;
int send_flag = 0;
int err;
int no_completion = 0;
struct psmx2_cq_event *event;
int have_data = (flags & FI_REMOTE_CQ_DATA) > 0;
ep_priv = container_of(ep, struct psmx2_fid_ep, ep);
if (flags & FI_TRIGGER)
return psmx2_trigger_queue_send(ep, buf, len, desc, dest_addr,
context, flags, data);
av = ep_priv->av;
assert(av);
psm2_epaddr = psmx2_av_translate_addr(av, ep_priv->tx, dest_addr, av->type);
PSMX2_SET_TAG(psm2_tag, 0, data, PSMX2_TYPE_MSG | PSMX2_IMM_BIT_SET(have_data));
if ((flags & PSMX2_NO_COMPLETION) ||
(ep_priv->send_selective_completion && !(flags & FI_COMPLETION)))
no_completion = 1;
if (flags & FI_INJECT) {
if (len > psmx2_env.inject_size)
return -FI_EMSGSIZE;
err = psm2_mq_send2(ep_priv->tx->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, buf, len);
if (err != PSM2_OK)
return psmx2_errno(err);
if (ep_priv->send_cntr)
psmx2_cntr_inc(ep_priv->send_cntr, 0);
if (ep_priv->send_cq && !no_completion) {
event = psmx2_cq_create_event(
ep_priv->send_cq,
context, (void *)buf, flags, len,
(uint64_t) data,
0 /* tag */,
0 /* olen */,
0 /* err */);
if (event)
psmx2_cq_enqueue_event(ep_priv->send_cq, event);
else
return -FI_ENOMEM;
}
return 0;
}
if (no_completion) {
fi_context = &ep_priv->nocomp_send_context;
} else {
assert(context);
fi_context = context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_SEND_CONTEXT;
PSMX2_CTXT_USER(fi_context) = (void *)buf;
PSMX2_CTXT_EP(fi_context) = ep_priv;
}
err = psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr,
send_flag, &psm2_tag, buf, len,
(void *)fi_context, &psm2_req);
if (err != PSM2_OK)
return psmx2_errno(err);
if (fi_context == context)
PSMX2_CTXT_REQ(fi_context) = psm2_req;
return 0;
}
int psmx2_handle_sendv_req(struct psmx2_fid_ep *ep,
PSMX2_STATUS_TYPE *status,
int multi_recv)
{
psm2_mq_req_t psm2_req;
psm2_mq_tag_t psm2_tag, psm2_tagsel;
struct psmx2_sendv_reply *rep;
struct psmx2_multi_recv *recv_req;
struct fi_context *fi_context;
struct fi_context *recv_context;
int i, err;
uint8_t *recv_buf;
size_t recv_len, len;
if (PSMX2_STATUS_ERROR(status) != PSM2_OK)
return psmx2_errno(PSMX2_STATUS_ERROR(status));
rep = malloc(sizeof(*rep));
if (!rep) {
PSMX2_STATUS_ERROR(status) = PSM2_NO_MEMORY;
return -FI_ENOMEM;
}
recv_context = PSMX2_STATUS_CONTEXT(status);
if (multi_recv) {
recv_req = PSMX2_CTXT_USER(recv_context);
recv_buf = recv_req->buf + recv_req->offset;
recv_len = recv_req->len - recv_req->offset;
rep->multi_recv = 1;
} else {
recv_buf = PSMX2_CTXT_USER(recv_context);
recv_len = PSMX2_CTXT_SIZE(recv_context);
rep->multi_recv = 0;
}
/* assert(PSMX2_STATUS_RCVLEN(status) <= PSMX2_IOV_BUF_SIZE); */
memcpy(&rep->iov_info, recv_buf, PSMX2_STATUS_RCVLEN(status));
rep->user_context = PSMX2_STATUS_CONTEXT(status);
rep->tag = PSMX2_STATUS_TAG(status);
rep->buf = recv_buf;
rep->no_completion = 0;
rep->iov_done = 0;
rep->bytes_received = 0;
rep->msg_length = 0;
rep->error_code = PSM2_OK;
fi_context = &rep->fi_context;
PSMX2_CTXT_TYPE(fi_context) = PSMX2_IOV_RECV_CONTEXT;
PSMX2_CTXT_USER(fi_context) = rep;
PSMX2_CTXT_EP(fi_context) = ep;
rep->comp_flag = PSMX2_IS_MSG(PSMX2_GET_FLAGS(rep->tag)) ? FI_MSG : FI_TAGGED;
if (PSMX2_HAS_IMM(PSMX2_GET_FLAGS(rep->tag)))
rep->comp_flag |= FI_REMOTE_CQ_DATA;
/* IOV payload uses a sequence number in place of a tag. */
PSMX2_SET_TAG(psm2_tag, rep->iov_info.seq_num, 0, PSMX2_TYPE_IOV_PAYLOAD);
PSMX2_SET_MASK(psm2_tagsel, PSMX2_MATCH_ALL, PSMX2_TYPE_MASK);
for (i=0; i<rep->iov_info.count; i++) {
if (recv_len) {
len = MIN(recv_len, rep->iov_info.len[i]);
err = psm2_mq_irecv2(ep->rx->psm2_mq,
PSMX2_STATUS_PEER(status),
&psm2_tag, &psm2_tagsel,
0/*flag*/, recv_buf, len,
(void *)fi_context, &psm2_req);
if (err) {
PSMX2_STATUS_ERROR(status) = err;
return psmx2_errno(err);
}
recv_buf += len;
recv_len -= len;
} else {
/* recv buffer full, post empty recvs */
err = psm2_mq_irecv2(ep->rx->psm2_mq,
PSMX2_STATUS_PEER(status),
&psm2_tag, &psm2_tagsel,
0/*flag*/, NULL, 0,
(void *)fi_context, &psm2_req);
if (err) {
PSMX2_STATUS_ERROR(status) = err;
return psmx2_errno(err);
}
}
}
if (multi_recv && recv_len < recv_req->min_buf_size)
rep->comp_flag |= FI_MULTI_RECV;
return 0;
}
