static ssize_t psmx2_ep_cancel(fid_t fid, void *context) { struct psmx2_fid_ep *ep; psm2_mq_status2_t status; struct fi_context *fi_context = context; uint64_t flags; struct psmx2_cq_event *event; int err; ep = container_of(fid, struct psmx2_fid_ep, ep.fid); if (!ep->domain) return -FI_EBADF; if (!fi_context) return -FI_EINVAL; switch (PSMX2_CTXT_TYPE(fi_context)) { case PSMX2_TRECV_CONTEXT: flags = FI_RECV | FI_TAGGED; break; case PSMX2_RECV_CONTEXT: case PSMX2_MULTI_RECV_CONTEXT: flags = FI_RECV | FI_MSG; break; default: return -FI_EOPNOTSUPP; } err = psm2_mq_cancel((psm2_mq_req_t *)&PSMX2_CTXT_REQ(fi_context)); if (err == PSM2_OK) { err = psm2_mq_test2((psm2_mq_req_t *)&PSMX2_CTXT_REQ(fi_context), &status); if (err == PSM2_OK && ep->recv_cq) { event = psmx2_cq_create_event( ep->recv_cq, status.context, NULL, /* buf */ flags, 0, /* len */ 0, /* data */ 0, /* tag */ 0 /* olen */, -FI_ECANCELED); if (event) psmx2_cq_enqueue_event(ep->recv_cq, event); else return -FI_ENOMEM; } } return psmx2_errno(err); }
/* op_flags=0, FI_SELECTIVE_COMPLETION not set, FI_AV_TABLE */ static ssize_t psmx2_tagged_recv_no_flag_av_table(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag, psm2_tagsel; uint32_t tag32, tagsel32; struct fi_context *fi_context; size_t idx; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_SIZE(fi_context) = len; if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) { av = ep_priv->av; if (av && PSMX2_SEP_ADDR_TEST(src_addr)) { psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr); vlane = 0; } else { idx = (size_t)src_addr; if (idx >= av->last) return -FI_EINVAL; psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane); tagsel32 = ~PSMX2_IOV_BIT; } else { psm2_epaddr = 0; tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane); tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS); } PSMX2_SET_TAG(psm2_tag, tag, tag32); PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32); err = psm2_mq_irecv2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, 0, buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
/* op_flags=0, FI_SELECTIVE_COMPLETION not set, FI_AV_TABLE */ static ssize_t psmx2_tagged_send_no_flag_av_table(struct fid_ep *ep, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, uint64_t tag, void *context) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag; uint32_t tag32; struct fi_context *fi_context; size_t idx; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); av = ep_priv->av; if (av && PSMX2_SEP_ADDR_TEST(dest_addr)) { psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, dest_addr); vlane = 0; } else { idx = (size_t)dest_addr; if (idx >= av->last) return -FI_EINVAL; psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } tag32 = PSMX2_TAG32(0, ep_priv->vlane, vlane); PSMX2_SET_TAG(psm2_tag, tag, tag32); fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_TSEND_CONTEXT; PSMX2_CTXT_USER(fi_context) = (void *)buf; PSMX2_CTXT_EP(fi_context) = ep_priv; err = psm2_mq_isend2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr, 0, &psm2_tag, buf, len, (void*)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
/* op_flags=0, FI_SELECTIVE_COMPLETION not set, FI_AV_MAP */ static ssize_t psmx2_tagged_recv_no_flag_av_map(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context) { struct psmx2_fid_ep *ep_priv; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag, psm2_tagsel; uint32_t tag32, tagsel32; struct fi_context *fi_context; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_SIZE(fi_context) = len; if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) { psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr); vlane = PSMX2_ADDR_TO_VL(src_addr); tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane); tagsel32 = ~PSMX2_IOV_BIT; } else { psm2_epaddr = 0; tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane); tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS); } PSMX2_SET_TAG(psm2_tag, tag, tag32); PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32); err = psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, 0, buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
/* op_flags=0, FI_SELECTIVE_COMPLETION not set, FI_AV_MAP */ static ssize_t psmx2_tagged_send_no_flag_av_map(struct fid_ep *ep, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, uint64_t tag, void *context) { struct psmx2_fid_ep *ep_priv; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag; uint32_t tag32; struct fi_context *fi_context; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr); vlane = PSMX2_ADDR_TO_VL(dest_addr); tag32 = PSMX2_TAG32(0, ep_priv->vlane, vlane); PSMX2_SET_TAG(psm2_tag, tag, tag32); fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_TSEND_CONTEXT; PSMX2_CTXT_USER(fi_context) = (void *)buf; PSMX2_CTXT_EP(fi_context) = ep_priv; err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, 0, &psm2_tag, buf, len, (void*)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
ssize_t psmx2_recv_generic(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, void *context, uint64_t flags) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag, psm2_tagsel; uint32_t tag32, tagsel32; struct fi_context *fi_context; int recv_flag = 0; size_t idx; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_TRIGGER) { struct psmx2_trigger *trigger; struct fi_triggered_context *ctxt = context; trigger = calloc(1, sizeof(*trigger)); if (!trigger) return -FI_ENOMEM; trigger->op = PSMX2_TRIGGERED_RECV; trigger->cntr = container_of(ctxt->trigger.threshold.cntr, struct psmx2_fid_cntr, cntr); trigger->threshold = ctxt->trigger.threshold.threshold; trigger->recv.ep = ep; trigger->recv.buf = buf; trigger->recv.len = len; trigger->recv.desc = desc; trigger->recv.src_addr = src_addr; trigger->recv.context = context; trigger->recv.flags = flags & ~FI_TRIGGER; psmx2_cntr_add_trigger(trigger->cntr, trigger); return 0; } if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) { av = ep_priv->av; if (av && av->type == FI_AV_TABLE) { idx = (size_t)src_addr; if (idx >= av->last) return -FI_EINVAL; psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } else { psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr); vlane = PSMX2_ADDR_TO_VL(src_addr); } tag32 = PSMX2_TAG32(PSMX2_MSG_BIT, vlane, ep_priv->vlane); tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_IMM_BIT); } else { psm2_epaddr = 0; tag32 = PSMX2_TAG32(PSMX2_MSG_BIT, 0, ep_priv->vlane); tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_IMM_BIT | PSMX2_SRC_BITS); } PSMX2_SET_TAG(psm2_tag, 0ULL, tag32); PSMX2_SET_TAG(psm2_tagsel, 0ULL, tagsel32); if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION)) { fi_context = psmx2_ep_get_op_context(ep_priv); PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_SIZE(fi_context) = len; } else { if (!context) return -FI_EINVAL; fi_context = context; if (flags & FI_MULTI_RECV) { struct psmx2_multi_recv *req; req = calloc(1, sizeof(*req)); if (!req) return -FI_ENOMEM; req->src_addr = psm2_epaddr; req->tag = psm2_tag; req->tagsel = psm2_tagsel; req->flag = recv_flag; req->buf = buf; req->len = len; req->offset = 0; req->min_buf_size = ep_priv->min_multi_recv; req->context = fi_context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_MULTI_RECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = req; if (len > PSMX2_MAX_MSG_SIZE) len = PSMX2_MAX_MSG_SIZE; } else { PSMX2_CTXT_TYPE(fi_context) = PSMX2_RECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; } PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_SIZE(fi_context) = len; } err = psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, recv_flag, buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); if (fi_context == context) PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
ssize_t psmx2_sendv_generic(struct fid_ep *ep, const struct iovec *iov, void *desc, size_t count, fi_addr_t dest_addr, void *context, uint64_t flags, uint64_t data) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag; uint32_t tag32, tag32_base; struct fi_context * fi_context; int send_flag = 0; int err; size_t idx; int no_completion = 0; struct psmx2_cq_event *event; size_t real_count; size_t len, total_len; char *p; uint32_t *q; int i; struct psmx2_sendv_request *req; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_TRIGGER) { struct psmx2_trigger *trigger; struct fi_triggered_context *ctxt = context; trigger = calloc(1, sizeof(*trigger)); if (!trigger) return -FI_ENOMEM; trigger->op = PSMX2_TRIGGERED_SENDV; trigger->cntr = container_of(ctxt->trigger.threshold.cntr, struct psmx2_fid_cntr, cntr); trigger->threshold = ctxt->trigger.threshold.threshold; trigger->sendv.ep = ep; trigger->sendv.iov = iov; trigger->sendv.desc = desc; trigger->sendv.count = count; trigger->sendv.dest_addr = dest_addr; trigger->sendv.context = context; trigger->sendv.flags = flags & ~FI_TRIGGER; trigger->sendv.data = data; psmx2_cntr_add_trigger(trigger->cntr, trigger); return 0; } total_len = 0; real_count = 0; for (i=0; i<count; i++) { if (iov[i].iov_len) { total_len += iov[i].iov_len; real_count++; } } req = malloc(sizeof(*req)); if (!req) return -FI_ENOMEM; if (total_len <= PSMX2_IOV_BUF_SIZE) { req->iov_protocol = PSMX2_IOV_PROTO_PACK; p = req->buf; for (i=0; i<count; i++) { if (iov[i].iov_len) { memcpy(p, iov[i].iov_base, iov[i].iov_len); p += iov[i].iov_len; } } tag32_base = PSMX2_MSG_BIT; len = total_len; } else { req->iov_protocol = PSMX2_IOV_PROTO_MULTI; req->iov_done = 0; req->iov_info.seq_num = (++ep_priv->iov_seq_num) % PSMX2_IOV_MAX_SEQ_NUM + 1; req->iov_info.count = (uint32_t)real_count; req->iov_info.total_len = (uint32_t)total_len; q = req->iov_info.len; for (i=0; i<count; i++) { if (iov[i].iov_len) *q++ = (uint32_t)iov[i].iov_len; } tag32_base = PSMX2_MSG_BIT | PSMX2_IOV_BIT; len = (3 + real_count) * sizeof(uint32_t); } av = ep_priv->av; if (av && av->type == FI_AV_TABLE) { idx = (size_t)dest_addr; if (idx >= av->last) { free(req); return -FI_EINVAL; } psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } else { psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr); vlane = PSMX2_ADDR_TO_VL(dest_addr); } tag32 = PSMX2_TAG32(tag32_base, ep_priv->vlane, vlane); if (flags & FI_REMOTE_CQ_DATA) tag32 |= PSMX2_IMM_BIT; PSMX2_SET_TAG(psm2_tag, data, tag32); if ((flags & PSMX2_NO_COMPLETION) || (ep_priv->send_selective_completion && !(flags & FI_COMPLETION))) no_completion = 1; if (flags & FI_INJECT) { if (len > PSMX2_INJECT_SIZE) { free(req); return -FI_EMSGSIZE; } err = psm2_mq_send2(ep_priv->domain->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, req->buf, len); free(req); if (err != PSM2_OK) return psmx2_errno(err); if (ep_priv->send_cntr) psmx2_cntr_inc(ep_priv->send_cntr); if (ep_priv->send_cq && !no_completion) { event = psmx2_cq_create_event( ep_priv->send_cq, context, NULL, flags, len, (uint64_t) data, 0 /* tag */, 0 /* olen */, 0 /* err */); if (event) psmx2_cq_enqueue_event(ep_priv->send_cq, event); else return -FI_ENOMEM; } return 0; } req->no_completion = no_completion; req->user_context = context; req->comp_flag = FI_MSG; fi_context = &req->fi_context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_SENDV_CONTEXT; PSMX2_CTXT_USER(fi_context) = req; PSMX2_CTXT_EP(fi_context) = ep_priv; err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, req->buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) { free(req); return psmx2_errno(err); } PSMX2_CTXT_REQ(fi_context) = psm2_req; if (req->iov_protocol == PSMX2_IOV_PROTO_MULTI) { fi_context = &req->fi_context_iov; PSMX2_CTXT_TYPE(fi_context) = PSMX2_IOV_SEND_CONTEXT; PSMX2_CTXT_USER(fi_context) = req; PSMX2_CTXT_EP(fi_context) = ep_priv; tag32 &= ~PSMX2_IOV_BIT; PSMX2_TAG32_SET_SEQ(tag32, req->iov_info.seq_num); PSMX2_SET_TAG(psm2_tag, data, tag32); for (i=0; i<count; i++) { if (iov[i].iov_len) { err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, iov[i].iov_base, iov[i].iov_len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); } } } return 0; }
ssize_t psmx2_send_generic(struct fid_ep *ep, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, void *context, uint64_t flags, uint64_t data) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag; uint32_t tag32; struct fi_context * fi_context; int send_flag = 0; int err; size_t idx; int no_completion = 0; struct psmx2_cq_event *event; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_TRIGGER) { struct psmx2_trigger *trigger; struct fi_triggered_context *ctxt = context; trigger = calloc(1, sizeof(*trigger)); if (!trigger) return -FI_ENOMEM; trigger->op = PSMX2_TRIGGERED_SEND; trigger->cntr = container_of(ctxt->trigger.threshold.cntr, struct psmx2_fid_cntr, cntr); trigger->threshold = ctxt->trigger.threshold.threshold; trigger->send.ep = ep; trigger->send.buf = buf; trigger->send.len = len; trigger->send.desc = desc; trigger->send.dest_addr = dest_addr; trigger->send.context = context; trigger->send.flags = flags & ~FI_TRIGGER; trigger->send.data = data; psmx2_cntr_add_trigger(trigger->cntr, trigger); return 0; } av = ep_priv->av; if (av && av->type == FI_AV_TABLE) { idx = (size_t)dest_addr; if (idx >= av->last) return -FI_EINVAL; psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } else { psm2_epaddr = PSMX2_ADDR_TO_EP(dest_addr); vlane = PSMX2_ADDR_TO_VL(dest_addr); } tag32 = PSMX2_TAG32(PSMX2_MSG_BIT, ep_priv->vlane, vlane); if (flags & FI_REMOTE_CQ_DATA) tag32 |= PSMX2_IMM_BIT; PSMX2_SET_TAG(psm2_tag, data, tag32); if ((flags & PSMX2_NO_COMPLETION) || (ep_priv->send_selective_completion && !(flags & FI_COMPLETION))) no_completion = 1; if (flags & FI_INJECT) { if (len > PSMX2_INJECT_SIZE) return -FI_EMSGSIZE; err = psm2_mq_send2(ep_priv->domain->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, buf, len); if (err != PSM2_OK) return psmx2_errno(err); if (ep_priv->send_cntr) psmx2_cntr_inc(ep_priv->send_cntr); if (ep_priv->send_cq && !no_completion) { event = psmx2_cq_create_event( ep_priv->send_cq, context, (void *)buf, flags, len, (uint64_t) data, 0 /* tag */, 0 /* olen */, 0 /* err */); if (event) psmx2_cq_enqueue_event(ep_priv->send_cq, event); else return -FI_ENOMEM; } return 0; } if (no_completion && !context) { fi_context = &ep_priv->nocomp_send_context; } else { if (!context) return -FI_EINVAL; fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_SEND_CONTEXT; PSMX2_CTXT_USER(fi_context) = (void *)buf; PSMX2_CTXT_EP(fi_context) = ep_priv; } err = psm2_mq_isend2(ep_priv->domain->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); if (fi_context == context) PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
static ssize_t psmx2_tagged_peek_generic(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context, uint64_t flags) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; struct psmx2_cq_event *event; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t req; psm2_mq_status2_t psm2_status; psm2_mq_tag_t psm2_tag, psm2_tagsel; uint32_t tag32, tagsel32; size_t idx; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (src_addr != FI_ADDR_UNSPEC) { av = ep_priv->av; if (av && PSMX2_SEP_ADDR_TEST(src_addr)) { psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr); vlane = 0; } else if (av && av->type == FI_AV_TABLE) { idx = (size_t)src_addr; if (idx >= av->last) return -FI_EINVAL; psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } else { psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr); vlane = PSMX2_ADDR_TO_VL(src_addr); } tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane); tagsel32 = -1; } else { psm2_epaddr = 0; tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane); tagsel32 = ~PSMX2_SRC_BITS; } PSMX2_SET_TAG(psm2_tag, tag, tag32); PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32); if (flags & (FI_CLAIM | FI_DISCARD)) err = psm2_mq_improbe2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, &req, &psm2_status); else err = psm2_mq_iprobe2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, &psm2_status); switch (err) { case PSM2_OK: if (ep_priv->recv_cq) { if (flags & FI_CLAIM) { if (context) PSMX2_CTXT_REQ((struct fi_context *)context) = req; } else if (flags & FI_DISCARD) { if (!psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0, NULL, 0, req, &req)) psm2_mq_wait2(&req, NULL); } tag = PSMX2_GET_TAG64(psm2_status.msg_tag); len = psm2_status.msg_length; event = psmx2_cq_create_event( ep_priv->recv_cq, context, /* op_context */ NULL, /* buf */ flags|FI_RECV|FI_TAGGED,/* flags */ len, /* len */ 0, /* data */ tag, /* tag */ len, /* olen */ 0); /* err */ if (!event) return -FI_ENOMEM; vlane = PSMX2_TAG32_GET_SRC(psm2_status.msg_tag.tag2); event->source_is_valid = 1; event->source = PSMX2_EP_TO_ADDR(psm2_status.msg_peer, vlane); event->source_av = ep_priv->av; psmx2_cq_enqueue_event(ep_priv->recv_cq, event); } return 0; case PSM2_MQ_NO_COMPLETIONS: if (ep_priv->recv_cq) { event = psmx2_cq_create_event( ep_priv->recv_cq, context, /* op_context */ NULL, /* buf */ flags|FI_RECV|FI_TAGGED,/* flags */ len, /* len */ 0, /* data */ tag, /* tag */ len, /* olen */ -FI_ENOMSG); /* err */ if (!event) return -FI_ENOMEM; event->source = 0; psmx2_cq_enqueue_event(ep_priv->recv_cq, event); } return 0; default: return psmx2_errno(err); } }
ssize_t psmx2_tagged_recv_generic(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context, uint64_t flags) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag, psm2_tagsel; uint32_t tag32, tagsel32; struct fi_context *fi_context; size_t idx; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_PEEK) return psmx2_tagged_peek_generic(ep, buf, len, desc, src_addr, tag, ignore, context, flags); if (flags & FI_TRIGGER) { struct psmx2_trigger *trigger; struct fi_triggered_context *ctxt = context; trigger = calloc(1, sizeof(*trigger)); if (!trigger) return -FI_ENOMEM; trigger->op = PSMX2_TRIGGERED_TRECV; trigger->cntr = container_of(ctxt->trigger.threshold.cntr, struct psmx2_fid_cntr, cntr); trigger->threshold = ctxt->trigger.threshold.threshold; trigger->trecv.ep = ep; trigger->trecv.buf = buf; trigger->trecv.len = len; trigger->trecv.desc = desc; trigger->trecv.src_addr = src_addr; trigger->trecv.tag = tag; trigger->trecv.ignore = ignore; trigger->trecv.context = context; trigger->trecv.flags = flags & ~FI_TRIGGER; psmx2_cntr_add_trigger(trigger->cntr, trigger); return 0; } if (flags & FI_CLAIM) { if (!context) return -FI_EINVAL; if (flags & FI_DISCARD) { psm2_mq_status2_t psm2_status; struct psmx2_cq_event *event; fi_context = context; psm2_req = PSMX2_CTXT_REQ(fi_context); err = psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0, NULL, 0, context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); psm2_mq_wait2(&psm2_req, &psm2_status); if (ep_priv->recv_cq && (!ep_priv->recv_selective_completion || (flags & FI_COMPLETION))) { tag = PSMX2_GET_TAG64(psm2_status.msg_tag); event = psmx2_cq_create_event( ep_priv->recv_cq, context, /* op_context */ NULL, /* buf */ flags|FI_RECV|FI_TAGGED,/* flags */ 0, /* len */ 0, /* data */ tag, /* tag */ 0, /* olen */ 0); /* err */ if (!event) return -FI_ENOMEM; vlane = PSMX2_TAG32_GET_SRC(psm2_status.msg_tag.tag2); event->source_is_valid = 1; event->source = PSMX2_EP_TO_ADDR(psm2_status.msg_peer, vlane); event->source_av = ep_priv->av; psmx2_cq_enqueue_event(ep_priv->recv_cq, event); } if (ep_priv->recv_cntr) psmx2_cntr_inc(ep_priv->recv_cntr); return 0; } fi_context = context; psm2_req = PSMX2_CTXT_REQ(fi_context); PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_EP(fi_context) = ep_priv; err = psm2_mq_imrecv(ep_priv->trx_ctxt->psm2_mq, 0, buf, len, context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; } if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION)) { fi_context = psmx2_ep_get_op_context(ep_priv); PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_SIZE(fi_context) = len; } else { if (!context) return -FI_EINVAL; fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_SIZE(fi_context) = len; } if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) { av = ep_priv->av; if (av && PSMX2_SEP_ADDR_TEST(src_addr)) { psm2_epaddr = psmx2_av_translate_sep(av, ep_priv->trx_ctxt, src_addr); vlane = 0; } else if (av && av->type == FI_AV_TABLE) { idx = (size_t)src_addr; if (idx >= av->last) return -FI_EINVAL; psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } else { psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr); vlane = PSMX2_ADDR_TO_VL(src_addr); } tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane); tagsel32 = ~PSMX2_IOV_BIT; } else { psm2_epaddr = 0; tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane); tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS); } PSMX2_SET_TAG(psm2_tag, tag, tag32); PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32); err = psm2_mq_irecv2(ep_priv->trx_ctxt->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, 0, buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); if (fi_context == context) PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
ssize_t psmx2_tagged_recv_generic(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, uint64_t tag, uint64_t ignore, void *context, uint64_t flags) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; uint8_t vlane; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag, psm2_tagsel; uint32_t tag32, tagsel32; struct fi_context *fi_context; size_t idx; int err; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_PEEK) return psmx2_tagged_peek_generic(ep, buf, len, desc, src_addr, tag, ignore, context, flags); if (flags & FI_TRIGGER) { struct psmx2_trigger *trigger; struct fi_triggered_context *ctxt = context; trigger = calloc(1, sizeof(*trigger)); if (!trigger) return -FI_ENOMEM; trigger->op = PSMX2_TRIGGERED_TRECV; trigger->cntr = container_of(ctxt->trigger.threshold.cntr, struct psmx2_fid_cntr, cntr); trigger->threshold = ctxt->trigger.threshold.threshold; trigger->trecv.ep = ep; trigger->trecv.buf = buf; trigger->trecv.len = len; trigger->trecv.desc = desc; trigger->trecv.src_addr = src_addr; trigger->trecv.tag = tag; trigger->trecv.ignore = ignore; trigger->trecv.context = context; trigger->trecv.flags = flags & ~FI_TRIGGER; psmx2_cntr_add_trigger(trigger->cntr, trigger); return 0; } if (flags & FI_CLAIM) { if (!context) return -FI_EINVAL; /* TODO: handle FI_DISCARD */ fi_context = context; psm2_req = PSMX2_CTXT_REQ(fi_context); PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_EP(fi_context) = ep_priv; err = psm2_mq_imrecv(ep_priv->domain->psm2_mq, 0, buf, len, context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; } if (ep_priv->recv_selective_completion && !(flags & FI_COMPLETION)) { fi_context = psmx2_ep_get_op_context(ep_priv); PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT_ALLOC; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_SIZE(fi_context) = len; } else { if (!context) return -FI_EINVAL; fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_TRECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_SIZE(fi_context) = len; } if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) { av = ep_priv->av; if (av && av->type == FI_AV_TABLE) { idx = (size_t)src_addr; if (idx >= av->last) return -FI_EINVAL; psm2_epaddr = av->epaddrs[idx]; vlane = av->vlanes[idx]; } else { psm2_epaddr = PSMX2_ADDR_TO_EP(src_addr); vlane = PSMX2_ADDR_TO_VL(src_addr); } tag32 = PSMX2_TAG32(0, vlane, ep_priv->vlane); tagsel32 = ~PSMX2_IOV_BIT; } else { psm2_epaddr = 0; tag32 = PSMX2_TAG32(0, 0, ep_priv->vlane); tagsel32 = ~(PSMX2_IOV_BIT | PSMX2_SRC_BITS); } PSMX2_SET_TAG(psm2_tag, tag, tag32); PSMX2_SET_TAG(psm2_tagsel, ~ignore, tagsel32); err = psm2_mq_irecv2(ep_priv->domain->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, 0, buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); if (fi_context == context) PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }
ssize_t psmx2_recv_generic(struct fid_ep *ep, void *buf, size_t len, void *desc, fi_addr_t src_addr, void *context, uint64_t flags) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag, psm2_tagsel; struct fi_context *fi_context; int recv_flag = 0; int err; int enable_completion; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_TRIGGER) return psmx2_trigger_queue_recv(ep, buf, len, desc, src_addr, context, flags); if ((ep_priv->caps & FI_DIRECTED_RECV) && src_addr != FI_ADDR_UNSPEC) { av = ep_priv->av; assert(av); psm2_epaddr = psmx2_av_translate_addr(av, ep_priv->rx, src_addr, av->type); } else { psm2_epaddr = 0; } PSMX2_SET_TAG(psm2_tag, 0ULL, 0, PSMX2_TYPE_MSG); PSMX2_SET_MASK(psm2_tagsel, PSMX2_MATCH_NONE, PSMX2_TYPE_MASK); enable_completion = !ep_priv->recv_selective_completion || (flags & FI_COMPLETION); if (enable_completion) { assert(context); fi_context = context; if (flags & FI_MULTI_RECV) { struct psmx2_multi_recv *req; req = calloc(1, sizeof(*req)); if (!req) return -FI_ENOMEM; req->src_addr = psm2_epaddr; req->tag = psm2_tag; req->tagsel = psm2_tagsel; req->flag = recv_flag; req->buf = buf; req->len = len; req->offset = 0; req->min_buf_size = ep_priv->min_multi_recv; req->context = fi_context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_MULTI_RECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = req; if (len > PSMX2_MAX_MSG_SIZE) len = PSMX2_MAX_MSG_SIZE; } else { PSMX2_CTXT_TYPE(fi_context) = PSMX2_RECV_CONTEXT; PSMX2_CTXT_USER(fi_context) = buf; } PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_SIZE(fi_context) = len; } else { PSMX2_EP_GET_OP_CONTEXT(ep_priv, fi_context); #if !PSMX2_USE_REQ_CONTEXT PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_SIZE(fi_context) = len; #endif } err = psm2_mq_irecv2(ep_priv->rx->psm2_mq, psm2_epaddr, &psm2_tag, &psm2_tagsel, recv_flag, buf, len, (void *)fi_context, &psm2_req); if (OFI_UNLIKELY(err != PSM2_OK)) return psmx2_errno(err); if (enable_completion) { PSMX2_CTXT_REQ(fi_context) = psm2_req; } else { #if PSMX2_USE_REQ_CONTEXT PSMX2_REQ_GET_OP_CONTEXT(psm2_req, fi_context); PSMX2_CTXT_TYPE(fi_context) = PSMX2_NOCOMP_RECV_CONTEXT; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_CTXT_USER(fi_context) = buf; PSMX2_CTXT_SIZE(fi_context) = len; #endif } return 0; }
ssize_t psmx2_sendv_generic(struct fid_ep *ep, const struct iovec *iov, void **desc, size_t count, fi_addr_t dest_addr, void *context, uint64_t flags, uint64_t data) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag; uint32_t msg_flags; struct fi_context * fi_context; int send_flag = 0; int err; int no_completion = 0; struct psmx2_cq_event *event; size_t real_count; size_t len, total_len; char *p; uint32_t *q; int i, j; struct psmx2_sendv_request *req; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_TRIGGER) return psmx2_trigger_queue_sendv(ep, iov, desc, count, dest_addr, context, flags, data); total_len = 0; real_count = 0; for (i=0; i<count; i++) { if (iov[i].iov_len) { total_len += iov[i].iov_len; real_count++; j = i; } } if (real_count == 1) return psmx2_send_generic(ep, iov[j].iov_base, iov[j].iov_len, desc ? desc[j] : NULL, dest_addr, context, flags, data); req = malloc(sizeof(*req)); if (!req) return -FI_ENOMEM; if (total_len <= PSMX2_IOV_BUF_SIZE) { req->iov_protocol = PSMX2_IOV_PROTO_PACK; p = req->buf; for (i=0; i<count; i++) { if (iov[i].iov_len) { memcpy(p, iov[i].iov_base, iov[i].iov_len); p += iov[i].iov_len; } } msg_flags = PSMX2_TYPE_MSG; len = total_len; } else { req->iov_protocol = PSMX2_IOV_PROTO_MULTI; req->iov_done = 0; req->iov_info.seq_num = (++ep_priv->iov_seq_num) % PSMX2_IOV_MAX_SEQ_NUM + 1; req->iov_info.count = (uint32_t)real_count; req->iov_info.total_len = (uint32_t)total_len; q = req->iov_info.len; for (i=0; i<count; i++) { if (iov[i].iov_len) *q++ = (uint32_t)iov[i].iov_len; } msg_flags = PSMX2_TYPE_MSG | PSMX2_IOV_BIT; len = (3 + real_count) * sizeof(uint32_t); } av = ep_priv->av; assert(av); psm2_epaddr = psmx2_av_translate_addr(av, ep_priv->tx, dest_addr, av->type); if (flags & FI_REMOTE_CQ_DATA) msg_flags |= PSMX2_IMM_BIT; PSMX2_SET_TAG(psm2_tag, 0ULL, data, msg_flags); if ((flags & PSMX2_NO_COMPLETION) || (ep_priv->send_selective_completion && !(flags & FI_COMPLETION))) no_completion = 1; if (flags & FI_INJECT) { if (len > psmx2_env.inject_size) { free(req); return -FI_EMSGSIZE; } err = psm2_mq_send2(ep_priv->tx->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, req->buf, len); free(req); if (err != PSM2_OK) return psmx2_errno(err); if (ep_priv->send_cntr) psmx2_cntr_inc(ep_priv->send_cntr, 0); if (ep_priv->send_cq && !no_completion) { event = psmx2_cq_create_event( ep_priv->send_cq, context, NULL, flags, len, (uint64_t) data, 0 /* tag */, 0 /* olen */, 0 /* err */); if (event) psmx2_cq_enqueue_event(ep_priv->send_cq, event); else return -FI_ENOMEM; } return 0; } req->no_completion = no_completion; req->user_context = context; req->comp_flag = FI_MSG; fi_context = &req->fi_context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_SENDV_CONTEXT; PSMX2_CTXT_USER(fi_context) = req; PSMX2_CTXT_EP(fi_context) = ep_priv; err = psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, req->buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) { free(req); return psmx2_errno(err); } PSMX2_CTXT_REQ(fi_context) = psm2_req; if (req->iov_protocol == PSMX2_IOV_PROTO_MULTI) { fi_context = &req->fi_context_iov; PSMX2_CTXT_TYPE(fi_context) = PSMX2_IOV_SEND_CONTEXT; PSMX2_CTXT_USER(fi_context) = req; PSMX2_CTXT_EP(fi_context) = ep_priv; PSMX2_SET_TAG(psm2_tag, req->iov_info.seq_num, 0, PSMX2_TYPE_IOV_PAYLOAD); for (i=0; i<count; i++) { if (iov[i].iov_len) { err = psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, iov[i].iov_base, iov[i].iov_len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); } } } return 0; }
ssize_t psmx2_send_generic(struct fid_ep *ep, const void *buf, size_t len, void *desc, fi_addr_t dest_addr, void *context, uint64_t flags, uint64_t data) { struct psmx2_fid_ep *ep_priv; struct psmx2_fid_av *av; psm2_epaddr_t psm2_epaddr; psm2_mq_req_t psm2_req; psm2_mq_tag_t psm2_tag; struct fi_context * fi_context; int send_flag = 0; int err; int no_completion = 0; struct psmx2_cq_event *event; int have_data = (flags & FI_REMOTE_CQ_DATA) > 0; ep_priv = container_of(ep, struct psmx2_fid_ep, ep); if (flags & FI_TRIGGER) return psmx2_trigger_queue_send(ep, buf, len, desc, dest_addr, context, flags, data); av = ep_priv->av; assert(av); psm2_epaddr = psmx2_av_translate_addr(av, ep_priv->tx, dest_addr, av->type); PSMX2_SET_TAG(psm2_tag, 0, data, PSMX2_TYPE_MSG | PSMX2_IMM_BIT_SET(have_data)); if ((flags & PSMX2_NO_COMPLETION) || (ep_priv->send_selective_completion && !(flags & FI_COMPLETION))) no_completion = 1; if (flags & FI_INJECT) { if (len > psmx2_env.inject_size) return -FI_EMSGSIZE; err = psm2_mq_send2(ep_priv->tx->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, buf, len); if (err != PSM2_OK) return psmx2_errno(err); if (ep_priv->send_cntr) psmx2_cntr_inc(ep_priv->send_cntr, 0); if (ep_priv->send_cq && !no_completion) { event = psmx2_cq_create_event( ep_priv->send_cq, context, (void *)buf, flags, len, (uint64_t) data, 0 /* tag */, 0 /* olen */, 0 /* err */); if (event) psmx2_cq_enqueue_event(ep_priv->send_cq, event); else return -FI_ENOMEM; } return 0; } if (no_completion) { fi_context = &ep_priv->nocomp_send_context; } else { assert(context); fi_context = context; PSMX2_CTXT_TYPE(fi_context) = PSMX2_SEND_CONTEXT; PSMX2_CTXT_USER(fi_context) = (void *)buf; PSMX2_CTXT_EP(fi_context) = ep_priv; } err = psm2_mq_isend2(ep_priv->tx->psm2_mq, psm2_epaddr, send_flag, &psm2_tag, buf, len, (void *)fi_context, &psm2_req); if (err != PSM2_OK) return psmx2_errno(err); if (fi_context == context) PSMX2_CTXT_REQ(fi_context) = psm2_req; return 0; }