int __recvpath psmi_mq_handle_rts(psm_mq_t mq, uint64_t tag, uintptr_t send_buf, uint32_t send_msglen, psm_epaddr_t peer, mq_rts_callback_fn_t cb, psm_mq_req_t *req_o) { psm_mq_req_t req; uint32_t msglen; int rc; PSMI_PLOCK_ASSERT(); req = mq_req_match(&(mq->expected_q), tag, 1); if (req) { /* we have a match, no need to callback */ msglen = mq_set_msglen(req, req->buf_len, send_msglen); req->type = MQE_TYPE_RECV; req->state = MQ_STATE_MATCHED; req->tag = tag; req->recv_msgoff = 0; req->rts_peer = peer; req->rts_sbuf = send_buf; *req_o = req; /* yes match */ rc = MQ_RET_MATCH_OK; } else { /* No match, keep track of callback */ req = psmi_mq_req_alloc(mq, MQE_TYPE_RECV); psmi_assert(req != NULL); req->type = MQE_TYPE_RECV; /* We don't know recv_msglen yet but we set it here for * mq_iprobe */ req->send_msglen = req->recv_msglen = send_msglen; req->state = MQ_STATE_UNEXP_RV; req->tag = tag; req->rts_callback = cb; req->recv_msgoff = 0; req->rts_peer = peer; req->rts_sbuf = send_buf; mq_sq_append(&mq->unexpected_q, req); *req_o = req; /* no match, will callback */ rc = MQ_RET_UNEXP_OK; } _IPATH_VDBG("from=%s match=%s (req=%p) mqtag=%" PRIx64" recvlen=%d " "sendlen=%d errcode=%d\n", psmi_epaddr_get_name(peer->epid), rc == MQ_RET_MATCH_OK ? "YES" : "NO", req, req->tag, req->recv_msglen, req->send_msglen, req->error_code); return rc; }
/* * Note, epaddr is the master. */ int __recvpath psmi_mq_handle_outoforder_queue(psm_epaddr_t epaddr) { psm_mq_t mq = epaddr->ep->mq; psm_mq_req_t ureq, ereq; uint32_t msglen; next_ooo: ureq = mq_ooo_match(&epaddr->outoforder_q, epaddr->mctxt_recv_seqnum); if (ureq == NULL) return 0; epaddr->mctxt_recv_seqnum++; epaddr->outoforder_c--; ereq = mq_req_match(&(mq->expected_q), ureq->tag, 1); if (ereq == NULL) { mq_sq_append(&mq->unexpected_q, ureq); if (epaddr->outoforder_c) goto next_ooo; return 0; } psmi_assert(MQE_TYPE_IS_RECV(ereq->type)); ereq->tag = ureq->tag; msglen = mq_set_msglen(ereq, ereq->buf_len, ureq->send_msglen); switch (ureq->state) { case MQ_STATE_COMPLETE: if (ureq->buf != NULL) { /* 0-byte don't alloc a sysbuf */ psmi_mq_mtucpy(ereq->buf, (const void *)ureq->buf, msglen); psmi_mq_sysbuf_free(mq, ureq->buf); } ereq->state = MQ_STATE_COMPLETE; mq_qq_append(&mq->completed_q, ereq); break; case MQ_STATE_UNEXP: /* not done yet */ ereq->type = ureq->type; ereq->egrid = ureq->egrid; ereq->epaddr = ureq->epaddr; ereq->send_msgoff = ureq->send_msgoff; ereq->recv_msgoff = min(ureq->recv_msgoff, msglen); psmi_mq_mtucpy(ereq->buf, (const void *)ureq->buf, ereq->recv_msgoff); psmi_mq_sysbuf_free(mq, ureq->buf); ereq->state = MQ_STATE_MATCHED; STAILQ_INSERT_AFTER(&ureq->epaddr->mctxt_master->egrlong, ureq, ereq, nextq); STAILQ_REMOVE(&ureq->epaddr->mctxt_master->egrlong, ureq, psm_mq_req, nextq); break; case MQ_STATE_UNEXP_RV: /* rendez-vous ... */ ereq->state = MQ_STATE_MATCHED; ereq->rts_peer = ureq->rts_peer; ereq->rts_sbuf = ureq->rts_sbuf; ereq->send_msgoff = 0; ereq->rts_callback = ureq->rts_callback; ereq->rts_reqidx_peer = ureq->rts_reqidx_peer; ereq->type = ureq->type; ereq->rts_callback(ereq, 0); break; default: fprintf(stderr, "Unexpected state %d in req %p\n", ureq->state, ureq); fprintf(stderr, "type=%d, mq=%p, tag=%p\n", ureq->type, ureq->mq, (void *)(uintptr_t)ureq->tag); abort(); } psmi_mq_req_free(ureq); if (epaddr->outoforder_c) goto next_ooo; return 0; }
/* * This handles the regular (i.e. non-rendezvous MPI envelopes) */ int __recvpath psmi_mq_handle_envelope(psm_mq_t mq, uint16_t mode, psm_epaddr_t epaddr, uint64_t tag, psmi_egrid_t egrid, uint32_t send_msglen, const void *payload, uint32_t paylen) { psm_mq_req_t req; uint32_t msglen; int rc; psmi_assert(epaddr != NULL); req = mq_req_match(&(mq->expected_q), tag, 1); if (req) { /* we have a match */ psmi_assert(MQE_TYPE_IS_RECV(req->type)); req->tag = tag; msglen = mq_set_msglen(req, req->buf_len, send_msglen); _IPATH_VDBG("from=%s match=YES (req=%p) mode=%x mqtag=%" PRIx64" msglen=%d paylen=%d\n", psmi_epaddr_get_name(epaddr->epid), req, mode, tag, msglen, paylen); switch(mode) { case MQ_MSG_TINY: PSM_VALGRIND_DEFINE_MQ_RECV(req->buf, req->buf_len, msglen); mq_copy_tiny((uint32_t *)req->buf, (uint32_t *)payload, msglen); req->state = MQ_STATE_COMPLETE; mq_qq_append(&mq->completed_q, req); break; case MQ_MSG_SHORT: /* message fits in 1 payload */ PSM_VALGRIND_DEFINE_MQ_RECV(req->buf, req->buf_len, msglen); psmi_mq_mtucpy(req->buf, payload, msglen); req->state = MQ_STATE_COMPLETE; mq_qq_append(&mq->completed_q, req); break; case MQ_MSG_LONG: req->egrid = egrid; req->state = MQ_STATE_MATCHED; req->type |= MQE_TYPE_EGRLONG; req->send_msgoff = req->recv_msgoff = 0; STAILQ_INSERT_TAIL(&epaddr->mctxt_master->egrlong, req, nextq); _IPATH_VDBG("exp MSG_LONG %d of length %d bytes pay=%d\n", egrid.egr_msgno, msglen, paylen); if (paylen > 0) psmi_mq_handle_data(req, epaddr, egrid.egr_data, 0, payload, paylen); psmi_mq_handle_egrdata(mq, req, epaddr); break; default: psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, "Internal error, unknown packet 0x%x", mode); } mq->stats.rx_user_bytes += msglen; mq->stats.rx_user_num++; rc = MQ_RET_MATCH_OK; if (mode == MQ_MSG_LONG) return rc; } else rc = psmi_mq_handle_envelope_unexpected(mq, mode, epaddr, tag, egrid, send_msglen, payload, paylen); return rc; }