int __recvpath psmi_mq_handle_rts_outoforder(psm_mq_t mq, uint64_t tag, uintptr_t send_buf, uint32_t send_msglen, psm_epaddr_t peer, uint16_t msg_seqnum, mq_rts_callback_fn_t cb, psm_mq_req_t *req_o) { psm_mq_req_t req; PSMI_PLOCK_ASSERT(); req = psmi_mq_req_alloc(mq, MQE_TYPE_RECV); psmi_assert(req != NULL); /* We don't know recv_msglen yet but we set it here for * mq_iprobe */ req->send_msglen = req->recv_msglen = send_msglen; req->state = MQ_STATE_UNEXP_RV; req->tag = tag; req->rts_callback = cb; req->recv_msgoff = 0; req->send_msgoff = 0; req->rts_peer = peer; req->rts_sbuf = send_buf; req->msg_seqnum = msg_seqnum; mq_sq_append(&peer->mctxt_master->outoforder_q, req); peer->mctxt_master->outoforder_c++; *req_o = req; /* no match, will callback */ _IPATH_VDBG("from=%s match=%s (req=%p) mqtag=%" PRIx64" recvlen=%d " "sendlen=%d errcode=%d\n", psmi_epaddr_get_name(peer->epid), "NO", req, req->tag, req->recv_msglen, req->send_msglen, req->error_code); return MQ_RET_UNEXP_OK; }
int __recvpath psmi_mq_handle_data(psm_mq_req_t req, psm_epaddr_t epaddr, uint32_t egrid, uint32_t offset, const void *buf, uint32_t nbytes) { psm_mq_t mq; int rc; if (req == NULL) goto no_req; mq = req->mq; if (req->state == MQ_STATE_MATCHED) rc = MQ_RET_MATCH_OK; else { psmi_assert(req->state == MQ_STATE_UNEXP); rc = MQ_RET_UNEXP_OK; } psmi_assert(req->egrid.egr_data == egrid); psmi_mq_req_copy(req, epaddr, offset, buf, nbytes); if (req->send_msgoff == req->send_msglen) { if (req->type & MQE_TYPE_EGRLONG) { STAILQ_REMOVE(&epaddr->mctxt_master->egrlong, req, psm_mq_req, nextq); } if (req->state == MQ_STATE_MATCHED) { req->state = MQ_STATE_COMPLETE; mq_qq_append(&mq->completed_q, req); } else { /* MQ_STATE_UNEXP */ req->state = MQ_STATE_COMPLETE; } _IPATH_VDBG("epaddr=%s completed %d byte send, state=%d\n", psmi_epaddr_get_name(epaddr->epid), (int)req->send_msglen, req->state); } return rc; no_req: mq = epaddr->ep->mq; req = psmi_mq_req_alloc(mq, MQE_TYPE_RECV); psmi_assert(req != NULL); req->egrid.egr_data = egrid; req->recv_msgoff = offset; req->recv_msglen = nbytes; req->buf = psmi_mq_sysbuf_alloc(mq, nbytes); psmi_mq_mtucpy(req->buf, buf, nbytes); STAILQ_INSERT_TAIL(&epaddr->mctxt_master->egrdata, req, nextq); return MQ_RET_UNEXP_OK; }
int __recvpath psmi_mq_handle_rts(psm_mq_t mq, uint64_t tag, uintptr_t send_buf, uint32_t send_msglen, psm_epaddr_t peer, mq_rts_callback_fn_t cb, psm_mq_req_t *req_o) { psm_mq_req_t req; uint32_t msglen; int rc; PSMI_PLOCK_ASSERT(); req = mq_req_match(&(mq->expected_q), tag, 1); if (req) { /* we have a match, no need to callback */ msglen = mq_set_msglen(req, req->buf_len, send_msglen); req->type = MQE_TYPE_RECV; req->state = MQ_STATE_MATCHED; req->tag = tag; req->recv_msgoff = 0; req->rts_peer = peer; req->rts_sbuf = send_buf; *req_o = req; /* yes match */ rc = MQ_RET_MATCH_OK; } else { /* No match, keep track of callback */ req = psmi_mq_req_alloc(mq, MQE_TYPE_RECV); psmi_assert(req != NULL); req->type = MQE_TYPE_RECV; /* We don't know recv_msglen yet but we set it here for * mq_iprobe */ req->send_msglen = req->recv_msglen = send_msglen; req->state = MQ_STATE_UNEXP_RV; req->tag = tag; req->rts_callback = cb; req->recv_msgoff = 0; req->rts_peer = peer; req->rts_sbuf = send_buf; mq_sq_append(&mq->unexpected_q, req); *req_o = req; /* no match, will callback */ rc = MQ_RET_UNEXP_OK; } _IPATH_VDBG("from=%s match=%s (req=%p) mqtag=%" PRIx64" recvlen=%d " "sendlen=%d errcode=%d\n", psmi_epaddr_get_name(peer->epid), rc == MQ_RET_MATCH_OK ? "YES" : "NO", req, req->tag, req->recv_msglen, req->send_msglen, req->error_code); return rc; }
int __recvpath psmi_mq_handle_envelope_outoforder(psm_mq_t mq, uint16_t mode, psm_epaddr_t epaddr, uint16_t msg_seqnum, uint64_t tag, psmi_egrid_t egrid, uint32_t send_msglen, const void *payload, uint32_t paylen) { psm_mq_req_t req; uint32_t msglen; req = psmi_mq_req_alloc(mq, MQE_TYPE_RECV); psmi_assert(req != NULL); req->tag = tag; req->recv_msgoff = 0; req->recv_msglen = req->send_msglen = req->buf_len = msglen = send_msglen; _IPATH_VDBG( "from=%s match=NO (req=%p) mode=%x mqtag=%" PRIx64 " send_msglen=%d\n", psmi_epaddr_get_name(epaddr->epid), req, mode, tag, send_msglen); switch (mode) { case MQ_MSG_TINY: if (msglen > 0) { req->buf = psmi_mq_sysbuf_alloc(mq, msglen); mq_copy_tiny((uint32_t *)req->buf, (uint32_t *)payload, msglen); } else req->buf = NULL; req->state = MQ_STATE_COMPLETE; break; case MQ_MSG_SHORT: req->buf = psmi_mq_sysbuf_alloc(mq, msglen); psmi_mq_mtucpy(req->buf, payload, msglen); req->state = MQ_STATE_COMPLETE; break; case MQ_MSG_LONG: req->egrid = egrid; req->epaddr = epaddr; req->send_msgoff = 0; req->buf = psmi_mq_sysbuf_alloc(mq, msglen); req->state = MQ_STATE_UNEXP; req->type |= MQE_TYPE_EGRLONG; STAILQ_INSERT_TAIL(&epaddr->mctxt_master->egrlong, req, nextq); _IPATH_VDBG("unexp MSG_LONG %d of length %d bytes pay=%d\n", egrid.egr_msgno, msglen, paylen); if (paylen > 0) psmi_mq_handle_data(req, epaddr, egrid.egr_data, 0, payload, paylen); psmi_mq_handle_egrdata(mq, req, epaddr); break; default: psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, "Internal error, unknown packet 0x%x", mode); } req->msg_seqnum = msg_seqnum; mq_sq_append(&epaddr->mctxt_master->outoforder_q, req); epaddr->mctxt_master->outoforder_c++; mq->stats.rx_sys_bytes += msglen; mq->stats.rx_sys_num++; return MQ_RET_UNEXP_OK; }
int __recvpath psmi_mq_handle_envelope_unexpected( psm_mq_t mq, uint16_t mode, psm_epaddr_t epaddr, uint64_t tag, psmi_egrid_t egrid, uint32_t send_msglen, const void *payload, uint32_t paylen) { psm_mq_req_t req; uint32_t msglen; /* * Keep a callback here in case we want to fit some other high-level * protocols over MQ (i.e. shmem). These protocols would bypass the * normal mesage handling and go to higher-level message handlers. */ if (mode >= MQ_MSG_USER_FIRST && mq->unexpected_callback) { mq->unexpected_callback(mq,mode,epaddr,tag,send_msglen,payload,paylen); return MQ_RET_UNEXP_OK; } req = psmi_mq_req_alloc(mq, MQE_TYPE_RECV); psmi_assert(req != NULL); req->tag = tag; req->recv_msgoff = 0; req->recv_msglen = req->send_msglen = req->buf_len = msglen = send_msglen; _IPATH_VDBG( "from=%s match=NO (req=%p) mode=%x mqtag=%" PRIx64 " send_msglen=%d\n", psmi_epaddr_get_name(epaddr->epid), req, mode, tag, send_msglen); #if 0 if (mq->cur_sysbuf_bytes+msglen > mq->max_sysbuf_bytes) { _IPATH_VDBG("req=%p with len=%d exceeds limit of %llu sysbuf_bytes\n", req, msglen, (unsigned long long) mq->max_sysbuf_bytes); return MQ_RET_UNEXP_NO_RESOURCES; } #endif switch (mode) { case MQ_MSG_TINY: if (msglen > 0) { req->buf = psmi_mq_sysbuf_alloc(mq, msglen); mq_copy_tiny((uint32_t *)req->buf, (uint32_t *)payload, msglen); } else req->buf = NULL; req->state = MQ_STATE_COMPLETE; break; case MQ_MSG_SHORT: req->buf = psmi_mq_sysbuf_alloc(mq, msglen); psmi_mq_mtucpy(req->buf, payload, msglen); req->state = MQ_STATE_COMPLETE; break; case MQ_MSG_LONG: req->egrid = egrid; req->send_msgoff = 0; req->buf = psmi_mq_sysbuf_alloc(mq, msglen); req->state = MQ_STATE_UNEXP; req->type |= MQE_TYPE_EGRLONG; STAILQ_INSERT_TAIL(&epaddr->mctxt_master->egrlong, req, nextq); _IPATH_VDBG("unexp MSG_LONG %d of length %d bytes pay=%d\n", egrid.egr_msgno, msglen, paylen); if (paylen > 0) psmi_mq_handle_data(req, epaddr, egrid.egr_data, 0, payload, paylen); psmi_mq_handle_egrdata(mq, req, epaddr); break; default: psmi_handle_error(PSMI_EP_NORETURN, PSM_INTERNAL_ERR, "Internal error, unknown packet 0x%x", mode); } mq_sq_append(&mq->unexpected_q, req); mq->stats.rx_sys_bytes += msglen; mq->stats.rx_sys_num++; return MQ_RET_UNEXP_OK; }