static void send_msg(int fd, short args, void *cbdata) { orte_rml_send_request_t *req = (orte_rml_send_request_t*)cbdata; orte_process_name_t *peer = &(req->post.dst); orte_rml_tag_t tag = req->post.tag; orte_rml_recv_t *rcv; orte_rml_send_t *snd; int bytes; orte_self_send_xfer_t *xfer; int i; char* ptr; OPAL_OUTPUT_VERBOSE((1, orte_rml_base_framework.framework_output, "%s rml_send_msg to peer %s at tag %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(peer), tag)); OPAL_TIMING_EVENT((&tm_rml, "to %s", ORTE_NAME_PRINT(peer))); /* if this is a message to myself, then just post the message * for receipt - no need to dive into the oob */ if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, peer, ORTE_PROC_MY_NAME)) { /* local delivery */ OPAL_OUTPUT_VERBOSE((1, orte_rml_base_framework.framework_output, "%s rml_send_iovec_to_self at tag %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tag)); /* send to self is a tad tricky - we really don't want * to track the send callback function throughout the recv * process and execute it upon receipt as this would provide * very different timing from a non-self message. Specifically, * if we just retain a pointer to the incoming data * and then execute the send callback prior to the receive, * then the caller will think we are done with the data and * can release it. So we have to copy the data in order to * execute the send callback prior to receiving the message. * * In truth, this really is a better mimic of the non-self * message behavior. If we actually pushed the message out * on the wire and had it loop back, then we would receive * a new block of data anyway. */ /* setup the send callback */ xfer = OBJ_NEW(orte_self_send_xfer_t); if (NULL != req->post.iov) { xfer->iov = req->post.iov; xfer->count = req->post.count; xfer->cbfunc.iov = req->post.cbfunc.iov; } else { xfer->buffer = req->post.buffer; xfer->cbfunc.buffer = req->post.cbfunc.buffer; } xfer->tag = tag; xfer->cbdata = req->post.cbdata; /* setup the event for the send callback */ opal_event_set(orte_event_base, &xfer->ev, -1, OPAL_EV_WRITE, send_self_exe, xfer); opal_event_set_priority(&xfer->ev, ORTE_MSG_PRI); opal_event_active(&xfer->ev, OPAL_EV_WRITE, 1); /* copy the message for the recv */ rcv = OBJ_NEW(orte_rml_recv_t); rcv->sender = *peer; rcv->tag = tag; if (NULL != req->post.iov) { /* get the total number of bytes in the iovec array */ bytes = 0; for (i = 0 ; i < req->post.count ; ++i) { bytes += req->post.iov[i].iov_len; } /* get the required memory allocation */ if (0 < bytes) { rcv->iov.iov_base = (IOVBASE_TYPE*)malloc(bytes); rcv->iov.iov_len = bytes; /* transfer the bytes */ ptr = (char*)rcv->iov.iov_base; for (i = 0 ; i < req->post.count ; ++i) { memcpy(ptr, req->post.iov[i].iov_base, req->post.iov[i].iov_len); ptr += req->post.iov[i].iov_len; } } } else if (0 < req->post.buffer->bytes_used) { rcv->iov.iov_base = (IOVBASE_TYPE*)malloc(req->post.buffer->bytes_used); memcpy(rcv->iov.iov_base, req->post.buffer->base_ptr, req->post.buffer->bytes_used); rcv->iov.iov_len = req->post.buffer->bytes_used; } /* post the message for receipt - since the send callback was posted * first and has the same priority, it will execute first */ ORTE_RML_ACTIVATE_MESSAGE(rcv); OBJ_RELEASE(req); return; } snd = OBJ_NEW(orte_rml_send_t); snd->dst = *peer; snd->origin = *ORTE_PROC_MY_NAME; snd->tag = tag; if (NULL != req->post.iov) { snd->iov = req->post.iov; snd->count = req->post.count; snd->cbfunc.iov = req->post.cbfunc.iov; } else { snd->buffer = req->post.buffer; snd->cbfunc.buffer = req->post.cbfunc.buffer; } snd->cbdata = req->post.cbdata; /* activate the OOB send state */ ORTE_OOB_SEND(snd); OBJ_RELEASE(req); }
int orte_rml_oob_send_nb(struct orte_rml_base_module_t *mod, orte_process_name_t* peer, struct iovec* iov, int count, orte_rml_tag_t tag, orte_rml_callback_fn_t cbfunc, void* cbdata) { orte_rml_recv_t *rcv; orte_rml_send_t *snd; int bytes; orte_self_send_xfer_t *xfer; int i; char* ptr; OPAL_OUTPUT_VERBOSE((1, orte_rml_base_framework.framework_output, "%s rml_send to peer %s at tag %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(peer), tag)); if (ORTE_RML_TAG_INVALID == tag) { /* cannot send to an invalid tag */ ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM); return ORTE_ERR_BAD_PARAM; } if (NULL == peer || OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, ORTE_NAME_INVALID, peer)) { /* cannot send to an invalid peer */ ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM); return ORTE_ERR_BAD_PARAM; } /* if this is a message to myself, then just post the message * for receipt - no need to dive into the oob */ if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, peer, ORTE_PROC_MY_NAME)) { /* local delivery */ OPAL_OUTPUT_VERBOSE((1, orte_rml_base_framework.framework_output, "%s rml_send_iovec_to_self at tag %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tag)); /* send to self is a tad tricky - we really don't want * to track the send callback function throughout the recv * process and execute it upon receipt as this would provide * very different timing from a non-self message. Specifically, * if we just retain a pointer to the incoming data * and then execute the send callback prior to the receive, * then the caller will think we are done with the data and * can release it. So we have to copy the data in order to * execute the send callback prior to receiving the message. * * In truth, this really is a better mimic of the non-self * message behavior. If we actually pushed the message out * on the wire and had it loop back, then we would receive * a new block of data anyway. */ /* setup the send callback */ xfer = OBJ_NEW(orte_self_send_xfer_t); xfer->iov = iov; xfer->count = count; xfer->cbfunc.iov = cbfunc; xfer->tag = tag; xfer->cbdata = cbdata; /* setup the event for the send callback */ ORTE_THREADSHIFT(xfer, orte_event_base, send_self_exe, ORTE_MSG_PRI); /* copy the message for the recv */ rcv = OBJ_NEW(orte_rml_recv_t); rcv->sender = *peer; rcv->tag = tag; /* get the total number of bytes in the iovec array */ bytes = 0; for (i = 0 ; i < count ; ++i) { bytes += iov[i].iov_len; } /* get the required memory allocation */ if (0 < bytes) { rcv->iov.iov_base = (IOVBASE_TYPE*)malloc(bytes); rcv->iov.iov_len = bytes; /* transfer the bytes */ ptr = (char*)rcv->iov.iov_base; for (i = 0 ; i < count ; ++i) { memcpy(ptr, iov[i].iov_base, iov[i].iov_len); ptr += iov[i].iov_len; } } /* post the message for receipt - since the send callback was posted * first and has the same priority, it will execute first */ ORTE_RML_ACTIVATE_MESSAGE(rcv); return ORTE_SUCCESS; } snd = OBJ_NEW(orte_rml_send_t); snd->dst = *peer; snd->origin = *ORTE_PROC_MY_NAME; snd->tag = tag; snd->iov = iov; snd->count = count; snd->cbfunc.iov = cbfunc; snd->cbdata = cbdata; snd->routed = strdup(mod->routed); /* activate the OOB send state */ ORTE_OOB_SEND(snd); return ORTE_SUCCESS; }
int orte_rml_oob_send_buffer_nb(struct orte_rml_base_module_t *mod, orte_process_name_t* peer, opal_buffer_t* buffer, orte_rml_tag_t tag, orte_rml_buffer_callback_fn_t cbfunc, void* cbdata) { orte_rml_recv_t *rcv; orte_rml_send_t *snd; orte_self_send_xfer_t *xfer; OPAL_OUTPUT_VERBOSE((1, orte_rml_base_framework.framework_output, "%s rml_send_buffer to peer %s at tag %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(peer), tag)); if (ORTE_RML_TAG_INVALID == tag) { /* cannot send to an invalid tag */ ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM); return ORTE_ERR_BAD_PARAM; } if (NULL == peer || OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, ORTE_NAME_INVALID, peer)) { /* cannot send to an invalid peer */ ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM); return ORTE_ERR_BAD_PARAM; } OPAL_TIMING_EVENT((&tm_rml, "to %s", ORTE_NAME_PRINT(peer))); /* if this is a message to myself, then just post the message * for receipt - no need to dive into the oob */ if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, peer, ORTE_PROC_MY_NAME)) { /* local delivery */ OPAL_OUTPUT_VERBOSE((1, orte_rml_base_framework.framework_output, "%s rml_send_iovec_to_self at tag %d", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tag)); /* send to self is a tad tricky - we really don't want * to track the send callback function throughout the recv * process and execute it upon receipt as this would provide * very different timing from a non-self message. Specifically, * if we just retain a pointer to the incoming data * and then execute the send callback prior to the receive, * then the caller will think we are done with the data and * can release it. So we have to copy the data in order to * execute the send callback prior to receiving the message. * * In truth, this really is a better mimic of the non-self * message behavior. If we actually pushed the message out * on the wire and had it loop back, then we would receive * a new block of data anyway. */ /* setup the send callback */ xfer = OBJ_NEW(orte_self_send_xfer_t); xfer->buffer = buffer; xfer->cbfunc.buffer = cbfunc; xfer->tag = tag; xfer->cbdata = cbdata; /* setup the event for the send callback */ opal_event_set(orte_event_base, &xfer->ev, -1, OPAL_EV_WRITE, send_self_exe, xfer); opal_event_set_priority(&xfer->ev, ORTE_MSG_PRI); opal_event_active(&xfer->ev, OPAL_EV_WRITE, 1); /* copy the message for the recv */ rcv = OBJ_NEW(orte_rml_recv_t); rcv->sender = *peer; rcv->tag = tag; rcv->iov.iov_base = (IOVBASE_TYPE*)malloc(buffer->bytes_used); memcpy(rcv->iov.iov_base, buffer->base_ptr, buffer->bytes_used); rcv->iov.iov_len = buffer->bytes_used; /* post the message for receipt - since the send callback was posted * first and has the same priority, it will execute first */ ORTE_RML_ACTIVATE_MESSAGE(rcv); return ORTE_SUCCESS; } snd = OBJ_NEW(orte_rml_send_t); snd->dst = *peer; snd->origin = *ORTE_PROC_MY_NAME; snd->tag = tag; snd->buffer = buffer; snd->cbfunc.buffer = cbfunc; snd->cbdata = cbdata; snd->routed = strdup(mod->routed); /* activate the OOB send state */ ORTE_OOB_SEND(snd); return ORTE_SUCCESS; }