static void mca_oob_tcp_peer_recv_handler(int sd, short flags, void* user) { mca_oob_tcp_peer_t* peer = (mca_oob_tcp_peer_t *)user; OPAL_THREAD_LOCK(&peer->peer_lock); switch(peer->peer_state) { case MCA_OOB_TCP_CONNECT_ACK: { mca_oob_tcp_peer_recv_connect_ack(peer); break; } case MCA_OOB_TCP_CONNECTED: { /* allocate a new message and setup for recv */ if(NULL == peer->peer_recv_msg) { int rc; mca_oob_tcp_msg_t* msg; MCA_OOB_TCP_MSG_ALLOC(msg, rc); if(NULL == msg) { opal_output(0, "[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_peer_recv_handler: unable to allocate recv message\n", ORTE_NAME_ARGS(orte_process_info.my_name), ORTE_NAME_ARGS(&(peer->peer_name))); return; } msg->msg_type = MCA_OOB_TCP_UNEXPECTED; msg->msg_rc = 0; msg->msg_flags = 0; msg->msg_peer = peer->peer_name; msg->msg_rwiov = mca_oob_tcp_msg_iov_alloc(msg,2); msg->msg_rwbuf = NULL; msg->msg_rwcnt = msg->msg_rwnum = 1; msg->msg_rwptr = msg->msg_rwiov; msg->msg_rwiov[0].iov_base = (ompi_iov_base_ptr_t)&msg->msg_hdr; msg->msg_rwiov[0].iov_len = sizeof(msg->msg_hdr); peer->peer_recv_msg = msg; } if (peer->peer_recv_msg && mca_oob_tcp_msg_recv_handler(peer->peer_recv_msg, peer)) { mca_oob_tcp_msg_t* msg = peer->peer_recv_msg; peer->peer_recv_msg = NULL; OPAL_THREAD_UNLOCK(&peer->peer_lock); mca_oob_tcp_msg_recv_complete(msg, peer); return; } break; } default: { opal_output(0, "[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_peer_recv_handler: invalid socket state(%d)", ORTE_NAME_ARGS(orte_process_info.my_name), ORTE_NAME_ARGS(&(peer->peer_name)), peer->peer_state); mca_oob_tcp_peer_close(peer); break; } } OPAL_THREAD_UNLOCK(&peer->peer_lock); }
int mca_oob_tcp_send( orte_process_name_t* name, struct iovec *iov, int count, int tag, int flags) { mca_oob_tcp_peer_t* peer = mca_oob_tcp_peer_lookup(name); mca_oob_tcp_msg_t* msg; int size; int rc; if(NULL == peer) return ORTE_ERR_UNREACH; if(mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_ALL) { opal_output(0, "[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_send: tag %d\n", ORTE_NAME_ARGS(orte_process_info.my_name), ORTE_NAME_ARGS(&(peer->peer_name)), tag); } MCA_OOB_TCP_MSG_ALLOC(msg, rc); if(NULL == msg) return rc; /* calculate the size of the message */ size = 0; for(rc = 0; rc < count; rc++) { size += iov[rc].iov_len; } /* turn the size to network byte order so there will be no problems */ msg->msg_hdr.msg_type = MCA_OOB_TCP_DATA; msg->msg_hdr.msg_size = size; msg->msg_hdr.msg_tag = tag; if (NULL == orte_process_info.my_name) { msg->msg_hdr.msg_src = *ORTE_NAME_INVALID; } else { msg->msg_hdr.msg_src = *orte_process_info.my_name; } msg->msg_hdr.msg_dst = *name; /* create one additional iovect that will hold the header */ msg->msg_type = MCA_OOB_TCP_POSTED; msg->msg_rc = 0; msg->msg_flags = flags; msg->msg_uiov = iov; msg->msg_ucnt = count; msg->msg_rwiov = mca_oob_tcp_msg_iov_alloc(msg, count+1); msg->msg_rwiov[0].iov_base = (ompi_iov_base_ptr_t)(&msg->msg_hdr); msg->msg_rwiov[0].iov_len = sizeof(msg->msg_hdr); msg->msg_rwptr = msg->msg_rwiov; msg->msg_rwcnt = msg->msg_rwnum = count + 1; memcpy(msg->msg_rwiov+1, msg->msg_uiov, sizeof(struct iovec)*msg->msg_ucnt); msg->msg_rwbuf = NULL; msg->msg_cbfunc = NULL; msg->msg_cbdata = NULL; msg->msg_complete = false; msg->msg_peer = peer->peer_name; if (NULL != name && NULL != orte_process_info.my_name && ORTE_EQUAL == mca_oob_tcp_process_name_compare(name, orte_process_info.my_name)) { /* local delivery */ return mca_oob_tcp_send_self(peer,msg,iov,count); } MCA_OOB_TCP_HDR_HTON(&msg->msg_hdr); rc = mca_oob_tcp_peer_send(peer, msg); if(rc != ORTE_SUCCESS) { MCA_OOB_TCP_MSG_RETURN(msg); return rc; } rc = mca_oob_tcp_msg_wait(msg, &size); MCA_OOB_TCP_MSG_RETURN(msg); if(rc != ORTE_SUCCESS) return rc; size -= sizeof(mca_oob_tcp_hdr_t); return size; }
/* * Non-blocking version of mca_oob_send(). * * @param peer (IN) Opaque name of peer process. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. * @param flags (IN) Currently unused. * @param cbfunc (IN) Callback function on send completion. * @param cbdata (IN) User data that is passed to callback function. * @return OMPI error code (<0) on error number of bytes actually sent. * */ int mca_oob_tcp_send_nb( orte_process_name_t* target, orte_process_name_t* origin, struct iovec* iov, int count, int tag, int flags, orte_rml_callback_fn_t cbfunc, void* cbdata) { mca_oob_tcp_peer_t* peer = mca_oob_tcp_peer_lookup(target); mca_oob_tcp_msg_t* msg; int size; int rc; if(NULL == peer) return ORTE_ERR_UNREACH; MCA_OOB_TCP_MSG_ALLOC(msg, rc); if(NULL == msg) { return rc; } /* calculate the size of the message */ size = 0; for(rc = 0; rc < count; rc++) { size += iov[rc].iov_len; } if(mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_ALL) { opal_output(0, "%s-%s mca_oob_tcp_send_nb: tag %d size %lu\n", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), ORTE_NAME_PRINT(&(peer->peer_name)), tag, (unsigned long)size ); } /* turn the size to network byte order so there will be no problems */ msg->msg_hdr.msg_type = MCA_OOB_TCP_DATA; msg->msg_hdr.msg_size = size; msg->msg_hdr.msg_tag = tag; msg->msg_hdr.msg_origin = *origin; msg->msg_hdr.msg_src = *ORTE_PROC_MY_NAME; msg->msg_hdr.msg_dst = *target; /* create one additional iovect that will hold the size of the message */ msg->msg_type = MCA_OOB_TCP_POSTED; msg->msg_rc = 0; msg->msg_flags = flags; msg->msg_uiov = iov; msg->msg_ucnt = count; msg->msg_rwiov = mca_oob_tcp_msg_iov_alloc(msg,count+1); msg->msg_rwiov[0].iov_base = (ompi_iov_base_ptr_t)(&msg->msg_hdr); msg->msg_rwiov[0].iov_len = sizeof(msg->msg_hdr); msg->msg_rwptr = msg->msg_rwiov; msg->msg_rwcnt = msg->msg_rwnum = count + 1; memcpy(msg->msg_rwiov+1, msg->msg_uiov, sizeof(struct iovec)*msg->msg_ucnt); msg->msg_rwbuf = NULL; msg->msg_cbfunc = cbfunc; msg->msg_cbdata = cbdata; msg->msg_complete = false; msg->msg_peer = peer->peer_name; if (OPAL_EQUAL == mca_oob_tcp_process_name_compare(target, ORTE_PROC_MY_NAME)) { /* local delivery */ rc = mca_oob_tcp_send_self(peer,msg,iov,count); if (rc < 0 ) { return rc; } else if (size == rc) { return ORTE_SUCCESS; } else { return ORTE_ERROR; } } MCA_OOB_TCP_HDR_HTON(&msg->msg_hdr); rc = mca_oob_tcp_peer_send(peer, msg); if(rc != ORTE_SUCCESS) { if (rc != ORTE_ERR_ADDRESSEE_UNKNOWN) { MCA_OOB_TCP_MSG_RETURN(msg); } return rc; } return ORTE_SUCCESS; }
/* * Non-blocking version of mca_oob_recv(). * * @param peer (IN) Opaque name of peer process or ORTE_NAME_WILDCARD for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. * @param tag (IN) User supplied tag for matching send/recv. * @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, * @param cbfunc (IN) Callback function on recv completion. * @param cbdata (IN) User data that is passed to callback function. * @return OMPI error code (<0) on error. */ int mca_oob_tcp_recv_nb( orte_process_name_t* peer, struct iovec* iov, int count, int tag, int flags, orte_rml_callback_fn_t cbfunc, void* cbdata) { mca_oob_tcp_msg_t *msg; mca_oob_tcp_msg_t *match; int i, rc, size = 0; /* validate params */ if(NULL == iov || 0 == count) { return ORTE_ERR_BAD_PARAM; } /* allocate/initialize the posted receive */ MCA_OOB_TCP_MSG_ALLOC(msg, rc); if(NULL == msg) { OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_match_lock); return rc; } /* determine overall size of user supplied buffer */ for(i = 0; i < count; i++) { size += iov[i].iov_len; } /* fill in the header */ msg->msg_hdr.msg_origin = *peer; msg->msg_hdr.msg_src = *ORTE_PROC_MY_NAME; msg->msg_hdr.msg_dst = *peer; msg->msg_hdr.msg_size = size; msg->msg_hdr.msg_tag = tag; msg->msg_type = MCA_OOB_TCP_POSTED; msg->msg_rc = 0; msg->msg_flags = flags; msg->msg_uiov = iov; msg->msg_ucnt = count; msg->msg_cbfunc = cbfunc; msg->msg_cbdata = cbdata; msg->msg_complete = false; msg->msg_peer = *peer; msg->msg_rwbuf = NULL; msg->msg_rwiov = NULL; /* acquire the match lock */ OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_match_lock); if(flags & ORTE_RML_PERSISTENT) { opal_list_append(&mca_oob_tcp_component.tcp_msg_post, (opal_list_item_t *) msg); while(NULL != (match = mca_oob_tcp_msg_match_recv(peer,tag))) { mca_oob_tcp_msg_matched(msg, match); } } else { /* check to see if a matching receive is on the list */ match = mca_oob_tcp_msg_match_recv(peer, tag); if(NULL != match) { mca_oob_tcp_msg_matched(msg, match); MCA_OOB_TCP_MSG_RETURN(msg); } else { opal_list_append(&mca_oob_tcp_component.tcp_msg_post, (opal_list_item_t *) msg); } } OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_match_lock); return 0; }