int mca_oob_tcp_peer_send_ident(mca_oob_tcp_peer_t* peer)
{
mca_oob_tcp_hdr_t hdr;
if(peer->peer_state != MCA_OOB_TCP_CONNECTED)
return ORTE_SUCCESS;
hdr.msg_src = *orte_process_info.my_name;
hdr.msg_dst = peer->peer_name;
hdr.msg_type = MCA_OOB_TCP_IDENT;
hdr.msg_size = 0;
hdr.msg_tag = 0;
MCA_OOB_TCP_HDR_HTON(&hdr);
if(mca_oob_tcp_peer_send_blocking(peer, &hdr, sizeof(hdr)) != sizeof(hdr))
return ORTE_ERR_UNREACH;
return ORTE_SUCCESS;
}
/*
* Send the globally unique identifier for this process to a peer on
* a newly connected socket.
*/
static int mca_oob_tcp_peer_send_connect_ack(mca_oob_tcp_peer_t* peer)
{
/* send process identifier of self and peer - note that we may
* have assigned the peer a unique process name - if it came up
* without one.
*/
mca_oob_tcp_hdr_t hdr;
memset(&hdr,0,sizeof(hdr));
if (NULL == orte_process_info.my_name) { /* my name isn't defined yet */
hdr.msg_src = *ORTE_NAME_INVALID;
} else {
hdr.msg_src = *(orte_process_info.my_name);
}
hdr.msg_dst = peer->peer_name;
hdr.msg_type = MCA_OOB_TCP_CONNECT;
MCA_OOB_TCP_HDR_HTON(&hdr);
if(mca_oob_tcp_peer_send_blocking(peer, &hdr, sizeof(hdr)) != sizeof(hdr)) {
return ORTE_ERR_UNREACH;
}
return ORTE_SUCCESS;
}
int mca_oob_tcp_send(
orte_process_name_t* name,
struct iovec *iov,
int count,
int tag,
int flags)
{
mca_oob_tcp_peer_t* peer = mca_oob_tcp_peer_lookup(name);
mca_oob_tcp_msg_t* msg;
int size;
int rc;
if(NULL == peer)
return ORTE_ERR_UNREACH;
if(mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_ALL) {
opal_output(0, "[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_send: tag %d\n",
ORTE_NAME_ARGS(orte_process_info.my_name),
ORTE_NAME_ARGS(&(peer->peer_name)),
tag);
}
MCA_OOB_TCP_MSG_ALLOC(msg, rc);
if(NULL == msg)
return rc;
/* calculate the size of the message */
size = 0;
for(rc = 0; rc < count; rc++) {
size += iov[rc].iov_len;
}
/* turn the size to network byte order so there will be no problems */
msg->msg_hdr.msg_type = MCA_OOB_TCP_DATA;
msg->msg_hdr.msg_size = size;
msg->msg_hdr.msg_tag = tag;
if (NULL == orte_process_info.my_name) {
msg->msg_hdr.msg_src = *ORTE_NAME_INVALID;
} else {
msg->msg_hdr.msg_src = *orte_process_info.my_name;
}
msg->msg_hdr.msg_dst = *name;
/* create one additional iovect that will hold the header */
msg->msg_type = MCA_OOB_TCP_POSTED;
msg->msg_rc = 0;
msg->msg_flags = flags;
msg->msg_uiov = iov;
msg->msg_ucnt = count;
msg->msg_rwiov = mca_oob_tcp_msg_iov_alloc(msg, count+1);
msg->msg_rwiov[0].iov_base = (ompi_iov_base_ptr_t)(&msg->msg_hdr);
msg->msg_rwiov[0].iov_len = sizeof(msg->msg_hdr);
msg->msg_rwptr = msg->msg_rwiov;
msg->msg_rwcnt = msg->msg_rwnum = count + 1;
memcpy(msg->msg_rwiov+1, msg->msg_uiov, sizeof(struct iovec)*msg->msg_ucnt);
msg->msg_rwbuf = NULL;
msg->msg_cbfunc = NULL;
msg->msg_cbdata = NULL;
msg->msg_complete = false;
msg->msg_peer = peer->peer_name;
if (NULL != name && NULL != orte_process_info.my_name &&
ORTE_EQUAL == mca_oob_tcp_process_name_compare(name, orte_process_info.my_name)) { /* local delivery */
return mca_oob_tcp_send_self(peer,msg,iov,count);
}
MCA_OOB_TCP_HDR_HTON(&msg->msg_hdr);
rc = mca_oob_tcp_peer_send(peer, msg);
if(rc != ORTE_SUCCESS) {
MCA_OOB_TCP_MSG_RETURN(msg);
return rc;
}
rc = mca_oob_tcp_msg_wait(msg, &size);
MCA_OOB_TCP_MSG_RETURN(msg);
if(rc != ORTE_SUCCESS)
return rc;
size -= sizeof(mca_oob_tcp_hdr_t);
return size;
}
/*
* Non-blocking version of mca_oob_send().
*
* @param peer (IN) Opaque name of peer process.
* @param msg (IN) Array of iovecs describing user buffers and lengths.
* @param count (IN) Number of elements in iovec array.
* @param flags (IN) Currently unused.
* @param cbfunc (IN) Callback function on send completion.
* @param cbdata (IN) User data that is passed to callback function.
* @return OMPI error code (<0) on error number of bytes actually sent.
*
*/
int mca_oob_tcp_send_nb(
orte_process_name_t* target,
orte_process_name_t* origin,
struct iovec* iov,
int count,
int tag,
int flags,
orte_rml_callback_fn_t cbfunc,
void* cbdata)
{
mca_oob_tcp_peer_t* peer = mca_oob_tcp_peer_lookup(target);
mca_oob_tcp_msg_t* msg;
int size;
int rc;
if(NULL == peer)
return ORTE_ERR_UNREACH;
MCA_OOB_TCP_MSG_ALLOC(msg, rc);
if(NULL == msg) {
return rc;
}
/* calculate the size of the message */
size = 0;
for(rc = 0; rc < count; rc++) {
size += iov[rc].iov_len;
}
if(mca_oob_tcp_component.tcp_debug >= OOB_TCP_DEBUG_ALL) {
opal_output(0, "%s-%s mca_oob_tcp_send_nb: tag %d size %lu\n",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_NAME_PRINT(&(peer->peer_name)),
tag, (unsigned long)size );
}
/* turn the size to network byte order so there will be no problems */
msg->msg_hdr.msg_type = MCA_OOB_TCP_DATA;
msg->msg_hdr.msg_size = size;
msg->msg_hdr.msg_tag = tag;
msg->msg_hdr.msg_origin = *origin;
msg->msg_hdr.msg_src = *ORTE_PROC_MY_NAME;
msg->msg_hdr.msg_dst = *target;
/* create one additional iovect that will hold the size of the message */
msg->msg_type = MCA_OOB_TCP_POSTED;
msg->msg_rc = 0;
msg->msg_flags = flags;
msg->msg_uiov = iov;
msg->msg_ucnt = count;
msg->msg_rwiov = mca_oob_tcp_msg_iov_alloc(msg,count+1);
msg->msg_rwiov[0].iov_base = (ompi_iov_base_ptr_t)(&msg->msg_hdr);
msg->msg_rwiov[0].iov_len = sizeof(msg->msg_hdr);
msg->msg_rwptr = msg->msg_rwiov;
msg->msg_rwcnt = msg->msg_rwnum = count + 1;
memcpy(msg->msg_rwiov+1, msg->msg_uiov, sizeof(struct iovec)*msg->msg_ucnt);
msg->msg_rwbuf = NULL;
msg->msg_cbfunc = cbfunc;
msg->msg_cbdata = cbdata;
msg->msg_complete = false;
msg->msg_peer = peer->peer_name;
if (OPAL_EQUAL == mca_oob_tcp_process_name_compare(target, ORTE_PROC_MY_NAME)) { /* local delivery */
rc = mca_oob_tcp_send_self(peer,msg,iov,count);
if (rc < 0 ) {
return rc;
} else if (size == rc) {
return ORTE_SUCCESS;
} else {
return ORTE_ERROR;
}
}
MCA_OOB_TCP_HDR_HTON(&msg->msg_hdr);
rc = mca_oob_tcp_peer_send(peer, msg);
if(rc != ORTE_SUCCESS) {
if (rc != ORTE_ERR_ADDRESSEE_UNKNOWN) {
MCA_OOB_TCP_MSG_RETURN(msg);
}
return rc;
}
return ORTE_SUCCESS;
}
int mca_oob_tcp_ping(
const orte_process_name_t* name,
const char* uri,
const struct timeval *timeout)
{
int sd, flags, rc;
struct sockaddr_in inaddr;
fd_set fdset;
mca_oob_tcp_hdr_t hdr;
struct timeval tv;
struct iovec iov;
#ifndef __WINDOWS__
struct opal_event sigpipe_handler;
#endif
/* parse uri string */
if(ORTE_SUCCESS != (rc = mca_oob_tcp_parse_uri(uri, &inaddr))) {
opal_output(0,
"[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_ping: invalid uri: %s\n",
ORTE_NAME_ARGS(orte_process_info.my_name),
ORTE_NAME_ARGS(name),
uri);
return rc;
}
/* create socket */
sd = socket(AF_INET, SOCK_STREAM, 0);
if (sd < 0) {
opal_output(0,
"[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_ping: socket() failed: %s (%d)\n",
ORTE_NAME_ARGS(orte_process_info.my_name),
ORTE_NAME_ARGS(name),
strerror(opal_socket_errno),
opal_socket_errno);
return ORTE_ERR_UNREACH;
}
/* setup the socket as non-blocking */
if((flags = fcntl(sd, F_GETFL, 0)) < 0) {
opal_output(0, "[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_ping: fcntl(F_GETFL) failed: %s (%d)\n",
ORTE_NAME_ARGS(orte_process_info.my_name),
ORTE_NAME_ARGS(name),
strerror(opal_socket_errno),
opal_socket_errno);
} else {
flags |= O_NONBLOCK;
if(fcntl(sd, F_SETFL, flags) < 0) {
opal_output(0, "[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_ping: fcntl(F_SETFL) failed: %s (%d)\n",
ORTE_NAME_ARGS(orte_process_info.my_name),
ORTE_NAME_ARGS(name),
strerror(opal_socket_errno),
opal_socket_errno);
}
}
/* start the connect - will likely fail with EINPROGRESS */
FD_ZERO(&fdset);
if(connect(sd, (struct sockaddr*)&inaddr, sizeof(inaddr)) < 0) {
/* connect failed? */
if(opal_socket_errno != EINPROGRESS && opal_socket_errno != EWOULDBLOCK) {
CLOSE_THE_SOCKET(sd);
return ORTE_ERR_UNREACH;
}
/* select with timeout to wait for connect to complete */
FD_SET(sd, &fdset);
tv = *timeout;
rc = select(sd+1, NULL, &fdset, NULL, &tv);
if(rc <= 0) {
CLOSE_THE_SOCKET(sd);
return ORTE_ERR_UNREACH;
}
}
/* set socket back to blocking */
flags &= ~O_NONBLOCK;
if(fcntl(sd, F_SETFL, flags) < 0) {
opal_output(0, "[%lu,%lu,%lu]-[%lu,%lu,%lu] mca_oob_tcp_ping: fcntl(F_SETFL) failed: %s (%d)\n",
ORTE_NAME_ARGS(orte_process_info.my_name),
ORTE_NAME_ARGS(name),
strerror(opal_socket_errno),
opal_socket_errno);
}
/* send a probe message */
memset(&hdr, 0, sizeof(hdr));
if(orte_process_info.my_name != NULL) {
hdr.msg_src = *orte_process_info.my_name;
} else {
hdr.msg_src = *ORTE_NAME_INVALID;
}
hdr.msg_dst = *name;
hdr.msg_type = MCA_OOB_TCP_PROBE;
MCA_OOB_TCP_HDR_HTON(&hdr);
#ifndef __WINDOWS__
/* Ignore SIGPIPE in the write -- determine success or failure in
the ping by looking at the return code from write() */
opal_signal_set(&sigpipe_handler, SIGPIPE,
noop, &sigpipe_handler);
opal_signal_add(&sigpipe_handler, NULL);
#endif
/* Do the write and see what happens. Use the writev version just to
* make Windows happy as there the write function is limitted to
* file operations.
*/
iov.iov_base = (IOVBASE_TYPE*)&hdr;
iov.iov_len = sizeof(hdr);
rc = writev(sd, &iov, 1 );
#ifndef __WINDOWS__
/* Now de-register the handler */
opal_signal_del(&sigpipe_handler);
#endif
if (rc != sizeof(hdr)) {
CLOSE_THE_SOCKET(sd);
return ORTE_ERR_UNREACH;
}
/* select with timeout to wait for response */
FD_SET(sd, &fdset);
tv = *timeout;
rc = select(sd+1, &fdset, NULL, NULL, &tv);
if(rc <= 0) {
CLOSE_THE_SOCKET(sd);
return ORTE_ERR_UNREACH;
}
if((rc = read(sd, &hdr, sizeof(hdr))) != sizeof(hdr)) {
CLOSE_THE_SOCKET(sd);
return ORTE_ERR_UNREACH;
}
MCA_OOB_TCP_HDR_NTOH(&hdr);
if(hdr.msg_type != MCA_OOB_TCP_PROBE) {
CLOSE_THE_SOCKET(sd);
return ORTE_ERR_UNREACH;
}
CLOSE_THE_SOCKET(sd);
return ORTE_SUCCESS;
}
