void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
{
pmix_status_t rc;
pmix_peer_t *peer = (pmix_peer_t*)cbdata;
pmix_ptl_recv_t *msg = NULL;
/* acquire the object */
PMIX_ACQUIRE_OBJECT(peer);
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:recv:handler called with peer %s:%d",
(NULL == peer) ? "NULL" : peer->info->pname.nspace,
(NULL == peer) ? PMIX_RANK_UNDEF : peer->info->pname.rank);
if (NULL == peer) {
return;
}
/* allocate a new message and setup for recv */
if (NULL == peer->recv_msg) {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:recv:handler allocate new recv msg");
peer->recv_msg = PMIX_NEW(pmix_ptl_recv_t);
if (NULL == peer->recv_msg) {
pmix_output(0, "usock_recv_handler: unable to allocate recv message\n");
goto err_close;
}
PMIX_RETAIN(peer);
peer->recv_msg->peer = peer; // provide a handle back to the peer object
/* start by reading the header */
peer->recv_msg->rdptr = (char*)&peer->recv_msg->hdr;
peer->recv_msg->rdbytes = sizeof(pmix_usock_hdr_t);
}
msg = peer->recv_msg;
msg->sd = sd;
/* if the header hasn't been completely read, read it */
if (!msg->hdr_recvd) {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:recv:handler read hdr on socket %d", peer->sd);
if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &msg->rdptr, &msg->rdbytes))) {
/* completed reading the header */
peer->recv_msg->hdr_recvd = true;
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"RECVD MSG FOR TAG %d SIZE %d",
(int)peer->recv_msg->hdr.tag,
(int)peer->recv_msg->hdr.nbytes);
/* if this is a zero-byte message, then we are done */
if (0 == peer->recv_msg->hdr.nbytes) {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"RECVD ZERO-BYTE MESSAGE FROM %s:%d for tag %d",
peer->info->pname.nspace, peer->info->pname.rank,
peer->recv_msg->hdr.tag);
peer->recv_msg->data = NULL; // make sure
peer->recv_msg->rdptr = NULL;
peer->recv_msg->rdbytes = 0;
/* post it for delivery */
PMIX_ACTIVATE_POST_MSG(peer->recv_msg);
peer->recv_msg = NULL;
PMIX_POST_OBJECT(peer);
return;
} else {
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"usock:recv:handler allocate data region of size %lu",
(unsigned long)peer->recv_msg->hdr.nbytes);
/* allocate the data region */
peer->recv_msg->data = (char*)malloc(peer->recv_msg->hdr.nbytes);
memset(peer->recv_msg->data, 0, peer->recv_msg->hdr.nbytes);
/* point to it */
peer->recv_msg->rdptr = peer->recv_msg->data;
peer->recv_msg->rdbytes = peer->recv_msg->hdr.nbytes;
}
/* fall thru and attempt to read the data */
} else if (PMIX_ERR_RESOURCE_BUSY == rc ||
PMIX_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
return;
} else {
/* the remote peer closed the connection - report that condition
* and let the caller know
*/
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"pmix_usock_msg_recv: peer closed connection");
goto err_close;
}
}
if (peer->recv_msg->hdr_recvd) {
/* continue to read the data block - we start from
* wherever we left off, which could be at the
* beginning or somewhere in the message
*/
if (PMIX_SUCCESS == (rc = read_bytes(peer->sd, &msg->rdptr, &msg->rdbytes))) {
/* we recvd all of the message */
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"RECVD COMPLETE MESSAGE FROM SERVER OF %d BYTES FOR TAG %d ON PEER SOCKET %d",
(int)peer->recv_msg->hdr.nbytes,
peer->recv_msg->hdr.tag, peer->sd);
/* post it for delivery */
PMIX_ACTIVATE_POST_MSG(peer->recv_msg);
peer->recv_msg = NULL;
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
return;
} else if (PMIX_ERR_RESOURCE_BUSY == rc ||
PMIX_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
return;
} else {
/* the remote peer closed the connection - report that condition
* and let the caller know
*/
pmix_output_verbose(2, pmix_ptl_base_framework.framework_output,
"pmix_usock_msg_recv: peer closed connection");
goto err_close;
}
}
/* success */
return;
err_close:
/* stop all events */
if (peer->recv_ev_active) {
pmix_event_del(&peer->recv_event);
peer->recv_ev_active = false;
}
if (peer->send_ev_active) {
pmix_event_del(&peer->send_event);
peer->send_ev_active = false;
}
if (NULL != peer->recv_msg) {
PMIX_RELEASE(peer->recv_msg);
peer->recv_msg = NULL;
}
pmix_ptl_base_lost_connection(peer, PMIX_ERR_UNREACH);
/* ensure we post the modified peer object before another thread
* picks it back up */
PMIX_POST_OBJECT(peer);
}
void pmix_usock_recv_handler(int sd, short flags, void *cbdata)
{
int rc;
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s usock:recv:handler called",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
switch (mca_pmix_native_component.state) {
case PMIX_USOCK_CONNECT_ACK:
if (OPAL_SUCCESS == (rc = usock_recv_connect_ack())) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s usock:recv:handler starting send/recv events",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* we connected! Start the send/recv events */
if (!mca_pmix_native_component.recv_ev_active) {
opal_event_add(&mca_pmix_native_component.recv_event, 0);
mca_pmix_native_component.recv_ev_active = true;
}
if (mca_pmix_native_component.timer_ev_active) {
opal_event_del(&mca_pmix_native_component.timer_event);
mca_pmix_native_component.timer_ev_active = false;
}
/* if there is a message waiting to be sent, queue it */
if (NULL == mca_pmix_native_component.send_msg) {
mca_pmix_native_component.send_msg = (pmix_usock_send_t*)opal_list_remove_first(&mca_pmix_native_component.send_queue);
}
if (NULL != mca_pmix_native_component.send_msg && !mca_pmix_native_component.send_ev_active) {
opal_event_add(&mca_pmix_native_component.send_event, 0);
mca_pmix_native_component.send_ev_active = true;
}
/* update our state */
mca_pmix_native_component.state = PMIX_USOCK_CONNECTED;
} else {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s UNABLE TO COMPLETE CONNECT ACK WITH SERVER",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
opal_event_del(&mca_pmix_native_component.recv_event);
mca_pmix_native_component.recv_ev_active = false;
return;
}
break;
case PMIX_USOCK_CONNECTED:
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s usock:recv:handler CONNECTED",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* allocate a new message and setup for recv */
if (NULL == mca_pmix_native_component.recv_msg) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s usock:recv:handler allocate new recv msg",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
mca_pmix_native_component.recv_msg = OBJ_NEW(pmix_usock_recv_t);
if (NULL == mca_pmix_native_component.recv_msg) {
opal_output(0, "%s usock_recv_handler: unable to allocate recv message\n",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
return;
}
/* start by reading the header */
mca_pmix_native_component.recv_msg->rdptr = (char*)&mca_pmix_native_component.recv_msg->hdr;
mca_pmix_native_component.recv_msg->rdbytes = sizeof(pmix_usock_hdr_t);
}
/* if the header hasn't been completely read, read it */
if (!mca_pmix_native_component.recv_msg->hdr_recvd) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"usock:recv:handler read hdr");
if (OPAL_SUCCESS == (rc = read_bytes(mca_pmix_native_component.recv_msg))) {
/* completed reading the header */
mca_pmix_native_component.recv_msg->hdr_recvd = true;
/* if this is a zero-byte message, then we are done */
if (0 == mca_pmix_native_component.recv_msg->hdr.nbytes) {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s RECVD ZERO-BYTE MESSAGE FROM SERVER for tag %d",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
mca_pmix_native_component.recv_msg->hdr.tag);
mca_pmix_native_component.recv_msg->data = NULL; // make sure
mca_pmix_native_component.recv_msg->rdptr = NULL;
mca_pmix_native_component.recv_msg->rdbytes = 0;
} else {
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s usock:recv:handler allocate data region of size %lu",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
(unsigned long)mca_pmix_native_component.recv_msg->hdr.nbytes);
/* allocate the data region */
mca_pmix_native_component.recv_msg->data = (char*)malloc(mca_pmix_native_component.recv_msg->hdr.nbytes);
/* point to it */
mca_pmix_native_component.recv_msg->rdptr = mca_pmix_native_component.recv_msg->data;
mca_pmix_native_component.recv_msg->rdbytes = mca_pmix_native_component.recv_msg->hdr.nbytes;
}
/* fall thru and attempt to read the data */
} else if (OPAL_ERR_RESOURCE_BUSY == rc ||
OPAL_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
return;
} else {
/* close the connection */
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s usock:recv:handler error reading bytes - closing connection",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
CLOSE_THE_SOCKET(mca_pmix_native_component.sd);
return;
}
}
if (mca_pmix_native_component.recv_msg->hdr_recvd) {
/* continue to read the data block - we start from
* wherever we left off, which could be at the
* beginning or somewhere in the message
*/
if (OPAL_SUCCESS == (rc = read_bytes(mca_pmix_native_component.recv_msg))) {
/* we recvd all of the message */
opal_output_verbose(2, opal_pmix_base_framework.framework_output,
"%s RECVD COMPLETE MESSAGE FROM SERVER OF %d BYTES FOR TAG %d",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
(int)mca_pmix_native_component.recv_msg->hdr.nbytes,
mca_pmix_native_component.recv_msg->hdr.tag);
/* post it for delivery */
PMIX_ACTIVATE_POST_MSG(mca_pmix_native_component.recv_msg);
mca_pmix_native_component.recv_msg = NULL;
} else if (OPAL_ERR_RESOURCE_BUSY == rc ||
OPAL_ERR_WOULD_BLOCK == rc) {
/* exit this event and let the event lib progress */
return;
} else {
// report the error
opal_output(0, "%s usock_peer_recv_handler: unable to recv message",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME));
/* turn off the recv event */
opal_event_del(&mca_pmix_native_component.recv_event);
mca_pmix_native_component.recv_ev_active = false;
CLOSE_THE_SOCKET(mca_pmix_native_component.sd);
return;
}
}
break;
default:
opal_output(0, "%s usock_peer_recv_handler: invalid socket state(%d)",
OPAL_NAME_PRINT(OPAL_PROC_MY_NAME),
mca_pmix_native_component.state);
CLOSE_THE_SOCKET(mca_pmix_native_component.sd);
break;
}
}
