int MPIDI_CH3I_Progress_finalize(void) { int mpi_errno; MPIDI_CH3I_Connection_t *conn = NULL; MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_FINALIZE); MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_FINALIZE); /* Shut down the listener */ mpi_errno = MPIDU_CH3I_ShutdownListener(); if (mpi_errno != MPI_SUCCESS) { MPIR_ERR_POP(mpi_errno); } /* Close open connections */ MPIDI_CH3I_Sock_close_open_sockets(MPIDI_CH3I_sock_set,(void**) &conn); while (conn != NULL) { conn->state = CONN_STATE_CLOSING; mpi_errno = MPIDI_CH3_Sockconn_handle_close_event(conn); if (mpi_errno) { MPIR_ERR_POP(mpi_errno); } MPIDI_CH3I_Sock_close_open_sockets(MPIDI_CH3I_sock_set,(void**) &conn); } /* * MT: in a multi-threaded environment, finalize() should signal any * thread(s) blocking on MPIDI_CH3I_Sock_wait() and wait for * those * threads to complete before destroying the progress engine * data structures. */ MPIDI_CH3I_Sock_destroy_set(MPIDI_CH3I_sock_set); MPIDI_CH3I_Sock_finalize(); MPIR_THREAD_CHECK_BEGIN; /* FIXME should be appropriately abstracted somehow */ # if defined(MPICH_IS_THREADED) && (MPICH_THREAD_GRANULARITY == MPICH_THREAD_GRANULARITY__GLOBAL) { int err; MPID_Thread_cond_destroy(&MPIDI_CH3I_progress_completion_cond, &err); MPIR_Assert(err == 0); } # endif MPIR_THREAD_CHECK_END; fn_exit: MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3I_PROGRESS_FINALIZE); return mpi_errno; fn_fail: goto fn_exit; }
static int MPIDI_CH3I_Progress_handle_sock_event(MPIDU_Sock_event_t * event) { int mpi_errno = MPI_SUCCESS; MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT); MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT); MPIU_DBG_MSG_D(CH3_OTHER,VERBOSE,"Socket event of type %d", event->op_type ); switch (event->op_type) { case MPIDU_SOCK_OP_READ: { MPIDI_CH3I_Connection_t * conn = (MPIDI_CH3I_Connection_t *) event->user_ptr; MPID_Request * rreq = conn->recv_active; /* --BEGIN ERROR HANDLING-- */ if (event->error != MPI_SUCCESS) { /* FIXME: the following should be handled by the close protocol */ if (MPIR_ERR_GET_CLASS(event->error) != MPIDU_SOCK_ERR_CONN_CLOSED) { mpi_errno = event->error; MPIU_ERR_POP(mpi_errno); } break; } /* --END ERROR HANDLING-- */ if (conn->state == CONN_STATE_CONNECTED) { if (conn->recv_active == NULL) { MPIDI_msg_sz_t buflen = sizeof (MPIDI_CH3_Pkt_t); MPIU_Assert(conn->pkt.type < MPIDI_CH3_PKT_END_CH3); mpi_errno = pktArray[conn->pkt.type]( conn->vc, &conn->pkt, &buflen, &rreq ); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } MPIU_Assert(buflen == sizeof (MPIDI_CH3_Pkt_t)); if (rreq == NULL) { if (conn->state != CONN_STATE_CLOSING) { /* conn->recv_active = NULL; -- already set to NULL */ mpi_errno = connection_post_recv_pkt(conn); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } } } else { mpi_errno = ReadMoreData( conn, rreq ); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } } } else /* incoming data */ { int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *); int complete; reqFn = rreq->dev.OnDataAvail; if (!reqFn) { MPIU_Assert(MPIDI_Request_get_type(rreq)!=MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(rreq); complete = TRUE; } else { mpi_errno = reqFn( conn->vc, rreq, &complete ); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } if (complete) { conn->recv_active = NULL; mpi_errno = connection_post_recv_pkt(conn); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } } else /* more data to be read */ { mpi_errno = ReadMoreData( conn, rreq ); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } } } } else if (conn->state == CONN_STATE_OPEN_LRECV_DATA) { mpi_errno = MPIDI_CH3_Sockconn_handle_connopen_event( conn ); if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); } } else /* Handling some internal connection establishment or tear down packet */ { mpi_errno = MPIDI_CH3_Sockconn_handle_conn_event( conn ); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } } break; } /* END OF SOCK_OP_READ */ case MPIDU_SOCK_OP_WRITE: { MPIDI_CH3I_Connection_t * conn = (MPIDI_CH3I_Connection_t *) event->user_ptr; /* --BEGIN ERROR HANDLING-- */ if (event->error != MPI_SUCCESS) { mpi_errno = event->error; MPIU_ERR_POP(mpi_errno); } /* --END ERROR HANDLING-- */ if (conn->send_active) { MPID_Request * sreq = conn->send_active; int (*reqFn)(MPIDI_VC_t *, MPID_Request *, int *); int complete; reqFn = sreq->dev.OnDataAvail; if (!reqFn) { MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(sreq); complete = TRUE; } else { mpi_errno = reqFn( conn->vc, sreq, &complete ); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } if (complete) { mpi_errno = connection_pop_sendq_req(conn); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } } else /* more data to send */ { for(;;) { MPID_IOV * iovp; MPIU_Size_t nb; iovp = sreq->dev.iov; mpi_errno = MPIDU_Sock_writev(conn->sock, iovp, sreq->dev.iov_count, &nb); /* --BEGIN ERROR HANDLING-- */ if (mpi_errno != MPI_SUCCESS) { mpi_errno = MPIR_Err_create_code(mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|immedwrite", "ch3|sock|immedwrite %p %p %p", sreq, conn, conn->vc); goto fn_fail; } /* --END ERROR HANDLING-- */ MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE, (MPIU_DBG_FDEST,"immediate writev, vc=%p, sreq=0x%08x, nb=" MPIDI_MSG_SZ_FMT, conn->vc, sreq->handle, nb)); if (nb > 0 && adjust_iov(&iovp, &sreq->dev.iov_count, nb)) { reqFn = sreq->dev.OnDataAvail; if (!reqFn) { MPIU_Assert(MPIDI_Request_get_type(sreq)!=MPIDI_REQUEST_TYPE_GET_RESP); MPIDI_CH3U_Request_complete(sreq); complete = TRUE; } else { mpi_errno = reqFn( conn->vc, sreq, &complete ); if (mpi_errno) MPIU_ERR_POP(mpi_errno); } if (complete) { mpi_errno = connection_pop_sendq_req(conn); if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); } break; } } else { MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE, (MPIU_DBG_FDEST,"posting writev, vc=%p, conn=%p, sreq=0x%08x", conn->vc, conn, sreq->handle)); mpi_errno = MPIDU_Sock_post_writev(conn->sock, iovp, sreq->dev.iov_count, NULL); /* --BEGIN ERROR HANDLING-- */ if (mpi_errno != MPI_SUCCESS) { mpi_errno = MPIR_Err_create_code( mpi_errno, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**ch3|sock|postwrite", "ch3|sock|postwrite %p %p %p", sreq, conn, conn->vc); goto fn_fail; } /* --END ERROR HANDLING-- */ break; } } } } else /* finished writing internal packet header */ { /* the connection is not active yet */ mpi_errno = MPIDI_CH3_Sockconn_handle_connwrite( conn ); if (mpi_errno) { MPIU_ERR_POP( mpi_errno ); } } break; } /* END OF SOCK_OP_WRITE */ case MPIDU_SOCK_OP_ACCEPT: { mpi_errno = MPIDI_CH3_Sockconn_handle_accept_event(); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } break; } case MPIDU_SOCK_OP_CONNECT: { mpi_errno = MPIDI_CH3_Sockconn_handle_connect_event( (MPIDI_CH3I_Connection_t *) event->user_ptr, event->error ); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } break; } case MPIDU_SOCK_OP_CLOSE: { mpi_errno = MPIDI_CH3_Sockconn_handle_close_event( (MPIDI_CH3I_Connection_t *) event->user_ptr ); if (mpi_errno) { MPIU_ERR_POP(mpi_errno); } break; } case MPIDU_SOCK_OP_WAKEUP: { MPIDI_CH3_Progress_signal_completion(); /* MPIDI_CH3I_progress_completion_count++; */ break; } } fn_exit: MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3I_PROGRESS_HANDLE_SOCK_EVENT); return mpi_errno; fn_fail: goto fn_exit; }