int
MPID_nem_gm_finalize()
{
int mpi_errno = MPI_SUCCESS;
int max_send_tokens;
MPID_nem_gm_send_queue_t *e;
max_send_tokens = gm_num_send_tokens (MPID_nem_module_gm_port);
while (MPID_nem_module_gm_num_send_tokens < max_send_tokens && !MPID_nem_gm_queue_empty (send))
{
mpi_errno = MPID_nem_gm_recv_poll();
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
}
while (MPID_nem_gm_send_free_queue)
{
e = MPID_nem_gm_send_free_queue;
MPID_nem_gm_send_free_queue = e->next;
MPIU_Free (e);
}
mpi_errno = MPID_nem_gm_lmt_finalize();
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
gm_finalize();
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
PMPI_LOCAL int MPIR_Ibsend_cancel( void *extra, int complete )
{
int mpi_errno = MPI_SUCCESS;
ibsend_req_info *ibsend_info = (ibsend_req_info *)extra;
MPI_Status status;
MPID_Request *req = ibsend_info->req;
/* FIXME: There should be no unreferenced args! */
/* Note that this value should always be 1 because
Grequest_complete is called on this request when it is
created */
MPIU_UNREFERENCED_ARG(complete);
/* Try to cancel the underlying request */
mpi_errno = MPIR_Cancel_impl(req);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIR_Wait_impl( &req->handle, &status );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIR_Test_cancelled_impl( &status, &ibsend_info->cancelled );
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int split_type(MPID_Comm * comm_ptr, int stype, int key,
MPID_Info *info_ptr, MPID_Comm ** newcomm_ptr)
{
MPID_Node_id_t id;
MPIR_Rank_t nid;
int mpi_errno = MPI_SUCCESS;
if (MPIDI_CH3I_Shm_supported()) {
mpi_errno = MPID_Get_node_id(comm_ptr, comm_ptr->rank, &id);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
else
id = comm_ptr->rank;
nid = (stype == MPI_COMM_TYPE_SHARED) ? id : MPI_UNDEFINED;
mpi_errno = MPIR_Comm_split_impl(comm_ptr, nid, key, newcomm_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPID_nem_ptl_init_id(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
char *bc;
int pmi_errno;
int val_max_sz;
MPIU_CHKLMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_PTL_INIT_ID);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_PTL_INIT_ID);
pmi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %d", pmi_errno);
MPIU_CHKLMEM_MALLOC(bc, char *, val_max_sz, mpi_errno, "bc");
mpi_errno = vc->pg->getConnInfo(vc->pg_rank, bc, val_max_sz, vc->pg);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_ptl_get_id_from_bc(bc, &vc_ptl->id, &vc_ptl->pt, &vc_ptl->ptg, &vc_ptl->ptc, &vc_ptl->ptr, &vc_ptl->ptrg, &vc_ptl->ptrc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
vc_ptl->id_initialized = TRUE;
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
fn_exit:
MPIU_CHKLMEM_FREEALL();
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_PTL_INIT_ID);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_SHM_Win_free(MPID_Win **win_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_SHM_WIN_FREE);
MPIDI_RMA_FUNC_ENTER(MPID_STATE_MPIDI_CH3_SHM_WIN_FREE);
/* Free shared memory region */
if ((*win_ptr)->shm_allocated) {
/* free shm_base_addrs that's only used for shared memory windows */
MPIU_Free((*win_ptr)->shm_base_addrs);
/* detach from shared memory segment */
mpi_errno = MPIU_SHMW_Seg_detach((*win_ptr)->shm_segment_handle, (char **)&(*win_ptr)->shm_base_addr,
(*win_ptr)->shm_segment_len);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_SHMW_Hnd_finalize(&(*win_ptr)->shm_segment_handle);
}
mpi_errno = MPIDI_Win_free(win_ptr);
if (mpi_errno != MPI_SUCCESS) { MPIU_ERR_POP(mpi_errno); }
fn_exit:
MPIDI_RMA_FUNC_EXIT(MPID_STATE_MPIDI_CH3_SHM_WIN_FREE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Nem_nd_init(MPIDI_PG_t *pg_p, int pg_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_INIT);
/* Initialize Executive */
mpi_errno = MPIU_ExInitialize();
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
/* Initialize ND state machine */
mpi_errno = MPID_Nem_nd_sm_init();
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
/* Listen for conns & create the business card */
mpi_errno = MPID_Nem_nd_listen_for_conn(pg_rank, bc_val_p, val_max_sz_p);
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_INIT);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPIR_Igatherv_impl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf,
const int recvcounts[], const int displs[], MPI_Datatype recvtype,
int root, MPID_Comm *comm_ptr, MPI_Request *request)
{
int mpi_errno = MPI_SUCCESS;
int tag = -1;
MPID_Request *reqp = NULL;
MPID_Sched_t s = MPID_SCHED_NULL;
*request = MPI_REQUEST_NULL;
mpi_errno = MPID_Sched_next_tag(comm_ptr, &tag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_Sched_create(&s);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_Assert(comm_ptr->coll_fns != NULL);
MPIU_Assert(comm_ptr->coll_fns->Igatherv != NULL);
mpi_errno = comm_ptr->coll_fns->Igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm_ptr, s);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_Sched_start(&s, comm_ptr, tag, &reqp);
if (reqp)
*request = reqp->handle;
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int get_addr(MPIDI_VC_t * vc, struct scif_portID *addr)
{
int mpi_errno = MPI_SUCCESS;
char *bc;
int pmi_errno;
int val_max_sz;
MPIU_CHKLMEM_DECL(1);
/* Allocate space for the business card */
pmi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
MPIU_ERR_CHKANDJUMP1(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**fail",
"**fail %d", pmi_errno);
MPIU_CHKLMEM_MALLOC(bc, char *, val_max_sz, mpi_errno, "bc");
mpi_errno = vc->pg->getConnInfo(vc->pg_rank, bc, val_max_sz, vc->pg);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
mpi_errno = scif_addr_from_bc(bc, &addr->node, &addr->port);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIU_CHKLMEM_FREEALL();
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3I_Comm_init(void)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_COMM_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_COMM_INIT);
MPIR_Add_finalize(register_hook_finalize, NULL, MPIR_FINALIZE_CALLBACK_PRIO-1);
/* register hooks for keeping track of communicators */
mpi_errno = MPIDI_CH3U_Comm_register_create_hook(comm_created, NULL);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
#if defined HAVE_LIBHCOLL
if (MPIR_CVAR_CH3_ENABLE_HCOLL) {
mpi_errno = MPIDI_CH3U_Comm_register_create_hook(hcoll_comm_create, NULL);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIDI_CH3U_Comm_register_destroy_hook(hcoll_comm_destroy, NULL);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
#endif
mpi_errno = MPIDI_CH3U_Comm_register_destroy_hook(comm_destroyed, NULL);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_COMM_INIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int send_sreq_data(MPIDI_VC_t *vc, MPID_Request *sreq, knem_cookie_t *s_cookiep)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig;
MPI_Aint dt_true_lb;
MPIDI_msg_sz_t data_sz;
MPID_Datatype * dt_ptr;
/* MT: this code assumes only one thread can be at this point at a time */
if (knem_fd < 0) {
mpi_errno = open_knem_dev();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
/* find out contig/noncontig, size, and lb for the datatype */
MPIDI_Datatype_get_info(sreq->dev.user_count, sreq->dev.datatype,
dt_contig, data_sz, dt_ptr, dt_true_lb);
if (dt_contig) {
/* handle the iov creation ourselves */
sreq->dev.iov[0].MPID_IOV_BUF = (char *)sreq->dev.user_buf + dt_true_lb;
sreq->dev.iov[0].MPID_IOV_LEN = data_sz;
sreq->dev.iov_count = 1;
}
else {
/* use the segment routines to handle the iovec creation */
if (sreq->dev.segment_ptr == NULL) {
sreq->dev.iov_count = MPID_IOV_LIMIT;
sreq->dev.iov_offset = 0;
/* segment_ptr may be non-null when this is a continuation of a
many-part message that we couldn't fit in one single flight of
iovs. */
sreq->dev.segment_ptr = MPID_Segment_alloc();
MPIU_ERR_CHKANDJUMP1((sreq->dev.segment_ptr == NULL), mpi_errno,
MPI_ERR_OTHER, "**nomem",
"**nomem %s", "MPID_Segment_alloc");
MPID_Segment_init(sreq->dev.user_buf, sreq->dev.user_count,
sreq->dev.datatype, sreq->dev.segment_ptr, 0);
sreq->dev.segment_first = 0;
sreq->dev.segment_size = data_sz;
/* FIXME we should write our own function that isn't dependent on
the in-request iov array. This will let us use IOVs that are
larger than MPID_IOV_LIMIT. */
mpi_errno = MPIDI_CH3U_Request_load_send_iov(sreq, &sreq->dev.iov[0],
&sreq->dev.iov_count);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
}
mpi_errno = do_dma_send(vc, sreq, sreq->dev.iov_count, sreq->dev.iov, s_cookiep);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Comm_dup_impl(MPID_Comm *comm_ptr, MPID_Comm **newcomm_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPID_Attribute *new_attributes = 0;
/* Copy attributes, executing the attribute copy functions */
/* This accesses the attribute dup function through the perprocess
structure to prevent comm_dup from forcing the linking of the
attribute functions. The actual function is (by default)
MPIR_Attr_dup_list
*/
if (MPIR_Process.attr_dup) {
mpi_errno = MPIR_Process.attr_dup( comm_ptr->handle,
comm_ptr->attributes,
&new_attributes );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
/* Generate a new context value and a new communicator structure */
/* We must use the local size, because this is compared to the
rank of the process in the communicator. For intercomms,
this must be the local size */
mpi_errno = MPIR_Comm_copy( comm_ptr, comm_ptr->local_size, newcomm_ptr );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
(*newcomm_ptr)->attributes = new_attributes;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Nem_nd_vc_terminate (MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_ND_VC_TERMINATE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_ND_VC_TERMINATE);
/* Poll till no more pending/posted sends */
while(!MPID_NEM_ND_VCCH_NETMOD_POSTED_SENDQ_EMPTY(vc)
|| !MPID_NEM_ND_VCCH_NETMOD_PENDING_SENDQ_EMPTY(vc)){
mpi_errno = MPID_Nem_nd_sm_poll(1);
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
}
vc_ch->next = NULL;
vc_ch->prev = NULL;
MPID_NEM_ND_VCCH_NETMOD_STATE_SET(vc, MPID_NEM_ND_VC_STATE_DISCONNECTED);
mpi_errno = MPID_Nem_nd_conn_disc(MPID_NEM_ND_VCCH_NETMOD_CONN_HND_GET(vc));
if(mpi_errno != MPI_SUCCESS) MPIU_ERR_POP(mpi_errno);
MPID_NEM_ND_VCCH_NETMOD_CONN_HND_SET(vc, MPID_NEM_ND_CONN_HND_INVALID);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_ND_VC_TERMINATE);
return mpi_errno;
fn_fail:
MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "failed, mpi_errno = %d", mpi_errno);
goto fn_exit;
}
int MPIR_Alltoallv(const void *sendbuf, const int *sendcounts, const int *sdispls,
MPI_Datatype sendtype, void *recvbuf, const int *recvcounts, const int *rdispls,
MPI_Datatype recvtype, MPID_Comm *comm_ptr, int *errflag)
{
int mpi_errno = MPI_SUCCESS;
if (comm_ptr->comm_kind == MPID_INTRACOMM) {
/* intracommunicator */
mpi_errno = MPIR_Alltoallv_intra(sendbuf, sendcounts, sdispls,
sendtype, recvbuf, recvcounts,
rdispls, recvtype, comm_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else {
/* intercommunicator */
mpi_errno = MPIR_Alltoallv_inter(sendbuf, sendcounts, sdispls,
sendtype, recvbuf, recvcounts,
rdispls, recvtype, comm_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Iallreduce_naive(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr, MPID_Sched_t s)
{
int mpi_errno = MPI_SUCCESS;
int rank;
rank = comm_ptr->rank;
if ((sendbuf == MPI_IN_PLACE) && (rank != 0)) {
mpi_errno = MPIR_Ireduce_intra(recvbuf, NULL, count, datatype, op, 0, comm_ptr, s);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
else {
mpi_errno = MPIR_Ireduce_intra(sendbuf, recvbuf, count, datatype, op, 0, comm_ptr, s);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
MPID_SCHED_BARRIER(s);
mpi_errno = MPIR_Ibcast_intra(recvbuf, count, datatype, 0, comm_ptr, s);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Allreduce_impl(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr,
int *errflag)
{
int mpi_errno = MPI_SUCCESS;
if (comm_ptr->coll_fns != NULL && comm_ptr->coll_fns->Allreduce != NULL)
{
mpi_errno = comm_ptr->coll_fns->Allreduce(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
else
{
if (comm_ptr->comm_kind == MPID_INTRACOMM) {
/* intracommunicator */
mpi_errno = MPIR_Allreduce_intra(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
else {
/* intercommunicator */
mpi_errno = MPIR_Allreduce_inter(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3I_RMA_Make_progress_win(MPID_Win * win_ptr, int *made_progress)
{
int temp_progress = 0;
int mpi_errno = MPI_SUCCESS;
(*made_progress) = 0;
/* check window state */
mpi_errno = check_window_state(win_ptr, &temp_progress);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
if (temp_progress)
(*made_progress) = 1;
/* issue operations on window */
mpi_errno = issue_ops_win(win_ptr, &temp_progress);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
if (temp_progress)
(*made_progress) = 1;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Topology_put( MPID_Comm *comm_ptr, MPIR_Topology *topo_ptr )
{
int mpi_errno = MPI_SUCCESS;
MPIU_Assert(comm_ptr != NULL);
if (MPIR_Topology_keyval == MPI_KEYVAL_INVALID) {
/* Create a new keyval */
/* FIXME - thread safe code needs a thread lock here, followed
by another test on the keyval to see if a different thread
got there first */
mpi_errno = MPIR_Comm_create_keyval_impl( MPIR_Topology_copy_fn,
MPIR_Topology_delete_fn,
&MPIR_Topology_keyval, 0 );
/* Register the finalize handler */
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIR_Add_finalize( MPIR_Topology_finalize, (void*)0,
MPIR_FINALIZE_CALLBACK_PRIO-1);
}
mpi_errno = MPIR_Comm_set_attr_impl(comm_ptr, MPIR_Topology_keyval, topo_ptr, MPIR_ATTR_PTR);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Iexscan_impl(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPID_Comm *comm_ptr, MPI_Request *request)
{
int mpi_errno = MPI_SUCCESS;
int tag = -1;
MPID_Request *reqp = NULL;
MPID_Sched_t s = MPID_SCHED_NULL;
*request = MPI_REQUEST_NULL;
mpi_errno = MPID_Sched_next_tag(comm_ptr, &tag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_Sched_create(&s);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_Assert(comm_ptr->coll_fns != NULL);
MPIU_Assert(comm_ptr->coll_fns->Iexscan != NULL);
mpi_errno = comm_ptr->coll_fns->Iexscan(sendbuf, recvbuf, count, datatype, op, comm_ptr, s);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_Sched_start(&s, comm_ptr, tag, &reqp);
if (reqp)
*request = reqp->handle;
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int vc_terminate(MPIDI_VC_t *vc)
{
int mpi_errno = MPI_SUCCESS;
int req_errno = MPI_SUCCESS;
MPID_nem_ptl_vc_area *const vc_ptl = VC_PTL(vc);
MPIDI_STATE_DECL(MPID_STATE_VC_TERMINATE);
MPIDI_FUNC_ENTER(MPID_STATE_VC_TERMINATE);
if (vc->state != MPIDI_VC_STATE_CLOSED) {
/* VC is terminated as a result of a fault. Complete
outstanding sends with an error and terminate
connection immediately. */
MPIU_ERR_SET1(req_errno, MPIX_ERR_PROC_FAILED, "**comm_fail", "**comm_fail %d", vc->pg_rank);
mpi_errno = MPID_nem_ptl_vc_terminated(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else if (vc_ptl->num_queued_sends == 0) {
mpi_errno = MPID_nem_ptl_vc_terminated(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else {
/* the send_queued function will call vc_terminated if vc->state is
CLOSED and the last queued send has been sent*/
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_VC_TERMINATE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIR_Alltoallv_impl(const void *sendbuf, const int *sendcounts, const int *sdispls,
MPI_Datatype sendtype, void *recvbuf, const int *recvcounts,
const int *rdispls, MPI_Datatype recvtype,
MPID_Comm *comm_ptr, int *errflag)
{
int mpi_errno = MPI_SUCCESS;
if (comm_ptr->coll_fns != NULL && comm_ptr->coll_fns->Alltoallv != NULL) {
/* --BEGIN USEREXTENSION-- */
mpi_errno = comm_ptr->coll_fns->Alltoallv(sendbuf, sendcounts, sdispls,
sendtype, recvbuf, recvcounts,
rdispls, recvtype, comm_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* --END USEREXTENSION-- */
} else {
mpi_errno = MPIR_Alltoallv(sendbuf, sendcounts, sdispls,
sendtype, recvbuf, recvcounts,
rdispls, recvtype, comm_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIC_Send(void *buf, int count, MPI_Datatype datatype, int dest, int tag,
MPI_Comm comm)
{
int mpi_errno, context_id;
MPID_Request *request_ptr=NULL;
MPID_Comm *comm_ptr=NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_SEND);
MPIDI_PT2PT_FUNC_ENTER_FRONT(MPID_STATE_MPIC_SEND);
MPID_Comm_get_ptr( comm, comm_ptr );
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
mpi_errno = MPID_Send(buf, count, datatype, dest, tag, comm_ptr,
context_id, &request_ptr);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
if (request_ptr) {
mpi_errno = MPIC_Wait(request_ptr);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
MPID_Request_release(request_ptr);
}
fn_exit:
MPIDI_PT2PT_FUNC_EXIT(MPID_STATE_MPIC_SEND);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (request_ptr) {
MPID_Request_release(request_ptr);
}
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int
MPID_nem_mx_init (MPIDI_PG_t *pg_p, int pg_rank,
char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS ;
MPID_nem_mx_internal_req_queue_init();
mpi_errno = init_mx(pg_p);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_mx_get_business_card (pg_rank, bc_val_p, val_max_sz_p);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
mx_register_unexp_handler(MPID_nem_mx_local_endpoint,MPID_nem_mx_get_adi_msg,NULL);
mpi_errno = MPIDI_CH3I_Register_anysource_notification(MPID_nem_mx_anysource_posted,
MPID_nem_mx_anysource_matched);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_mxm_vc_init(MPIDI_VC_t * vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_CH3I_VC *vc_ch = &vc->ch;
MPID_nem_mxm_vc_area *vc_area = VC_BASE(vc);
MPIDI_STATE_DECL(MPID_STATE_MXM_VC_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MXM_VC_INIT);
/* local connection is used for any source communication */
MPIU_Assert(MPID_nem_mem_region.rank != vc->lpid);
MPIU_DBG_MSG_FMT(CH3_CHANNEL, VERBOSE,
(MPIU_DBG_FDEST,
"[%i]=== connecting to %i \n", MPID_nem_mem_region.rank, vc->lpid));
{
char *business_card;
int val_max_sz;
#ifdef USE_PMI2_API
val_max_sz = PMI2_MAX_VALLEN;
#else
mpi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
#endif
business_card = (char *) MPIU_Malloc(val_max_sz);
mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card, val_max_sz, vc->pg);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
vc_area->ctx = vc;
vc_area->mxm_ep = &_mxm_obj.endpoint[vc->pg_rank];
mpi_errno = _mxm_connect(&_mxm_obj.endpoint[vc->pg_rank], business_card, vc_area);
if (mpi_errno)
MPIU_ERR_POP(mpi_errno);
MPIU_Free(business_card);
}
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
vc_area->pending_sends = 0;
vc->rndvSend_fn = NULL;
vc->rndvRecv_fn = NULL;
vc->sendNoncontig_fn = MPID_nem_mxm_SendNoncontig;
vc->comm_ops = &comm_ops;
vc_ch->iStartContigMsg = MPID_nem_mxm_iStartContigMsg;
vc_ch->iSendContig = MPID_nem_mxm_iSendContig;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MXM_VC_INIT);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static inline int issue_ops_win(MPID_Win * win_ptr, int *made_progress)
{
int mpi_errno = MPI_SUCCESS;
int start_slot, end_slot, i, idx;
MPIDI_RMA_Target_t *target = NULL;
(*made_progress) = 0;
if (win_ptr->non_empty_slots == 0)
goto fn_exit;
/* Exit if window-wide sync is not finished */
if (win_ptr->states.access_state == MPIDI_RMA_NONE ||
win_ptr->states.access_state == MPIDI_RMA_FENCE_ISSUED ||
win_ptr->states.access_state == MPIDI_RMA_PSCW_ISSUED ||
win_ptr->states.access_state == MPIDI_RMA_LOCK_ALL_ISSUED)
goto fn_exit;
/* FIXME: we should optimize the issuing pattern here. */
start_slot = win_ptr->comm_ptr->rank % win_ptr->num_slots;
end_slot = start_slot + win_ptr->num_slots;
for (i = start_slot; i < end_slot; i++) {
if (i < win_ptr->num_slots)
idx = i;
else
idx = i - win_ptr->num_slots;
target = win_ptr->slots[idx].target_list;
while (target != NULL) {
int temp_progress = 0;
/* check target state */
mpi_errno = check_target_state(win_ptr, target, &temp_progress);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
if (temp_progress)
(*made_progress) = 1;
/* issue operations to this target */
mpi_errno = issue_ops_target(win_ptr, target, &temp_progress);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
if (temp_progress)
(*made_progress) = 1;
target = target->next;
}
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_nem_ckpt_finish(void)
{
int mpi_errno = MPI_SUCCESS;
int i;
int ret;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_NEM_CKPT_FINISH);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_NEM_CKPT_FINISH);
/* Since we're checkpointing the shared memory region (i.e., the
channels between local procs), we don't have to flush those
channels, just make sure no one is sending or receiving during
the checkpoint */
mpi_errno = MPID_nem_barrier();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
do {
ret = sem_post(&ckpt_sem);
} while (ret == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**sem_post", "**sem_post %s", MPIU_Strerror(errno));
do {
ret = sem_wait(&cont_sem);
} while (ret == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP1(ret, mpi_errno, MPI_ERR_OTHER, "**sem_wait", "**sem_wait %s", MPIU_Strerror(errno));
mpi_errno = MPID_nem_barrier();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (ckpt_result == CKPT_CONTINUE) {
for (i = 0; i < MPIDI_Process.my_pg->size; ++i) {
MPIDI_VC_t *vc;
MPIDI_CH3I_VC *vc_ch;
/* We didn't send a marker to ourselves. */
if (i == MPIDI_Process.my_pg_rank)
continue;
MPIDI_PG_Get_vc(MPIDI_Process.my_pg, i, &vc);
vc_ch = &vc->ch;
if (!vc_ch->is_local) {
mpi_errno = vc_ch->ckpt_continue_vc(vc);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
}
}
checkpointing = FALSE;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_NEM_CKPT_FINISH);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_Iprobe(int source, int tag, MPID_Comm *comm, int context_offset,
int *flag, MPI_Status *status)
{
const int context = comm->recvcontext_id + context_offset;
int found = 0;
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPID_IPROBE);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_IPROBE);
if (source == MPI_PROC_NULL)
{
MPIR_Status_set_procnull(status);
/* We set the flag to true because an MPI_Recv with this rank will
return immediately */
*flag = TRUE;
goto fn_exit;
}
/* Check to make sure the communicator hasn't already been revoked */
if (comm->revoked &&
MPIR_AGREE_TAG != MPIR_TAG_MASK_ERROR_BITS(tag & ~MPIR_Process.tagged_coll_mask) &&
MPIR_SHRINK_TAG != MPIR_TAG_MASK_ERROR_BITS(tag & ~MPIR_Process.tagged_coll_mask)) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPIX_ERR_REVOKED,"**revoked");
}
#ifdef ENABLE_COMM_OVERRIDES
if (MPIDI_Anysource_iprobe_fn) {
if (source == MPI_ANY_SOURCE) {
/* if it's anysource, check shm, then check the network.
If still not found, call progress, and check again. */
/* check shm*/
MPIU_THREAD_CS_ENTER(MSGQUEUE,);
found = MPIDI_CH3U_Recvq_FU(source, tag, context, status);
MPIU_THREAD_CS_EXIT(MSGQUEUE,);
if (!found) {
/* not found, check network */
mpi_errno = MPIDI_Anysource_iprobe_fn(tag, comm, context_offset, &found, status);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (!found) {
/* still not found, make some progress*/
mpi_errno = MPIDI_CH3_Progress_poke();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* check shm again */
MPIU_THREAD_CS_ENTER(MSGQUEUE,);
found = MPIDI_CH3U_Recvq_FU(source, tag, context, status);
MPIU_THREAD_CS_EXIT(MSGQUEUE,);
if (!found) {
/* check network again */
mpi_errno = MPIDI_Anysource_iprobe_fn(tag, comm, context_offset, &found, status);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
}
}
int
MPID_nem_mx_vc_init (MPIDI_VC_t *vc)
{
uint32_t threshold;
MPIDI_CH3I_VC *vc_ch = VC_CH(vc);
int mpi_errno = MPI_SUCCESS;
/* first make sure that our private fields in the vc fit into the area provided */
MPIU_Assert(sizeof(MPID_nem_mx_vc_area) <= MPID_NEM_VC_NETMOD_AREA_LEN);
#ifdef ONDEMAND
VC_FIELD(vc, local_connected) = 0;
VC_FIELD(vc, remote_connected) = 0;
#else
{
char *business_card;
int val_max_sz;
int ret;
#ifdef USE_PMI2_API
val_max_sz = PMI2_MAX_VALLEN;
#else
mpi_errno = PMI_KVS_Get_value_length_max(&val_max_sz);
#endif
business_card = (char *)MPIU_Malloc(val_max_sz);
mpi_errno = vc->pg->getConnInfo(vc->pg_rank, business_card,val_max_sz, vc->pg);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_mx_get_from_bc (business_card, &VC_FIELD(vc, remote_endpoint_id), &VC_FIELD(vc, remote_nic_id));
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
MPIU_Free(business_card);
ret = mx_connect(MPID_nem_mx_local_endpoint,VC_FIELD(vc, remote_nic_id),VC_FIELD(vc, remote_endpoint_id),
MPID_NEM_MX_FILTER,MX_INFINITE,&(VC_FIELD(vc, remote_endpoint_addr)));
MPIU_ERR_CHKANDJUMP1 (ret != MX_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**mx_connect", "**mx_connect %s", mx_strerror (ret));
mx_set_endpoint_addr_context(VC_FIELD(vc, remote_endpoint_addr),(void *)vc);
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
}
#endif
mx_get_info(MPID_nem_mx_local_endpoint, MX_COPY_SEND_MAX, NULL, 0, &threshold, sizeof(uint32_t));
vc->eager_max_msg_sz = threshold;
vc->rndvSend_fn = NULL;
vc->sendNoncontig_fn = MPID_nem_mx_SendNoncontig;
vc->comm_ops = &comm_ops;
vc_ch->iStartContigMsg = MPID_nem_mx_iStartContigMsg;
vc_ch->iSendContig = MPID_nem_mx_iSendContig;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int
MPID_nem_newmad_init (MPIDI_PG_t *pg_p, int pg_rank,
char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS ;
int index;
/*
fprintf(stdout,"Size of MPID_nem_mad_module_vc_area_internal_t : %i | size of nm_sr_request_t :%i | Size of req_area : %i\n",
sizeof(MPID_nem_newmad_vc_area_internal_t),sizeof(nm_sr_request_t), sizeof(MPID_nem_newmad_req_area));
*/
/*
MPIU_Assert( sizeof(MPID_nem_newmad_vc_area_internal_t) <= MPID_NEM_VC_NETMOD_AREA_LEN);
MPIU_Assert( sizeof(MPID_nem_newmad_req_area) <= MPID_NEM_REQ_NETMOD_AREA_LEN);
*/
if (sizeof(MPID_nem_newmad_vc_area) > MPID_NEM_VC_NETMOD_AREA_LEN)
{
fprintf(stdout,"===========================================================\n");
fprintf(stdout,"=== Error : Newmad data structure size is too long ===\n");
fprintf(stdout,"=== VC netmod area is %4i | Nmad struct size is %4i ===\n",
MPID_NEM_VC_NETMOD_AREA_LEN, sizeof(MPID_nem_newmad_vc_area));
fprintf(stdout,"===========================================================\n");
exit(0);
}
if (sizeof(MPID_nem_newmad_req_area) > MPID_NEM_REQ_NETMOD_AREA_LEN)
{
fprintf(stdout,"===========================================================\n");
fprintf(stdout,"=== Error : Newmad data structure size is too long ===\n");
fprintf(stdout,"=== Req netmod area is %4i | Nmad struct size is %4i ===\n",
MPID_NEM_REQ_NETMOD_AREA_LEN, sizeof(MPID_nem_newmad_req_area));
fprintf(stdout,"===========================================================\n");
exit(0);
}
mpid_nem_newmad_myrank = pg_rank;
init_mad(pg_p);
mpi_errno = MPID_nem_newmad_get_business_card(pg_rank,bc_val_p, val_max_sz_p);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
mpi_errno = MPIDI_CH3I_Register_anysource_notification(MPID_nem_newmad_anysource_posted, MPID_nem_newmad_anysource_matched);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_nem_register_initcomp_cb(MPID_nem_newmad_init_completed);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIC_Ssend(const void *buf, MPI_Aint count, MPI_Datatype datatype, int dest, int tag,
MPID_Comm *comm_ptr, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
int context_id;
MPID_Request *request_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_SSEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SSEND);
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "IN: errflag = %d", *errflag);
MPIU_ERR_CHKANDJUMP1((count < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", count);
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
switch(*errflag) {
case MPIR_ERR_NONE:
break;
case MPIR_ERR_PROC_FAILED:
MPIR_TAG_SET_PROC_FAILURE_BIT(tag);
default:
MPIR_TAG_SET_ERROR_BIT(tag);
}
mpi_errno = MPID_Ssend(buf, count, datatype, dest, tag, comm_ptr,
context_id, &request_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (request_ptr) {
mpi_errno = MPIC_Wait(request_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPID_Request_release(request_ptr);
}
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SSEND);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
if (request_ptr) MPID_Request_release(request_ptr);
if (mpi_errno && !*errflag) {
if (MPIX_ERR_PROC_FAILED == MPIR_ERR_GET_CLASS(mpi_errno)) {
*errflag = MPIR_ERR_PROC_FAILED;
} else {
*errflag = MPIR_ERR_OTHER;
}
}
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIDI_CH3I_RMA_Cleanup_ops_aggressive(MPID_Win * win_ptr)
{
int i, local_completed = 0, remote_completed = 0;
int mpi_errno = MPI_SUCCESS;
MPIDI_RMA_Target_t *curr_target = NULL;
int made_progress = 0;
/* If we are in an aggressive cleanup, the window must be holding
* up resources. If it isn't, we are in the wrong window and
* incorrectly entered this function. */
MPIU_ERR_CHKANDJUMP(win_ptr->non_empty_slots == 0, mpi_errno, MPI_ERR_OTHER, "**rmanoop");
/* find the first target that has something to issue */
for (i = 0; i < win_ptr->num_slots; i++) {
if (win_ptr->slots[i].target_list != NULL) {
curr_target = win_ptr->slots[i].target_list;
while (curr_target != NULL && curr_target->pending_op_list == NULL)
curr_target = curr_target->next;
if (curr_target != NULL)
break;
}
}
if (curr_target == NULL)
goto fn_exit;
if (curr_target->sync.sync_flag < MPIDI_RMA_SYNC_FLUSH_LOCAL)
curr_target->sync.sync_flag = MPIDI_RMA_SYNC_FLUSH_LOCAL;
/* Issue out all operations. */
mpi_errno = MPIDI_CH3I_RMA_Make_progress_target(win_ptr, curr_target->target_rank,
&made_progress);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
/* Wait for local completion. */
do {
mpi_errno = MPIDI_CH3I_RMA_Cleanup_ops_target(win_ptr, curr_target,
&local_completed, &remote_completed);
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
if (!local_completed) {
mpi_errno = wait_progress_engine();
if (mpi_errno != MPI_SUCCESS)
MPIU_ERR_POP(mpi_errno);
}
} while (!local_completed);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
