void
MPIDI_WinLockAdvance(pami_context_t context,
MPID_Win * win)
{
struct MPIDI_Win_sync_lock* slock = &win->mpid.sync.lock;
struct MPIDI_Win_queue* q = &slock->local.requested;
if (
(q->head != NULL ) &&
( (slock->local.count == 0) ||
(
(slock->local.type == MPI_LOCK_SHARED) &&
(q->head->type == MPI_LOCK_SHARED)
)
)
)
{
struct MPIDI_Win_lock* lock = q->head;
q->head = lock->next;
if (q->head == NULL)
q->tail = NULL;
++slock->local.count;
slock->local.type = lock->type;
if (lock->mtype == MPIDI_REQUEST_LOCK)
MPIDI_WinLockAck_post(context, lock->rank, win);
else if (lock->mtype == MPIDI_REQUEST_LOCKALL)
MPIDI_WinLockAllAck_post(context, lock->rank, win);
else
MPID_assert_always(0);
MPL_free(lock);
MPIDI_WinLockAdvance(context, win);
}
}
int MPIDI_Win_set_info(MPID_Win *win, MPID_Info *info)
{
int mpi_errno = MPI_SUCCESS;
MPID_Info *curr_ptr;
char *value, *token;
char *savePtr;
curr_ptr = info->next;
uint save_ordering;
while (curr_ptr) {
if (!strcmp(curr_ptr->key,"no_locks")) {
if (!strcmp(curr_ptr->value,"true")) {
win->mpid.info_args.no_locks=1;
} else
win->mpid.info_args.no_locks=0;
} else if (!strcmp(curr_ptr->key,"accumulate_ordering"))
{
save_ordering=(uint) win->mpid.info_args.accumulate_ordering;
win->mpid.info_args.accumulate_ordering=0;
value = curr_ptr->value;
token = (char *) strtok_r(value,"," , &savePtr);
while (token) {
if (!memcmp(token,"rar",3))
{
win->mpid.info_args.accumulate_ordering =
(win->mpid.info_args.accumulate_ordering | MPIDI_ACCU_ORDER_RAR);
} else if (!memcmp(token,"raw",3))
{
win->mpid.info_args.accumulate_ordering =
(win->mpid.info_args.accumulate_ordering | MPIDI_ACCU_ORDER_RAW);
} else if (!memcmp(token,"war",3))
{
win->mpid.info_args.accumulate_ordering =
(win->mpid.info_args.accumulate_ordering | MPIDI_ACCU_ORDER_WAR);
} else if (!memcmp(token,"waw",3))
{
win->mpid.info_args.accumulate_ordering =
(win->mpid.info_args.accumulate_ordering | MPIDI_ACCU_ORDER_WAW);
} else
MPID_assert_always(0);
token = (char *) strtok_r(NULL,"," , &savePtr);
}
if (win->mpid.info_args.accumulate_ordering == 0) {
win->mpid.info_args.accumulate_ordering=
(MPIDI_Win_info_accumulate_ordering) save_ordering;
}
} else if (!strcmp(curr_ptr->key,"accumulate_ops"))
{
/* the default setting is MPIDI_ACCU_SAME_OP_NO_OP */
if (!strcmp(curr_ptr->value,"same_op"))
win->mpid.info_args.accumulate_ops = MPIDI_ACCU_SAME_OP;
}
curr_ptr = curr_ptr->next;
}
return mpi_errno;
}
static void MPIDI_Update_coll(pami_algorithm_t coll,
int type, /* must query vs always works */
int index,
MPID_Comm *comm)
{
comm->mpid.user_selected_type[coll] = type;
TRACE_ERR("Update_coll for protocol %s, type: %d index: %d\n",
comm->mpid.coll_metadata[coll][type][index].name, type, index);
/* Are we in the 'must query' list? If so determine how "bad" it is */
if(type == MPID_COLL_QUERY)
{
/* First, is a check always required? */
if(comm->mpid.coll_metadata[coll][type][index].check_correct.values.checkrequired)
{
TRACE_ERR("Protocol %s check_fn required always\n", comm->mpid.coll_metadata[coll][type][index].name);
/* We must have a check_fn */
MPID_assert_always(comm->mpid.coll_metadata[coll][type][index].check_fn !=NULL);
comm->mpid.user_selected_type[coll] = MPID_COLL_CHECK_FN_REQUIRED;
}
else if(comm->mpid.coll_metadata[coll][type][index].check_fn != NULL)
{
/* For now, if there's a check_fn we will always call it and not cache.
We *could* be smarter about this eventually. */
TRACE_ERR("Protocol %s setting to always query\n", comm->mpid.coll_metadata[coll][type][index].name);
comm->mpid.user_selected_type[coll] = MPID_COLL_ALWAYS_QUERY;
}
}
comm->mpid.user_selected[coll] =
comm->mpid.coll_algorithm[coll][type][index];
memcpy(&comm->mpid.user_metadata[coll],
&comm->mpid.coll_metadata[coll][type][index],
sizeof(pami_metadata_t));
}
static inline void
MPIDI_RecvShortCB(pami_context_t context,
const void * _msginfo,
const void * sndbuf,
size_t sndlen,
pami_endpoint_t sender,
unsigned isSync)
{
MPID_assert(_msginfo != NULL);
const MPIDI_MsgInfo *msginfo = (const MPIDI_MsgInfo *)_msginfo;
MPID_Request * rreq = NULL;
pami_task_t source;
#if TOKEN_FLOW_CONTROL
int rettoks=0;
#endif
/* -------------------- */
/* Match the request. */
/* -------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
source = PAMIX_Endpoint_query(sender);
MPIDI_Receive_tokens(msginfo,source);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, source, tag, context_id, msginfo->MPIseqno);
#endif
/* Match not found */
if (unlikely(rreq == NULL))
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivals);
#endif
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request *newreq = MPIDI_Request_create2();
MPID_assert(newreq != NULL);
if (sndlen)
{
newreq->mpid.uebuflen = sndlen;
if (!TOKEN_FLOW_CONTROL_ON)
{
newreq->mpid.uebuf = MPL_malloc(sndlen);
newreq->mpid.uebuf_malloc = mpiuMalloc;
}
else
{
#if TOKEN_FLOW_CONTROL
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
newreq->mpid.uebuf = MPIDI_mm_alloc(sndlen);
newreq->mpid.uebuf_malloc = mpidiBufMM;
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#else
MPID_assert_always(0);
#endif
}
MPID_assert(newreq->mpid.uebuf != NULL);
}
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP(rank, tag, context_id);
#else
rreq = MPIDI_Recvq_FDP(rank, PAMIX_Endpoint_query(sender), tag, context_id, msginfo->MPIseqno);
#endif
if (unlikely(rreq == NULL))
{
MPIDI_Callback_process_unexp(newreq, context, msginfo, sndlen, sender, sndbuf, NULL, isSync);
/* request is always complete now */
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Token_cntr[source].unmatched++;
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_release(newreq);
goto fn_exit_short;
}
else
{
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
MPID_Request_discard(newreq);
}
}
else
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivalsMatched);
#endif
if (TOKEN_FLOW_CONTROL_ON && sndlen)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Update_rettoks(source);
MPIDI_Must_return_tokens(context,source);
#else
MPID_assert_always(0);
#endif
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
}
/* the receive queue processing has been completed and we found match*/
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, sndlen);
MPIDI_Request_setCA (rreq, MPIDI_CA_COMPLETE);
MPIDI_Request_cpyPeerRequestH(rreq, msginfo);
MPIDI_Request_setSync (rreq, isSync);
MPIDI_Request_setRzv (rreq, 0);
/* ----------------------------- */
/* Request was already posted. */
/* ----------------------------- */
if (unlikely(isSync))
MPIDI_SyncAck_post(context, rreq, PAMIX_Endpoint_query(sender));
if (unlikely(HANDLE_GET_KIND(rreq->mpid.datatype) != HANDLE_KIND_BUILTIN))
{
MPIDI_Callback_process_userdefined_dt(context, sndbuf, sndlen, rreq);
goto fn_exit_short;
}
size_t dt_size = rreq->mpid.userbufcount * MPID_Datatype_get_basic_size(rreq->mpid.datatype);
/* ----------------------------- */
/* Test for truncated message. */
/* ----------------------------- */
if (unlikely(sndlen > dt_size))
{
#if ASSERT_LEVEL > 0
MPIDI_Callback_process_trunc(context, rreq, NULL, sndbuf);
goto fn_exit_short;
#else
sndlen = dt_size;
#endif
}
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
void* rcvbuf = rreq->mpid.userbuf;
if (sndlen > 0)
{
#if CUDA_AWARE_SUPPORT
if(MPIDI_Process.cuda_aware_support_on && MPIDI_cuda_is_device_buf(rcvbuf))
{
cudaError_t cudaerr = CudaMemcpy(rcvbuf, sndbuf, (size_t)sndlen, cudaMemcpyHostToDevice);
}
else
#endif
memcpy(rcvbuf, sndbuf, sndlen);
}
TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,sndlen);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.comp_in_HH);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf);
MPIDI_Request_complete(rreq);
fn_exit_short:
#ifdef OUT_OF_ORDER_HANDLING
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
if (MPIDI_In_cntr[source].n_OutOfOrderMsgs>0) {
MPIDI_Recvq_process_out_of_order_msgs(source, context);
}
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
#endif
/* ---------------------------------------- */
/* Signal that the recv has been started. */
/* ---------------------------------------- */
MPIDI_Progress_signal();
}
void MPIDI_Comm_coll_envvars(MPID_Comm *comm)
{
char *envopts;
int i;
MPID_assert_always(comm!=NULL);
TRACE_ERR("MPIDI_Comm_coll_envvars enter\n");
/* Set up always-works defaults */
for(i = 0; i < PAMI_XFER_COUNT; i++)
{
if(i == PAMI_XFER_AMBROADCAST || i == PAMI_XFER_AMSCATTER ||
i == PAMI_XFER_AMGATHER || i == PAMI_XFER_AMREDUCE)
continue;
/* Initialize to noselection instead of noquery for PE/FCA stuff. Is this the right answer? */
comm->mpid.user_selected_type[i] = MPID_COLL_NOSELECTION;
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Setting up collective %d on comm %p\n", i, comm);
if(comm->mpid.coll_count[i][0] == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"There are no 'always works' protocols of type %d. This could be a problem later in your app\n", i);
comm->mpid.user_selected_type[i] = MPID_COLL_USE_MPICH;
comm->mpid.user_selected[i] = 0;
}
else
{
comm->mpid.user_selected[i] = comm->mpid.coll_algorithm[i][0][0];
memcpy(&comm->mpid.user_metadata[i], &comm->mpid.coll_metadata[i][0][0],
sizeof(pami_metadata_t));
}
}
TRACE_ERR("Checking env vars\n");
MPIDI_Check_preallreduce("PAMID_COLLECTIVE_ALLGATHER_PREALLREDUCE", comm, "allgather",
MPID_ALLGATHER_PREALLREDUCE);
MPIDI_Check_preallreduce("PAMID_COLLECTIVE_ALLGATHERV_PREALLREDUCE", comm, "allgatherv",
MPID_ALLGATHERV_PREALLREDUCE);
MPIDI_Check_preallreduce("PAMID_COLLECTIVE_ALLREDUCE_PREALLREDUCE", comm, "allreduce",
MPID_ALLREDUCE_PREALLREDUCE);
MPIDI_Check_preallreduce("PAMID_COLLECTIVE_BCAST_PREALLREDUCE", comm, "broadcast",
MPID_BCAST_PREALLREDUCE);
MPIDI_Check_preallreduce("PAMID_COLLECTIVE_SCATTERV_PREALLREDUCE", comm, "scatterv",
MPID_SCATTERV_PREALLREDUCE);
{
TRACE_ERR("Checking bcast\n");
char* names[] = {"PAMID_COLLECTIVE_BCAST", "MP_S_MPI_BCAST", NULL};
MPIDI_Check_protocols(names, comm, "broadcast", PAMI_XFER_BROADCAST);
}
{
TRACE_ERR("Checking allreduce\n");
char* names[] = {"PAMID_COLLECTIVE_ALLREDUCE", "MP_S_MPI_ALLREDUCE", NULL};
MPIDI_Check_protocols(names, comm, "allreduce", PAMI_XFER_ALLREDUCE);
}
{
TRACE_ERR("Checking barrier\n");
char* names[] = {"PAMID_COLLECTIVE_BARRIER", "MP_S_MPI_BARRIER", NULL};
MPIDI_Check_protocols(names, comm, "barrier", PAMI_XFER_BARRIER);
}
{
TRACE_ERR("Checking alltaoll\n");
char* names[] = {"PAMID_COLLECTIVE_ALLTOALL", NULL};
MPIDI_Check_protocols(names, comm, "alltoall", PAMI_XFER_ALLTOALL);
}
{
TRACE_ERR("Checking reduce\n");
char* names[] = {"PAMID_COLLECTIVE_REDUCE", "MP_S_MPI_REDUCE", NULL};
MPIDI_Check_protocols(names, comm, "reduce", PAMI_XFER_REDUCE);
}
{
TRACE_ERR("Checking alltoallv\n");
char* names[] = {"PAMID_COLLECTIVE_ALLTOALLV", NULL};
MPIDI_Check_protocols(names, comm, "alltoallv", PAMI_XFER_ALLTOALLV_INT);
}
{
TRACE_ERR("Checking gatherv\n");
char* names[] = {"PAMID_COLLECTIVE_GATHERV", NULL};
MPIDI_Check_protocols(names, comm, "gatherv", PAMI_XFER_GATHERV_INT);
}
{
TRACE_ERR("Checking scan\n");
char* names[] = {"PAMID_COLLECTIVE_SCAN", NULL};
MPIDI_Check_protocols(names, comm, "scan", PAMI_XFER_SCAN);
}
comm->mpid.scattervs[0] = comm->mpid.scattervs[1] = 0;
TRACE_ERR("Checking scatterv\n");
envopts = getenv("PAMID_COLLECTIVE_SCATTERV");
if(envopts != NULL)
{
if(strcasecmp(envopts, "GLUE_BCAST") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue bcast for scatterv\n");
comm->mpid.scattervs[0] = 1;
}
else if(strcasecmp(envopts, "GLUE_ALLTOALLV") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue alltoallv for scatterv\n");
comm->mpid.scattervs[1] = 1;
}
}
{ /* In addition to glue protocols, check for other PAMI protocols and check for PE now */
char* names[] = {"PAMID_COLLECTIVE_SCATTERV", NULL};
MPIDI_Check_protocols(names, comm, "scatterv", PAMI_XFER_SCATTERV_INT);
/* Use MPICH on large communicators (Issue 7516 and ticket 595)*/
if((comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] ==
MPID_COLL_NOSELECTION) /* no env var selected */
&& (comm->local_size > (16*1024))) /* and > 16k ranks */
{
comm->mpid.user_selected_type[PAMI_XFER_SCATTERV_INT] = MPID_COLL_USE_MPICH;
comm->mpid.user_selected[PAMI_XFER_SCATTERV_INT] = 0;
}
}
TRACE_ERR("Checking scatter\n");
comm->mpid.optscatter = 0;
envopts = getenv("PAMID_COLLECTIVE_SCATTER");
if(envopts != NULL)
{
if(strcasecmp(envopts, "GLUE_BCAST") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue_bcast for scatter\n");
comm->mpid.optscatter = 1;
}
}
{ /* In addition to glue protocols, check for other PAMI protocols and check for PE now */
char* names[] = {"PAMID_COLLECTIVE_SCATTER", NULL};
MPIDI_Check_protocols(names, comm, "scatter", PAMI_XFER_SCATTER);
}
TRACE_ERR("Checking allgather\n");
comm->mpid.allgathers[0] = comm->mpid.allgathers[1] = comm->mpid.allgathers[2] = 0;
envopts = getenv("PAMID_COLLECTIVE_ALLGATHER");
if(envopts != NULL)
{
if(strcasecmp(envopts, "GLUE_ALLREDUCE") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue_allreduce for allgather\n");
comm->mpid.allgathers[0] = 1;
}
else if(strcasecmp(envopts, "GLUE_BCAST") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue_bcast for allgather\n");
comm->mpid.allgathers[1] = 1;
}
else if(strcasecmp(envopts, "GLUE_ALLTOALL") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue_alltoall for allgather\n");
comm->mpid.allgathers[2] = 1;
}
}
{ /* In addition to glue protocols, check for other PAMI protocols and check for PE now */
char* names[] = {"PAMID_COLLECTIVE_ALLGATHER", "MP_S_MPI_ALLGATHER", NULL};
MPIDI_Check_protocols(names, comm, "allgather", PAMI_XFER_ALLGATHER);
}
TRACE_ERR("Checking allgatherv\n");
comm->mpid.allgathervs[0] = comm->mpid.allgathervs[1] = comm->mpid.allgathervs[2] = 0;
envopts = getenv("PAMID_COLLECTIVE_ALLGATHERV");
if(envopts != NULL)
{
if(strcasecmp(envopts, "GLUE_ALLREDUCE") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue_allreduce for allgatherv\n");
comm->mpid.allgathervs[0] = 1;
}
else if(strcasecmp(envopts, "GLUE_BCAST") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue_bcast for allgatherv\n");
comm->mpid.allgathervs[1] = 1;
}
else if(strcasecmp(envopts, "GLUE_ALLTOALL") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"Selecting glue_alltoall for allgatherv\n");
comm->mpid.allgathervs[2] = 1;
}
}
{ /* In addition to glue protocols, check for other PAMI protocols and check for PE now */
char* names[] = {"PAMID_COLLECTIVE_ALLGATHERV", "MP_S_MPI_ALLGATHERV", NULL};
MPIDI_Check_protocols(names, comm, "allgatherv", PAMI_XFER_ALLGATHERV_INT);
}
TRACE_ERR("CHecking gather\n");
comm->mpid.optgather = 0;
envopts = getenv("PAMID_COLLECTIVE_GATHER");
if(envopts != NULL)
{
if(strcasecmp(envopts, "GLUE_REDUCE") == 0)
{
if(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_0 && comm->rank == 0)
fprintf(stderr,"using glue_reduce for gather\n");
comm->mpid.optgather = 1;
}
}
{ /* In addition to glue protocols, check for other PAMI protocols and check for PE now */
char* names[] = {"PAMID_COLLECTIVE_GATHER", NULL};
MPIDI_Check_protocols(names, comm, "gather", PAMI_XFER_GATHER);
}
TRACE_ERR("MPIDI_Comm_coll_envvars exit\n");
}
int MPIDO_Barrier(MPID_Comm *comm_ptr, int *mpierrno)
{
TRACE_ERR("Entering MPIDO_Barrier\n");
volatile unsigned active=1;
MPIDI_Post_coll_t barrier_post;
pami_xfer_t barrier;
pami_algorithm_t my_barrier;
pami_metadata_t *my_barrier_md;
int queryreq = 0;
if(comm_ptr->mpid.user_selected_type[PAMI_XFER_BARRIER] == MPID_COLL_USE_MPICH)
{
if(unlikely(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL && comm_ptr->rank == 0))
fprintf(stderr,"Using MPICH barrier\n");
TRACE_ERR("Using MPICH Barrier\n");
return MPIR_Barrier(comm_ptr, mpierrno);
}
barrier.cb_done = cb_barrier;
barrier.cookie = (void *)&active;
if(comm_ptr->mpid.user_selected_type[PAMI_XFER_BARRIER] == MPID_COLL_OPTIMIZED)
{
TRACE_ERR("Optimized barrier (%s) was pre-selected\n", comm_ptr->mpid.opt_protocol_md[PAMI_XFER_BARRIER][0].name);
my_barrier = comm_ptr->mpid.opt_protocol[PAMI_XFER_BARRIER][0];
my_barrier_md = &comm_ptr->mpid.opt_protocol_md[PAMI_XFER_BARRIER][0];
queryreq = comm_ptr->mpid.must_query[PAMI_XFER_BARRIER][0];
}
else
{
TRACE_ERR("Barrier (%s) was specified by user\n", comm_ptr->mpid.user_metadata[PAMI_XFER_BARRIER].name);
my_barrier = comm_ptr->mpid.user_selected[PAMI_XFER_BARRIER];
my_barrier_md = &comm_ptr->mpid.user_metadata[PAMI_XFER_BARRIER];
queryreq = comm_ptr->mpid.user_selected_type[PAMI_XFER_BARRIER];
}
barrier.algorithm = my_barrier;
/* There is no support for query-required barrier protocols here */
MPID_assert_always(queryreq != MPID_COLL_ALWAYS_QUERY);
MPID_assert_always(queryreq != MPID_COLL_CHECK_FN_REQUIRED);
/* TODO Name needs fixed somehow */
MPIDI_Update_last_algorithm(comm_ptr, my_barrier_md->name);
if(unlikely(MPIDI_Process.verbose >= MPIDI_VERBOSE_DETAILS_ALL && comm_ptr->rank == 0))
{
unsigned long long int threadID;
MPIU_Thread_id_t tid;
MPIU_Thread_self(&tid);
threadID = (unsigned long long int)tid;
fprintf(stderr,"<%llx> Using protocol %s for barrier on %u\n",
threadID,
my_barrier_md->name,
/* comm_ptr->rank,comm_ptr->local_size,comm_ptr->remote_size,*/
(unsigned) comm_ptr->context_id);
}
TRACE_ERR("%s barrier\n", MPIDI_Process.context_post.active>0?"posting":"invoking");
MPIDI_Context_post(MPIDI_Context[0], &barrier_post.state,
MPIDI_Pami_post_wrapper, (void *)&barrier);
TRACE_ERR("barrier %s rc: %d\n", MPIDI_Process.context_post.active>0?"posted":"invoked", rc);
TRACE_ERR("advance spinning\n");
MPID_PROGRESS_WAIT_WHILE(active);
TRACE_ERR("exiting mpido_barrier\n");
return 0;
}
/**
* \brief Shut down the system
*
* At this time, no attempt is made to free memory being used for MPI structures.
* \return MPI_SUCCESS
*/
int MPID_Finalize()
{
pami_result_t rc;
int mpierrno = MPI_SUCCESS;
mpir_errflag_t errflag=MPIR_ERR_NONE;
MPIR_Barrier_impl(MPIR_Process.comm_world, &errflag);
#ifdef MPIDI_STATISTICS
if (MPIDI_Process.mp_statistics) {
MPIDI_print_statistics();
}
MPIDI_close_pe_extension();
#endif
#ifdef DYNAMIC_TASKING
mpidi_finalized = 1;
if(mpidi_dynamic_tasking) {
/* Tell the process group code that we're done with the process groups.
This will notify PMI (with PMI_Finalize) if necessary. It
also frees all PG structures, including the PG for COMM_WORLD, whose
pointer is also saved in MPIDI_Process.my_pg */
mpierrno = MPIDI_PG_Finalize();
if (mpierrno) {
TRACE_ERR("MPIDI_PG_Finalize returned with mpierrno=%d\n", mpierrno);
}
MPIDI_FreeParentPort();
}
if(_conn_info_list)
MPIU_Free(_conn_info_list);
MPIDI_free_all_tranid_node();
#endif
/* ------------------------- */
/* shutdown request queues */
/* ------------------------- */
MPIDI_Recvq_finalize();
PAMIX_Finalize(MPIDI_Client);
#ifdef MPID_NEEDS_ICOMM_WORLD
MPIR_Comm_release_always(MPIR_Process.icomm_world, 0);
#endif
MPIR_Comm_release_always(MPIR_Process.comm_self,0);
MPIR_Comm_release_always(MPIR_Process.comm_world,0);
rc = PAMI_Context_destroyv(MPIDI_Context, MPIDI_Process.avail_contexts);
MPID_assert_always(rc == PAMI_SUCCESS);
rc = PAMI_Client_destroy(&MPIDI_Client);
MPID_assert_always(rc == PAMI_SUCCESS);
#ifdef MPIDI_TRACE
{ int i;
for (i=0; i< MPIDI_Process.numTasks; i++) {
if (MPIDI_Trace_buf[i].R)
MPIU_Free(MPIDI_Trace_buf[i].R);
if (MPIDI_Trace_buf[i].PR)
MPIU_Free(MPIDI_Trace_buf[i].PR);
if (MPIDI_Trace_buf[i].S)
MPIU_Free(MPIDI_Trace_buf[i].S);
}
}
MPIU_Free(MPIDI_Trace_buf);
#endif
#ifdef OUT_OF_ORDER_HANDLING
MPIU_Free(MPIDI_In_cntr);
MPIU_Free(MPIDI_Out_cntr);
#endif
if (TOKEN_FLOW_CONTROL_ON)
{
#if TOKEN_FLOW_CONTROL
extern char *EagerLimit;
if (EagerLimit) MPIU_Free(EagerLimit);
MPIU_Free(MPIDI_Token_cntr);
MPIDI_close_mm();
#else
MPID_assert_always(0);
#endif
}
return MPI_SUCCESS;
}
/**
* \brief The callback for a new RZV RTS
* \note Because this is a short message, the data is already received
* \param[in] context The context on which the message is being received.
* \param[in] sender The origin endpoint
* \param[in] _msginfo The extended header information
* \param[in] msginfo_size The size of the extended header information
* \param[in] is_zero_byte The rendezvous message is zero bytes in length.
*/
void
MPIDI_RecvRzvCB_impl(pami_context_t context,
pami_endpoint_t sender,
const void * _msginfo,
size_t msginfo_size,
const unsigned is_zero_byte)
{
MPID_assert(_msginfo != NULL);
MPID_assert(msginfo_size == sizeof(MPIDI_MsgEnvelope));
const MPIDI_MsgEnvelope * envelope = (const MPIDI_MsgEnvelope *)_msginfo;
const MPIDI_MsgInfo * msginfo = (const MPIDI_MsgInfo *)&envelope->msginfo;
MPID_Request * rreq = NULL;
int found;
pami_task_t source;
#if TOKEN_FLOW_CONTROL
int rettoks=0;
#endif
/* -------------------- */
/* Match the request. */
/* -------------------- */
unsigned rank = msginfo->MPIrank;
unsigned tag = msginfo->MPItag;
unsigned context_id = msginfo->MPIctxt;
MPID_Request *newreq = MPIDI_Request_create2();
MPIU_THREAD_CS_ENTER(MSGQUEUE,0);
source = PAMIX_Endpoint_query(sender);
MPIDI_Receive_tokens(msginfo,source);
#ifndef OUT_OF_ORDER_HANDLING
rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, tag, context_id, &found);
#else
rreq = MPIDI_Recvq_FDP_or_AEU(newreq, rank, source, tag, context_id, msginfo->MPIseqno, &found);
#endif
TRACE_ERR("RZV CB for req=%p remote-mr=0x%llx bytes=%zu (%sfound)\n",
rreq,
*(unsigned long long*)&envelope->envelope.memregion,
envelope->envelope.length,
found?"":"not ");
/* ---------------------- */
/* Copy in information. */
/* ---------------------- */
rreq->status.MPI_SOURCE = rank;
rreq->status.MPI_TAG = tag;
MPIR_STATUS_SET_COUNT(rreq->status, envelope->length);
MPIDI_Request_setPeerRank_comm(rreq, rank);
MPIDI_Request_setPeerRank_pami(rreq, source);
MPIDI_Request_cpyPeerRequestH (rreq, msginfo);
MPIDI_Request_setSync (rreq, msginfo->isSync);
MPIDI_Request_setRzv (rreq, 1);
/* ----------------------------------------------------- */
/* Save the rendezvous information for when the target */
/* node calls a receive function and the data is */
/* retreived from the origin node. */
/* ----------------------------------------------------- */
if (is_zero_byte)
{
rreq->mpid.envelope.length = 0;
rreq->mpid.envelope.data = NULL;
}
else
{
#ifdef USE_PAMI_RDMA
memcpy(&rreq->mpid.envelope.memregion,
&envelope->memregion,
sizeof(pami_memregion_t));
#else
rreq->mpid.envelope.memregion_used = envelope->memregion_used;
if(envelope->memregion_used)
{
memcpy(&rreq->mpid.envelope.memregion,
&envelope->memregion,
sizeof(pami_memregion_t));
}
rreq->mpid.envelope.data = envelope->data;
#endif
rreq->mpid.envelope.length = envelope->length;
TRACE_SET_R_VAL(source,(rreq->mpid.idx),req,rreq);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),rlen,envelope->length);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),fl.f.sync,msginfo->isSync);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.rzv);
if (TOKEN_FLOW_CONTROL_ON)
{
#if TOKEN_FLOW_CONTROL
MPIDI_Must_return_tokens(context,source);
#else
MPID_assert_always(0);
#endif
}
}
/* ----------------------------------------- */
/* figure out target buffer for request data */
/* ----------------------------------------- */
if (found)
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivalsMatched);
#endif
/* --------------------------- */
/* if synchronized, post ack. */
/* --------------------------- */
if (unlikely(MPIDI_Request_isSync(rreq)))
MPIDI_SyncAck_post(context, rreq, MPIDI_Request_getPeerRank_pami(rreq));
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
if (is_zero_byte)
MPIDI_RecvRzvDoneCB_zerobyte(context, rreq, PAMI_SUCCESS);
else
{
MPIDI_RendezvousTransfer(context, rreq);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.sync_com_in_HH);
TRACE_SET_R_BIT(source,(rreq->mpid.idx),fl.f.matchedInHH);
TRACE_SET_R_VAL(source,(rreq->mpid.idx),bufadd,rreq->mpid.userbuf);
}
MPID_Request_discard(newreq);
}
/* ------------------------------------------------------------- */
/* Request was not posted. */
/* ------------------------------------------------------------- */
else
{
#if (MPIDI_STATISTICS)
MPID_NSTAT(mpid_statp->earlyArrivals);
#endif
/*
* This is to test that the fields don't need to be
* initialized. Remove after this doesn't fail for a while.
*/
MPID_assert(rreq->mpid.uebuf == NULL);
MPID_assert(rreq->mpid.uebuflen == 0);
/* rreq->mpid.uebuf = NULL; */
/* rreq->mpid.uebuflen = 0; */
#ifdef OUT_OF_ORDER_HANDLING
if (MPIDI_In_cntr[source].n_OutOfOrderMsgs > 0) {
MPIDI_Recvq_process_out_of_order_msgs(source, context);
}
#endif
MPIU_THREAD_CS_EXIT(MSGQUEUE,0);
}
/* ---------------------------------------- */
/* Signal that the recv has been started. */
/* ---------------------------------------- */
MPIDI_Progress_signal();
}
