int MPID_nem_tcp_get_business_card (int my_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
int str_errno = MPIU_STR_SUCCESS;
MPIDU_Sock_ifaddr_t ifaddr;
char ifname[MAX_HOST_DESCRIPTION_LEN];
int ret;
struct sockaddr_in sock_id;
socklen_t len;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_GET_BUSINESS_CARD);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_GET_BUSINESS_CARD);
mpi_errno = GetSockInterfaceAddr(my_rank, ifname, sizeof(ifname), &ifaddr);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
str_errno = MPIU_Str_add_string_arg(bc_val_p, val_max_sz_p, MPIDI_CH3I_HOST_DESCRIPTION_KEY, ifname);
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
len = sizeof(sock_id);
ret = getsockname (MPID_nem_tcp_g_lstn_sc.fd, (struct sockaddr *)&sock_id, &len);
MPIR_ERR_CHKANDJUMP1 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**getsockname", "**getsockname %s", MPIU_Strerror (errno));
str_errno = MPIU_Str_add_int_arg (bc_val_p, val_max_sz_p, MPIDI_CH3I_PORT_KEY, ntohs(sock_id.sin_port));
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
if (ifaddr.len > 0 && ifaddr.type == AF_INET)
{
unsigned char *p;
p = (unsigned char *)(ifaddr.ifaddr);
MPL_snprintf( ifname, sizeof(ifname), "%u.%u.%u.%u", p[0], p[1], p[2], p[3] );
MPIU_DBG_MSG_S(CH3_CONNECT,VERBOSE,"ifname = %s",ifname );
str_errno = MPIU_Str_add_string_arg(bc_val_p, val_max_sz_p, MPIDI_CH3I_IFNAME_KEY, ifname);
if (str_errno) {
MPIR_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno, MPI_ERR_OTHER, "**buscard_len");
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
}
/* printf("MPID_nem_tcp_get_business_card. port=%d\n", sock_id.sin_port); */
fn_exit:
/* fprintf(stdout, "MPID_nem_tcp_get_business_card Exit, mpi_errno=%d\n", mpi_errno); fflush(stdout); */
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_GET_BUSINESS_CARD);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIC_Wait(MPID_Request * request_ptr, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIC_WAIT);
MPIDI_PT2PT_FUNC_ENTER(MPID_STATE_MPIC_WAIT);
MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");
if (request_ptr->kind == MPID_REQUEST_SEND)
request_ptr->status.MPI_TAG = 0;
if (!MPID_Request_is_complete(request_ptr))
{
MPID_Progress_state progress_state;
MPID_Progress_start(&progress_state);
while (!MPID_Request_is_complete(request_ptr))
{
mpi_errno = MPID_Progress_wait(&progress_state);
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
MPID_Progress_end(&progress_state);
}
if (request_ptr->kind == MPID_REQUEST_RECV)
MPIR_Process_status(&request_ptr->status, errflag);
MPIR_TAG_CLEAR_ERROR_BITS(request_ptr->status.MPI_TAG);
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_PT2PT_FUNC_EXIT(MPID_STATE_MPIC_WAIT);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIC_Sendrecv(const void *sendbuf, MPI_Aint sendcount, MPI_Datatype sendtype,
int dest, int sendtag, void *recvbuf, MPI_Aint recvcount,
MPI_Datatype recvtype, int source, int recvtag,
MPID_Comm *comm_ptr, MPI_Status *status, mpir_errflag_t *errflag)
{
int mpi_errno = MPI_SUCCESS;
int context_id;
MPI_Status mystatus;
MPID_Request *recv_req_ptr = NULL, *send_req_ptr = NULL;
MPIDI_STATE_DECL(MPID_STATE_MPIC_SENDRECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPIC_SENDRECV);
MPIU_DBG_MSG_S(PT2PT, TYPICAL, "IN: errflag = %s", *errflag?"TRUE":"FALSE");
MPIU_ERR_CHKANDJUMP1((sendcount < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", sendcount);
MPIU_ERR_CHKANDJUMP1((recvcount < 0), mpi_errno, MPI_ERR_COUNT,
"**countneg", "**countneg %d", recvcount);
context_id = (comm_ptr->comm_kind == MPID_INTRACOMM) ?
MPID_CONTEXT_INTRA_COLL : MPID_CONTEXT_INTER_COLL;
if (status == MPI_STATUS_IGNORE) status = &mystatus;
switch(*errflag) {
case MPIR_ERR_NONE:
break;
case MPIR_ERR_PROC_FAILED:
MPIR_TAG_SET_PROC_FAILURE_BIT(sendtag);
default:
MPIR_TAG_SET_ERROR_BIT(sendtag);
}
mpi_errno = MPID_Irecv(recvbuf, recvcount, recvtype, source, recvtag,
comm_ptr, context_id, &recv_req_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPID_Isend(sendbuf, sendcount, sendtype, dest, sendtag,
comm_ptr, context_id, &send_req_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIC_Wait(send_req_ptr, errflag);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = MPIC_Wait(recv_req_ptr, errflag);
if (mpi_errno) MPIU_ERR_POPFATAL(mpi_errno);
*status = recv_req_ptr->status;
if (mpi_errno == MPI_SUCCESS) {
mpi_errno = recv_req_ptr->status.MPI_ERROR;
if (mpi_errno == MPI_SUCCESS) {
mpi_errno = send_req_ptr->status.MPI_ERROR;
}
}
MPID_Request_release(send_req_ptr);
MPID_Request_release(recv_req_ptr);
fn_exit:
MPIU_DBG_MSG_D(PT2PT, TYPICAL, "OUT: errflag = %d", *errflag);
MPIDI_FUNC_EXIT(MPID_STATE_MPIC_SENDRECV);
return mpi_errno;
fn_fail:
if (send_req_ptr)
MPID_Request_release(send_req_ptr);
if (recv_req_ptr)
MPID_Request_release(recv_req_ptr);
goto fn_exit;
}
int MPIR_Get_contextid_sparse_group(MPID_Comm * comm_ptr, MPID_Group * group_ptr, int tag,
MPIU_Context_id_t * context_id, int ignore_id)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
struct gcn_state st;
struct gcn_state *tmp;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_GET_CONTEXTID);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_GET_CONTEXTID);
st.first_iter = 1;
st.comm_ptr = comm_ptr;
st.tag = tag;
st.own_mask = 0;
st.own_eager_mask = 0;
/* Group-collective and ignore_id should never be combined */
MPIU_Assert(!(group_ptr != NULL && ignore_id));
*context_id = 0;
MPIU_DBG_MSG_FMT(COMM, VERBOSE, (MPIU_DBG_FDEST,
"Entering; shared state is %d:%d, my ctx id is %d, tag=%d",
mask_in_use, eager_in_use, comm_ptr->context_id, tag));
while (*context_id == 0) {
/* We lock only around access to the mask (except in the global locking
* case). If another thread is using the mask, we take a mask of zero. */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
if (initialize_context_mask) {
context_id_init();
}
if (eager_nelem < 0) {
/* Ensure that at least one word of deadlock-free context IDs is
* always set aside for the base protocol */
MPIU_Assert(MPIR_CVAR_CTXID_EAGER_SIZE >= 0 &&
MPIR_CVAR_CTXID_EAGER_SIZE < MPIR_MAX_CONTEXT_MASK - 1);
eager_nelem = MPIR_CVAR_CTXID_EAGER_SIZE;
}
if (ignore_id) {
/* We are not participating in the resulting communicator, so our
* context ID space doesn't matter. Set the mask to "all available". */
memset(st.local_mask, 0xff, MPIR_MAX_CONTEXT_MASK * sizeof(int));
st.own_mask = 0;
/* don't need to touch mask_in_use/lowest_context_id b/c our thread
* doesn't ever need to "win" the mask */
}
/* Deadlock avoidance: Only participate in context id loop when all
* processes have called this routine. On the first iteration, use the
* "eager" allocation protocol.
*/
else if (st.first_iter) {
memset(st.local_mask, 0, MPIR_MAX_CONTEXT_MASK * sizeof(int));
st.own_eager_mask = 0;
/* Attempt to reserve the eager mask segment */
if (!eager_in_use && eager_nelem > 0) {
int i;
for (i = 0; i < eager_nelem; i++)
st.local_mask[i] = context_mask[i];
eager_in_use = 1;
st.own_eager_mask = 1;
}
}
else {
MPIU_Assert(next_gcn != NULL);
/*If we are here, at least one element must be in the list, at least myself */
/* only the first element in the list can own the mask. However, maybe the mask is used
* by another thread, which added another allcoation to the list bevore. So we have to check,
* if the mask is used and mark, if we own it */
if (mask_in_use || &st != next_gcn) {
memset(st.local_mask, 0, MPIR_MAX_CONTEXT_MASK * sizeof(int));
st.own_mask = 0;
MPIU_DBG_MSG_FMT(COMM, VERBOSE, (MPIU_DBG_FDEST,
"Mask is in use, my context_id is %d, owner context id is %d",
st.comm_ptr->context_id,
next_gcn->comm_ptr->context_id));
}
else {
int i;
/* Copy safe mask segment to local_mask */
for (i = 0; i < eager_nelem; i++)
st.local_mask[i] = 0;
for (i = eager_nelem; i < MPIR_MAX_CONTEXT_MASK; i++)
st.local_mask[i] = context_mask[i];
mask_in_use = 1;
st.own_mask = 1;
MPIU_DBG_MSG(COMM, VERBOSE, "Copied local_mask");
}
}
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
/* Note: MPIR_MAX_CONTEXT_MASK elements of local_mask are used by the
* context ID allocation algorithm. The additional element is ignored
* by the context ID mask access routines and is used as a flag for
* detecting context ID exhaustion (explained below). */
if (st.own_mask || ignore_id)
st.local_mask[ALL_OWN_MASK_FLAG] = 1;
else
st.local_mask[ALL_OWN_MASK_FLAG] = 0;
/* Now, try to get a context id */
MPIU_Assert(comm_ptr->comm_kind == MPID_INTRACOMM);
/* In the global and brief-global cases, note that this routine will
* release that global lock when it needs to wait. That will allow
* other processes to enter the global or brief global critical section.
*/
if (group_ptr != NULL) {
int coll_tag = tag | MPIR_Process.tagged_coll_mask; /* Shift tag into the tagged coll space */
mpi_errno = MPIR_Allreduce_group(MPI_IN_PLACE, st.local_mask, MPIR_MAX_CONTEXT_MASK + 1,
MPI_INT, MPI_BAND, comm_ptr, group_ptr, coll_tag,
&errflag);
}
else {
mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, st.local_mask, MPIR_MAX_CONTEXT_MASK + 1,
MPI_INT, MPI_BAND, comm_ptr, &errflag);
}
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
/* MT FIXME 2/3 cases don't seem to need the CONTEXTID CS, check and
* narrow this region */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
if (ignore_id) {
/* we don't care what the value was, but make sure that everyone
* who did care agreed on a value */
*context_id = locate_context_bit(st.local_mask);
/* used later in out-of-context ids check and outer while loop condition */
}
else if (st.own_eager_mask) {
/* There is a chance that we've found a context id */
/* Find_and_allocate_context_id updates the context_mask if it finds a match */
*context_id = find_and_allocate_context_id(st.local_mask);
MPIU_DBG_MSG_D(COMM, VERBOSE, "Context id is now %hd", *context_id);
st.own_eager_mask = 0;
eager_in_use = 0;
if (*context_id <= 0) {
/* else we did not find a context id. Give up the mask in case
* there is another thread (with a lower input context id)
* waiting for it. We need to ensure that any other threads
* have the opportunity to run, hence yielding */
/* FIXME: Do we need to do an GLOBAL yield here?
* When we do a collective operation, we anyway yield
* for other others */
MPID_THREAD_CS_YIELD(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX); /* FG: TODO Double-check */
MPID_THREAD_CS_YIELD(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
}
else if (st.own_mask) {
/* There is a chance that we've found a context id */
/* Find_and_allocate_context_id updates the context_mask if it finds a match */
*context_id = find_and_allocate_context_id(st.local_mask);
MPIU_DBG_MSG_D(COMM, VERBOSE, "Context id is now %hd", *context_id);
mask_in_use = 0;
if (*context_id > 0) {
/* If we found a new context id, we have to remove the element from the list, so the
* next allocation can own the mask */
if (next_gcn == &st) {
next_gcn = st.next;
}
else {
for (tmp = next_gcn; tmp->next != &st; tmp = tmp->next); /* avoid compiler warnings */
tmp->next = st.next;
}
}
else {
/* else we did not find a context id. Give up the mask in case
* there is another thread in the gcn_next_list
* waiting for it. We need to ensure that any other threads
* have the opportunity to run, hence yielding */
/* FIXME: Do we need to do an GLOBAL yield here?
* When we do a collective operation, we anyway yield
* for other others */
MPID_THREAD_CS_YIELD(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPID_THREAD_CS_YIELD(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
}
else {
/* As above, force this thread to yield */
/* FIXME: Do we need to do an GLOBAL yield here? When we
* do a collective operation, we anyway yield for other
* others */
MPID_THREAD_CS_YIELD(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
MPID_THREAD_CS_YIELD(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
/* Test for context ID exhaustion: All threads that will participate in
* the new communicator owned the mask and could not allocate a context
* ID. This indicates that either some process has no context IDs
* available, or that some are available, but the allocation cannot
* succeed because there is no common context ID. */
if (*context_id == 0 && st.local_mask[ALL_OWN_MASK_FLAG] == 1) {
/* --BEGIN ERROR HANDLING-- */
int nfree = 0;
int ntotal = 0;
int minfree;
if (st.own_mask) {
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
mask_in_use = 0;
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
}
context_mask_stats(&nfree, &ntotal);
if (ignore_id)
minfree = INT_MAX;
else
minfree = nfree;
if (group_ptr != NULL) {
int coll_tag = tag | MPIR_Process.tagged_coll_mask; /* Shift tag into the tagged coll space */
mpi_errno = MPIR_Allreduce_group(MPI_IN_PLACE, &minfree, 1, MPI_INT, MPI_MIN,
comm_ptr, group_ptr, coll_tag, &errflag);
}
else {
mpi_errno = MPIR_Allreduce_impl(MPI_IN_PLACE, &minfree, 1, MPI_INT,
MPI_MIN, comm_ptr, &errflag);
}
if (minfree > 0) {
MPIR_ERR_SETANDJUMP3(mpi_errno, MPI_ERR_OTHER,
"**toomanycommfrag", "**toomanycommfrag %d %d %d",
nfree, ntotal, ignore_id);
}
else {
MPIR_ERR_SETANDJUMP3(mpi_errno, MPI_ERR_OTHER,
"**toomanycomm", "**toomanycomm %d %d %d",
nfree, ntotal, ignore_id);
}
/* --END ERROR HANDLING-- */
}
if (st.first_iter == 1) {
st.first_iter = 0;
/* to avoid deadlocks, the element is not added to the list bevore the first iteration */
if (!ignore_id && *context_id == 0)
add_gcn_to_list(&st);
}
}
fn_exit:
if (ignore_id)
*context_id = MPIU_INVALID_CONTEXT_ID;
MPIU_DBG_MSG_S(COMM, VERBOSE, "Context mask = %s", context_mask_to_str());
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_GET_CONTEXTID);
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
/* Release the masks */
if (st.own_mask) {
/* is it safe to access this without holding the CS? */
mask_in_use = 0;
}
/*If in list, remove it */
if (!st.first_iter && !ignore_id) {
if (next_gcn == &st) {
next_gcn = st.next;
}
else {
for (tmp = next_gcn; tmp->next != &st; tmp = tmp->next);
tmp->next = st.next;
}
}
goto fn_exit;
/* --END ERROR HANDLING-- */
}
void MPIR_Free_contextid(MPIU_Context_id_t context_id)
{
int idx, bitpos, raw_prefix;
MPID_MPI_STATE_DECL(MPID_STATE_MPIR_FREE_CONTEXTID);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPIR_FREE_CONTEXTID);
/* Convert the context id to the bit position */
raw_prefix = MPID_CONTEXT_READ_FIELD(PREFIX, context_id);
idx = raw_prefix / MPIR_CONTEXT_INT_BITS;
bitpos = raw_prefix % MPIR_CONTEXT_INT_BITS;
/* --BEGIN ERROR HANDLING-- */
if (idx < 0 || idx >= MPIR_MAX_CONTEXT_MASK) {
MPID_Abort(0, MPI_ERR_INTERN, 1, "In MPIR_Free_contextid, idx is out of range");
}
/* --END ERROR HANDLING-- */
/* The low order bits for dynamic context IDs don't have meaning the
* same way that low bits of non-dynamic ctx IDs do. So we have to
* check the dynamic case first. */
if (MPID_CONTEXT_READ_FIELD(DYNAMIC_PROC, context_id)) {
MPIU_DBG_MSG_D(COMM, VERBOSE, "skipping dynamic process ctx id, context_id=%d", context_id);
goto fn_exit;
}
else { /* non-dynamic context ID */
/* In terms of the context ID bit vector, intercomms and their constituent
* localcomms have the same value. To avoid a double-free situation we just
* don't free the context ID for localcomms and assume it will be cleaned up
* when the parent intercomm is itself completely freed. */
if (MPID_CONTEXT_READ_FIELD(IS_LOCALCOMM, context_id)) {
#ifdef USE_DBG_LOGGING
char dump_str[1024];
dump_context_id(context_id, dump_str, sizeof(dump_str));
MPIU_DBG_MSG_S(COMM, VERBOSE, "skipping localcomm id: %s", dump_str);
#endif
goto fn_exit;
}
else if (MPID_CONTEXT_READ_FIELD(SUBCOMM, context_id)) {
MPIU_DBG_MSG_D(COMM, VERBOSE, "skipping non-parent communicator ctx id, context_id=%d",
context_id);
goto fn_exit;
}
}
/* --BEGIN ERROR HANDLING-- */
/* Check that this context id has been allocated */
if ((context_mask[idx] & (0x1 << bitpos)) != 0) {
#ifdef USE_DBG_LOGGING
char dump_str[1024];
dump_context_id(context_id, dump_str, sizeof(dump_str));
MPIU_DBG_MSG_S(COMM, VERBOSE, "context dump: %s", dump_str);
MPIU_DBG_MSG_S(COMM, VERBOSE, "context mask = %s", context_mask_to_str());
#endif
MPID_Abort(0, MPI_ERR_INTERN, 1, "In MPIR_Free_contextid, the context id is not in use");
}
/* --END ERROR HANDLING-- */
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
/* MT: Note that this update must be done atomically in the multithreaedd
* case. In the "one, single lock" implementation, that lock is indeed
* held when this operation is called. */
context_mask[idx] |= (0x1 << bitpos);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_CTX_MUTEX);
MPIU_DBG_MSG_FMT(COMM, VERBOSE,
(MPIU_DBG_FDEST,
"Freed context %d, mask[%d] bit %d (prefix=%#x)",
context_id, idx, bitpos, raw_prefix));
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPIR_FREE_CONTEXTID);
}
int MPIDI_CH3U_Check_for_failed_procs(void)
{
#if defined(FINEGRAIN_MPI)
return (MPI_SUCCESS); /* FG: Temporary bypass. Uses MPI_Group */
#endif
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
int len;
char *kvsname;
MPID_Group *prev_failed_group, *new_failed_group;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_CHECK_FOR_FAILED_PROCS);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_CHECK_FOR_FAILED_PROCS);
/* FIXME: Currently this only handles failed processes in
comm_world. We need to fix hydra to include the pgid along
with the rank, then we need to create the failed group from
something bigger than comm_world. */
mpi_errno = MPIDI_PG_GetConnKVSname(&kvsname);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
#ifdef USE_PMI2_API
{
int vallen = 0;
pmi_errno = PMI2_KVS_Get(kvsname, PMI2_ID_NULL, "PMI_dead_processes", MPIDI_failed_procs_string, PMI2_MAX_VALLEN, &vallen);
MPIR_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get");
}
#else
pmi_errno = PMI_KVS_Get_value_length_max(&len);
MPIR_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get_value_length_max");
pmi_errno = PMI_KVS_Get(kvsname, "PMI_dead_processes", MPIDI_failed_procs_string, len);
MPIR_ERR_CHKANDJUMP(pmi_errno, mpi_errno, MPI_ERR_OTHER, "**pmi_kvs_get");
#endif
if (*MPIDI_failed_procs_string == '\0') {
/* there are no failed processes */
MPIDI_Failed_procs_group = MPID_Group_empty;
goto fn_exit;
}
MPIU_DBG_MSG_S(CH3_OTHER, TYPICAL, "Received proc fail notification: %s", MPIDI_failed_procs_string);
/* save reference to previous group so we can identify new failures */
prev_failed_group = MPIDI_Failed_procs_group;
/* Parse the list of failed processes */
MPIDI_CH3U_Get_failed_group(-2, &MPIDI_Failed_procs_group);
/* get group of newly failed processes */
mpi_errno = MPIR_Group_difference_impl(MPIDI_Failed_procs_group, prev_failed_group, &new_failed_group);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (new_failed_group != MPID_Group_empty) {
mpi_errno = MPIDI_CH3I_Comm_handle_failed_procs(new_failed_group);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = terminate_failed_VCs(new_failed_group);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Group_release(new_failed_group);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
/* free prev group */
if (prev_failed_group != MPID_Group_empty) {
mpi_errno = MPIR_Group_release(prev_failed_group);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_CHECK_FOR_FAILED_PROCS);
return mpi_errno;
fn_oom: /* out-of-memory handler for utarray operations */
MPIR_ERR_SET1(mpi_errno, MPI_ERR_OTHER, "**nomem", "**nomem %s", "utarray");
fn_fail:
goto fn_exit;
}
int MPIR_Group_translate_ranks_impl(MPID_Group *gp1, int n, const int ranks1[],
MPID_Group *gp2, int ranks2[])
{
int mpi_errno = MPI_SUCCESS;
int i, g2_idx, l1_pid, l2_pid;
MPIU_DBG_MSG_S(OTHER,VERBOSE,"gp2->is_local_dense_monotonic=%s", (gp2->is_local_dense_monotonic ? "TRUE" : "FALSE"));
/* Initialize the output ranks */
for (i=0; i<n; i++)
ranks2[i] = MPI_UNDEFINED;
if (gp2->size > 0 && gp2->is_local_dense_monotonic) {
/* g2 probably == group_of(MPI_COMM_WORLD); use fast, constant-time lookup */
int lpid_offset = gp2->lrank_to_lpid[0].lpid;
MPIU_Assert(lpid_offset >= 0);
for (i = 0; i < n; ++i) {
int g1_lpid;
if (ranks1[i] == MPI_PROC_NULL) {
ranks2[i] = MPI_PROC_NULL;
continue;
}
/* "adjusted" lpid from g1 */
g1_lpid = gp1->lrank_to_lpid[ranks1[i]].lpid - lpid_offset;
if ((g1_lpid >= 0) && (g1_lpid < gp2->size)) {
ranks2[i] = g1_lpid;
}
/* else leave UNDEFINED */
}
}
else {
/* general, slow path; lookup time is dependent on the user-provided rank values! */
g2_idx = gp2->idx_of_first_lpid;
if (g2_idx < 0) {
MPIR_Group_setup_lpid_list( gp2 );
g2_idx = gp2->idx_of_first_lpid;
}
if (g2_idx >= 0) {
/* g2_idx can be < 0 if the g2 group is empty */
l2_pid = gp2->lrank_to_lpid[g2_idx].lpid;
for (i=0; i<n; i++) {
if (ranks1[i] == MPI_PROC_NULL) {
ranks2[i] = MPI_PROC_NULL;
continue;
}
l1_pid = gp1->lrank_to_lpid[ranks1[i]].lpid;
/* Search for this l1_pid in group2. Use the following
optimization: start from the last position in the lpid list
if possible. A more sophisticated version could use a
tree based or even hashed search to speed the translation. */
if (l1_pid < l2_pid || g2_idx < 0) {
/* Start over from the beginning */
g2_idx = gp2->idx_of_first_lpid;
l2_pid = gp2->lrank_to_lpid[g2_idx].lpid;
}
while (g2_idx >= 0 && l1_pid > l2_pid) {
g2_idx = gp2->lrank_to_lpid[g2_idx].next_lpid;
if (g2_idx >= 0)
l2_pid = gp2->lrank_to_lpid[g2_idx].lpid;
else
l2_pid = -1;
}
if (l1_pid == l2_pid)
ranks2[i] = gp2->lrank_to_lpid[g2_idx].lrank;
}
}
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
