int MPIDI_CH3U_Post_data_receive_unexpected(MPIR_Request * rreq)
{
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_POST_DATA_RECEIVE_UNEXPECTED);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_POST_DATA_RECEIVE_UNEXPECTED);
/* FIXME: to improve performance, allocate temporary buffer from a
specialized buffer pool. */
/* FIXME: to avoid memory exhaustion, integrate buffer pool management
with flow control */
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"unexpected request allocated");
rreq->dev.tmpbuf = MPL_malloc(rreq->dev.recv_data_sz, MPL_MEM_BUFFER);
if (!rreq->dev.tmpbuf) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d",
rreq->dev.recv_data_sz);
}
rreq->dev.tmpbuf_sz = rreq->dev.recv_data_sz;
rreq->dev.iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)rreq->dev.tmpbuf;
rreq->dev.iov[0].MPL_IOV_LEN = rreq->dev.recv_data_sz;
rreq->dev.iov_count = 1;
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackUEBufComplete;
rreq->dev.recv_pending_count = 2;
fn_fail:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_POST_DATA_RECEIVE_UNEXPECTED);
return mpi_errno;
}
int MPIDI_CH3_PktHandler_RndvSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp )
{
MPIDI_CH3_Pkt_rndv_send_t * rs_pkt = &pkt->rndv_send;
int mpi_errno = MPI_SUCCESS;
int complete;
char *data_buf;
MPIDI_msg_sz_t data_len;
MPID_Request *req;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"received rndv send (data) pkt");
MPID_Request_get_ptr(rs_pkt->receiver_req_id, req);
data_len = ((*buflen - sizeof(MPIDI_CH3_Pkt_t) >= req->dev.recv_data_sz)
? req->dev.recv_data_sz : *buflen - sizeof(MPIDI_CH3_Pkt_t));
data_buf = (char *)pkt + sizeof(MPIDI_CH3_Pkt_t);
if (req->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else {
mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len,
&complete);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**ch3|postrecv",
"**ch3|postrecv %s", "MPIDI_CH3_PKT_RNDV_SEND");
}
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
if (complete)
{
mpi_errno = MPID_Request_complete(req);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else
{
*rreqp = req;
}
}
fn_fail:
return mpi_errno;
}
int MPIDI_CH3U_Receive_data_unexpected(MPIR_Request * rreq, void *buf, intptr_t *buflen, int *complete)
{
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
/* FIXME: to improve performance, allocate temporary buffer from a
specialized buffer pool. */
/* FIXME: to avoid memory exhaustion, integrate buffer pool management
with flow control */
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"unexpected request allocated");
rreq->dev.tmpbuf = MPL_malloc(rreq->dev.recv_data_sz, MPL_MEM_BUFFER);
if (!rreq->dev.tmpbuf) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d",
rreq->dev.recv_data_sz);
}
rreq->dev.tmpbuf_sz = rreq->dev.recv_data_sz;
/* if all of the data has already been received, copy it
now, otherwise build an iov and let the channel copy it */
if (rreq->dev.recv_data_sz <= *buflen)
{
MPIR_Memcpy(rreq->dev.tmpbuf, buf, rreq->dev.recv_data_sz);
*buflen = rreq->dev.recv_data_sz;
rreq->dev.recv_pending_count = 1;
*complete = TRUE;
}
else
{
rreq->dev.iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)((char *)rreq->dev.tmpbuf);
rreq->dev.iov[0].MPL_IOV_LEN = rreq->dev.recv_data_sz;
rreq->dev.iov_count = 1;
rreq->dev.recv_pending_count = 2;
*buflen = 0;
*complete = FALSE;
}
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_PVAR_LEVEL_INC(RECVQ, unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackUEBufComplete;
fn_fail:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
return mpi_errno;
}
int MPIR_T_cvar_read_impl(MPI_T_cvar_handle handle, void *buf)
{
int mpi_errno = MPI_SUCCESS;
int i, count;
void *addr;
MPIR_T_cvar_handle_t *hnd = handle;
count = hnd->count;
addr = hnd->addr;
MPIR_Assert(addr != NULL);
switch (hnd->datatype) {
case MPI_INT:
for (i = 0; i < count; i++)
((int *)buf)[i] = ((int *)addr)[i];
break;
case MPI_UNSIGNED:
for (i = 0; i < count; i++)
((unsigned *)buf)[i] = ((unsigned *)addr)[i];
break;
case MPI_UNSIGNED_LONG:
for (i = 0; i < count; i++)
((unsigned long *)buf)[i] = ((unsigned long *)addr)[i];
break;
case MPI_UNSIGNED_LONG_LONG:
for (i = 0; i < count; i++)
((unsigned long long *)buf)[i] = ((unsigned long long *)addr)[i];
break;
case MPI_DOUBLE:
for (i = 0; i < count; i++)
((double *)buf)[i] = ((double *)addr)[i];
break;
case MPI_CHAR:
MPL_strncpy(buf, addr, count);
break;
default:
/* FIXME the error handling code may not have been setup yet */
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_INTERN, "**intern", "**intern %s", "unexpected parameter type");
break;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_GetParentPort(char ** parent_port)
{
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
char val[MPIDI_MAX_KVS_VALUE_LEN];
if (parent_port_name == NULL)
{
char *kvsname = NULL;
/* We can always use PMI_KVS_Get on our own process group */
MPIDI_PG_GetConnKVSname( &kvsname );
#ifdef USE_PMI2_API
{
int vallen = 0;
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);
pmi_errno = PMI2_KVS_Get(kvsname, PMI2_ID_NULL, PARENT_PORT_KVSKEY, val, sizeof(val), &vallen);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);
if (pmi_errno)
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**pmi_kvsget", "**pmi_kvsget %s", PARENT_PORT_KVSKEY);
}
#else
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);
pmi_errno = PMI_KVS_Get( kvsname, PARENT_PORT_KVSKEY, val, sizeof(val));
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);
if (pmi_errno) {
mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME, __LINE__, MPI_ERR_OTHER, "**pmi_kvsget", "**pmi_kvsget %d", pmi_errno);
goto fn_exit;
}
#endif
parent_port_name = MPL_strdup(val);
if (parent_port_name == NULL) {
MPIR_ERR_POP(mpi_errno); /* FIXME DARIUS */
}
}
*parent_port = parent_port_name;
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_choose_netmod(void)
{
int mpi_errno = MPI_SUCCESS;
int i;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_NEM_CHOOSE_NETMOD);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_NEM_CHOOSE_NETMOD);
MPIR_Assert(MPIR_CVAR_NEMESIS_NETMOD != NULL);
if (strcmp(MPIR_CVAR_NEMESIS_NETMOD, "") == 0)
{
/* netmod not specified, using the default */
MPID_nem_netmod_func = MPID_nem_netmod_funcs[0];
MPID_nem_netmod_id = 0;
#ifdef ENABLE_COMM_OVERRIDES
MPIDI_Anysource_iprobe_fn = MPID_nem_netmod_func->anysource_iprobe;
#endif
goto fn_exit;
}
for (i = 0; i < MPID_nem_num_netmods; ++i)
{
if (!MPIR_Strncasecmp(MPIR_CVAR_NEMESIS_NETMOD, MPID_nem_netmod_strings[i], MPID_NEM_MAX_NETMOD_STRING_LEN))
{
MPID_nem_netmod_func = MPID_nem_netmod_funcs[i];
MPID_nem_netmod_id = i;
#ifdef ENABLE_COMM_OVERRIDES
MPIDI_Anysource_iprobe_fn = MPID_nem_netmod_func->anysource_iprobe;
#endif
goto fn_exit;
}
}
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**invalid_netmod", "**invalid_netmod %s", MPIR_CVAR_NEMESIS_NETMOD);
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_NEM_CHOOSE_NETMOD);
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*
* MPID_Get_universe_size - Get the universe size from the process manager
*
* Notes: The ch3 device requires that the PMI routines are used to
* communicate with the process manager. If a channel wishes to
* bypass the standard PMI implementations, it is the responsibility of the
* channel to provide an implementation of the PMI routines.
*/
int MPID_Get_universe_size(int * universe_size)
{
int mpi_errno = MPI_SUCCESS;
#ifdef USE_PMI2_API
char val[PMI2_MAX_VALLEN];
int found = 0;
char *endptr;
mpi_errno = PMI2_Info_GetJobAttr("universeSize", val, sizeof(val), &found);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
if (!found)
*universe_size = MPIR_UNIVERSE_SIZE_NOT_AVAILABLE;
else {
*universe_size = strtol(val, &endptr, 0);
MPIR_ERR_CHKINTERNAL(endptr - val != strlen(val), mpi_errno, "can't parse universe size");
}
#else
int pmi_errno = PMI_SUCCESS;
pmi_errno = PMI_Get_universe_size(universe_size);
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
"**pmi_get_universe_size",
"**pmi_get_universe_size %d", pmi_errno);
}
if (*universe_size < 0)
{
*universe_size = MPIR_UNIVERSE_SIZE_NOT_AVAILABLE;
}
#endif
fn_exit:
return mpi_errno;
/* --BEGIN ERROR HANDLING-- */
fn_fail:
*universe_size = MPIR_UNIVERSE_SIZE_NOT_AVAILABLE;
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIDI_PG_SetConnInfo( int rank, const char *connString )
{
#ifdef USE_PMI2_API
int mpi_errno = MPI_SUCCESS;
int len;
char key[PMI2_MAX_KEYLEN];
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_PG_SetConnInfo);
len = MPL_snprintf(key, sizeof(key), "P%d-businesscard", rank);
MPIR_ERR_CHKANDJUMP1(len < 0 || len > sizeof(key), mpi_errno, MPI_ERR_OTHER, "**snprintf", "**snprintf %d", len);
mpi_errno = PMI2_KVS_Put(key, connString);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = PMI2_KVS_Fence();
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_PG_SetConnInfo);
return mpi_errno;
fn_fail:
goto fn_exit;
#else
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
int len;
char key[128];
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIR_Assert(pg_world->connData);
len = MPL_snprintf(key, sizeof(key), "P%d-businesscard", rank);
if (len < 0 || len > sizeof(key)) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**snprintf",
"**snprintf %d", len);
}
pmi_errno = PMI_KVS_Put(pg_world->connData, key, connString );
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_kvs_put",
"**pmi_kvs_put %d", pmi_errno);
}
pmi_errno = PMI_KVS_Commit(pg_world->connData);
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_kvs_commit",
"**pmi_kvs_commit %d", pmi_errno);
}
pmi_errno = PMI_Barrier();
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_barrier",
"**pmi_barrier %d", pmi_errno);
}
fn_exit:
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_PG_SetConnInfo);
return mpi_errno;
fn_fail:
goto fn_exit;
#endif
}
int MPID_Comm_spawn_multiple(int count, char *commands[], char **argvs[], const int maxprocs[],
MPIR_Info * info_ptrs[], int root, MPIR_Comm * comm_ptr,
MPIR_Comm ** intercomm, int errcodes[])
{
char port_name[MPI_MAX_PORT_NAME];
int *info_keyval_sizes = 0, i, mpi_errno = MPI_SUCCESS;
PMI_keyval_t **info_keyval_vectors = 0, preput_keyval_vector;
int *pmi_errcodes = 0, pmi_errno = 0;
int total_num_processes, should_accept = 1;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPID_COMM_SPAWN_MULTIPLE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPID_COMM_SPAWN_MULTIPLE);
memset(port_name, 0, sizeof(port_name));
if (comm_ptr->rank == root) {
total_num_processes = 0;
for (i = 0; i < count; i++)
total_num_processes += maxprocs[i];
pmi_errcodes = (int *) MPL_malloc(sizeof(int) * total_num_processes, MPL_MEM_BUFFER);
MPIR_ERR_CHKANDJUMP(!pmi_errcodes, mpi_errno, MPI_ERR_OTHER, "**nomem");
for (i = 0; i < total_num_processes; i++)
pmi_errcodes[i] = 0;
mpi_errno = MPID_Open_port(NULL, port_name);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
info_keyval_sizes = (int *) MPL_malloc(count * sizeof(int), MPL_MEM_BUFFER);
MPIR_ERR_CHKANDJUMP(!info_keyval_sizes, mpi_errno, MPI_ERR_OTHER, "**nomem");
info_keyval_vectors =
(PMI_keyval_t **) MPL_malloc(count * sizeof(PMI_keyval_t *), MPL_MEM_BUFFER);
MPIR_ERR_CHKANDJUMP(!info_keyval_vectors, mpi_errno, MPI_ERR_OTHER, "**nomem");
if (!info_ptrs)
for (i = 0; i < count; i++) {
info_keyval_vectors[i] = 0;
info_keyval_sizes[i] = 0;
} else
for (i = 0; i < count; i++) {
mpi_errno = mpi_to_pmi_keyvals(info_ptrs[i], &info_keyval_vectors[i],
&info_keyval_sizes[i]);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
preput_keyval_vector.key = MPIDI_PARENT_PORT_KVSKEY;
preput_keyval_vector.val = port_name;
pmi_errno = PMI_Spawn_multiple(count, (const char **)
commands,
(const char ***) argvs,
maxprocs, info_keyval_sizes, (const PMI_keyval_t **)
info_keyval_vectors, 1, &preput_keyval_vector, pmi_errcodes);
if (pmi_errno != PMI_SUCCESS)
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
"**pmi_spawn_multiple", "**pmi_spawn_multiple %d", pmi_errno);
if (errcodes != MPI_ERRCODES_IGNORE) {
for (i = 0; i < total_num_processes; i++) {
errcodes[i] = pmi_errcodes[0];
should_accept = should_accept && errcodes[i];
}
should_accept = !should_accept;
}
}
if (errcodes != MPI_ERRCODES_IGNORE) {
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
mpi_errno = MPIR_Bcast(&should_accept, 1, MPI_INT, root, comm_ptr, &errflag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Bcast(&pmi_errno, 1, MPI_INT, root, comm_ptr, &errflag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Bcast(&total_num_processes, 1, MPI_INT, root, comm_ptr, &errflag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Bcast(errcodes, total_num_processes, MPI_INT, root, comm_ptr, &errflag);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
if (should_accept) {
mpi_errno = MPID_Comm_accept(port_name, NULL, root, comm_ptr, intercomm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
} else {
if ((pmi_errno == PMI_SUCCESS) && (errcodes[0] != 0)) {
mpi_errno = MPIR_Comm_create(intercomm);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
}
if (comm_ptr->rank == root) {
mpi_errno = MPID_Close_port(port_name);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
}
fn_exit:
if (info_keyval_vectors) {
free_pmi_keyvals(info_keyval_vectors, count, info_keyval_sizes);
MPL_free(info_keyval_vectors);
}
MPL_free(info_keyval_sizes);
MPL_free(pmi_errcodes);
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPID_COMM_SPAWN_MULTIPLE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_Comm_spawn_multiple(int count, char **commands,
char ***argvs, const int *maxprocs,
MPIR_Info **info_ptrs, int root,
MPIR_Comm *comm_ptr, MPIR_Comm
**intercomm, int *errcodes)
{
char port_name[MPI_MAX_PORT_NAME];
int *info_keyval_sizes=0, i, mpi_errno=MPI_SUCCESS;
PMI_keyval_t **info_keyval_vectors=0, preput_keyval_vector;
int *pmi_errcodes = 0, pmi_errno;
int total_num_processes, should_accept = 1;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_COMM_SPAWN_MULTIPLE);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDI_COMM_SPAWN_MULTIPLE);
if (comm_ptr->rank == root) {
/* create an array for the pmi error codes */
total_num_processes = 0;
for (i=0; i<count; i++) {
total_num_processes += maxprocs[i];
}
pmi_errcodes = (int*)MPL_malloc(sizeof(int) * total_num_processes);
if (pmi_errcodes == NULL) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
/* initialize them to 0 */
for (i=0; i<total_num_processes; i++)
pmi_errcodes[i] = 0;
/* Open a port for the spawned processes to connect to */
/* FIXME: info may be needed for port name */
mpi_errno = MPID_Open_port(NULL, port_name);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
/* --END ERROR HANDLING-- */
/* Spawn the processes */
#ifdef USE_PMI2_API
MPIR_Assert(count > 0);
{
int *argcs = MPL_malloc(count*sizeof(int));
struct MPIR_Info preput;
struct MPIR_Info *preput_p[1] = { &preput };
MPIR_Assert(argcs);
/*
info_keyval_sizes = MPL_malloc(count * sizeof(int));
*/
/* FIXME cheating on constness */
preput.key = (char *)PARENT_PORT_KVSKEY;
preput.value = port_name;
preput.next = NULL;
/* compute argcs array */
for (i = 0; i < count; ++i) {
argcs[i] = 0;
if (argvs != NULL && argvs[i] != NULL) {
while (argvs[i][argcs[i]]) {
++argcs[i];
}
}
/* a fib for now */
/*
info_keyval_sizes[i] = 0;
*/
}
/* XXX DJG don't need this, PMI API is thread-safe? */
/*MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);*/
/* release the global CS for spawn PMI calls */
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
pmi_errno = PMI2_Job_Spawn(count, (const char **)commands,
argcs, (const char ***)argvs,
maxprocs,
info_keyval_sizes, (const MPIR_Info **)info_ptrs,
1, (const struct MPIR_Info **)preput_p,
NULL, 0,
/*jobId, jobIdSize,*/ /* XXX DJG job stuff? */
pmi_errcodes);
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
/*MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);*/
MPL_free(argcs);
if (pmi_errno != PMI2_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
"**pmi_spawn_multiple", "**pmi_spawn_multiple %d", pmi_errno);
}
}
#else
/* FIXME: This is *really* awkward. We should either
Fix on MPI-style info data structures for PMI (avoid unnecessary
duplication) or add an MPIU_Info_getall(...) that creates
the necessary arrays of key/value pairs */
/* convert the infos into PMI keyvals */
info_keyval_sizes = (int *) MPL_malloc(count * sizeof(int));
info_keyval_vectors =
(PMI_keyval_t**) MPL_malloc(count * sizeof(PMI_keyval_t*));
if (!info_keyval_sizes || !info_keyval_vectors) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
if (!info_ptrs) {
for (i=0; i<count; i++) {
info_keyval_vectors[i] = 0;
info_keyval_sizes[i] = 0;
}
}
else {
for (i=0; i<count; i++) {
mpi_errno = mpi_to_pmi_keyvals( info_ptrs[i],
&info_keyval_vectors[i],
&info_keyval_sizes[i] );
if (mpi_errno) { MPIR_ERR_POP(mpi_errno); }
}
}
preput_keyval_vector.key = PARENT_PORT_KVSKEY;
preput_keyval_vector.val = port_name;
MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);
pmi_errno = PMI_Spawn_multiple(count, (const char **)
commands,
(const char ***) argvs,
maxprocs, info_keyval_sizes,
(const PMI_keyval_t **)
info_keyval_vectors, 1,
&preput_keyval_vector,
pmi_errcodes);
MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_PMI_MUTEX);
if (pmi_errno != PMI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
"**pmi_spawn_multiple", "**pmi_spawn_multiple %d", pmi_errno);
}
#endif
if (errcodes != MPI_ERRCODES_IGNORE) {
for (i=0; i<total_num_processes; i++) {
/* FIXME: translate the pmi error codes here */
errcodes[i] = pmi_errcodes[i];
/* We want to accept if any of the spawns succeeded.
Alternatively, this is the same as we want to NOT accept if
all of them failed. should_accept = NAND(e_0, ..., e_n)
Remember, success equals false (0). */
should_accept = should_accept && errcodes[i];
}
should_accept = !should_accept; /* the `N' in NAND */
}
}
if (errcodes != MPI_ERRCODES_IGNORE) {
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
mpi_errno = MPIR_Bcast_impl(&should_accept, 1, MPI_INT, root, comm_ptr, &errflag);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Bcast_impl(&total_num_processes, 1, MPI_INT, root, comm_ptr, &errflag);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
mpi_errno = MPIR_Bcast_impl(errcodes, total_num_processes, MPI_INT, root, comm_ptr, &errflag);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");
}
if (should_accept) {
mpi_errno = MPID_Comm_accept(port_name, NULL, root, comm_ptr, intercomm);
if (mpi_errno) MPIR_ERR_POP(mpi_errno);
}
else {
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**pmi_spawn_multiple");
}
if (comm_ptr->rank == root) {
/* Close the port opened for the spawned processes to connect to */
mpi_errno = MPID_Close_port(port_name);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS)
{
MPIR_ERR_POP(mpi_errno);
}
/* --END ERROR HANDLING-- */
}
fn_exit:
if (info_keyval_vectors) {
free_pmi_keyvals(info_keyval_vectors, count, info_keyval_sizes);
MPL_free(info_keyval_sizes);
MPL_free(info_keyval_vectors);
}
if (pmi_errcodes) {
MPL_free(pmi_errcodes);
}
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDI_COMM_SPAWN_MULTIPLE);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_PktHandler_EagerSyncSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
intptr_t *buflen, MPIR_Request **rreqp )
{
MPIDI_CH3_Pkt_eager_send_t * es_pkt = &pkt->eager_send;
MPIR_Request * rreq;
int found;
int complete;
char *data_buf;
intptr_t data_len;
int mpi_errno = MPI_SUCCESS;
MPL_DBG_MSG_FMT(MPIDI_CH3_DBG_OTHER,VERBOSE,(MPL_DBG_FDEST,
"received eager sync send pkt, sreq=0x%08x, rank=%d, tag=%d, context=%d",
es_pkt->sender_req_id, es_pkt->match.parts.rank,
es_pkt->match.parts.tag,
es_pkt->match.parts.context_id));
MPL_DBG_MSGPKT(vc,es_pkt->match.parts.tag,es_pkt->match.parts.context_id,
es_pkt->match.parts.rank,es_pkt->data_sz,
"ReceivedEagerSync");
rreq = MPIDI_CH3U_Recvq_FDP_or_AEU(&es_pkt->match, &found);
MPIR_ERR_CHKANDJUMP1(!rreq, mpi_errno,MPI_ERR_OTHER, "**nomemreq", "**nomemuereq %d", MPIDI_CH3U_Recvq_count_unexp());
/* If the completion counter is 0, that means that the communicator to
* which this message is being sent has been revoked and we shouldn't
* bother finishing this. */
if (!found && MPIR_cc_get(rreq->cc) == 0) {
*rreqp = NULL;
goto fn_fail;
}
set_request_info(rreq, es_pkt, MPIDI_REQUEST_EAGER_MSG);
data_len = ((*buflen - sizeof(MPIDI_CH3_Pkt_t) >= rreq->dev.recv_data_sz)
? rreq->dev.recv_data_sz : *buflen - sizeof(MPIDI_CH3_Pkt_t));
data_buf = (char *)pkt + sizeof(MPIDI_CH3_Pkt_t);
if (found)
{
MPIDI_CH3_Pkt_t upkt;
MPIDI_CH3_Pkt_eager_sync_ack_t * const esa_pkt = &upkt.eager_sync_ack;
MPIR_Request * esa_req;
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else {
mpi_errno = MPIDI_CH3U_Receive_data_found( rreq, data_buf,
&data_len, &complete );
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**ch3|postrecv",
"**ch3|postrecv %s", "MPIDI_CH3_PKT_EAGER_SYNC_SEND");
}
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
if (complete)
{
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else
{
*rreqp = rreq;
}
}
MPL_DBG_MSG(MPIDI_CH3_DBG_OTHER,VERBOSE,"sending eager sync ack");
MPIDI_Pkt_init(esa_pkt, MPIDI_CH3_PKT_EAGER_SYNC_ACK);
esa_pkt->sender_req_id = rreq->dev.sender_req_id;
/* Because this is a packet handler, it is already within a CH3 CS */
/* MPID_THREAD_CS_ENTER(POBJ, vc->pobj_mutex); */
mpi_errno = MPIDI_CH3_iStartMsg(vc, esa_pkt, sizeof(*esa_pkt), &esa_req);
/* MPID_THREAD_CS_EXIT(POBJ, vc->pobj_mutex); */
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,
"**ch3|syncack");
}
if (esa_req != NULL) {
MPIR_Request_free(esa_req);
}
}
else
{
if (rreq->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else {
mpi_errno = MPIDI_CH3U_Receive_data_unexpected( rreq, data_buf,
&data_len, &complete );
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**ch3|postrecv",
"**ch3|postrecv %s", "MPIDI_CH3_PKT_EAGER_SYNC_SEND");
}
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
if (complete)
{
mpi_errno = MPID_Request_complete(rreq);
if (mpi_errno != MPI_SUCCESS) {
MPIR_ERR_POP(mpi_errno);
}
*rreqp = NULL;
}
else
{
*rreqp = rreq;
}
}
MPIDI_Request_set_sync_send_flag(rreq, TRUE);
}
fn_fail:
return mpi_errno;
}
