int MPIDI_CH3U_Post_data_receive_unexpected(MPID_Request * rreq)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_POST_DATA_RECEIVE_UNEXPECTED);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_POST_DATA_RECEIVE_UNEXPECTED);
/* FIXME: to improve performance, allocate temporary buffer from a
specialized buffer pool. */
/* FIXME: to avoid memory exhaustion, integrate buffer pool management
with flow control */
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"unexpected request allocated");
rreq->dev.tmpbuf = MPIU_Malloc(rreq->dev.recv_data_sz);
if (!rreq->dev.tmpbuf) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d",
rreq->dev.recv_data_sz);
}
rreq->dev.tmpbuf_sz = rreq->dev.recv_data_sz;
rreq->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST)rreq->dev.tmpbuf;
rreq->dev.iov[0].MPID_IOV_LEN = rreq->dev.recv_data_sz;
rreq->dev.iov_count = 1;
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackUEBufComplete;
rreq->dev.recv_pending_count = 2;
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_POST_DATA_RECEIVE_UNEXPECTED);
return mpi_errno;
}
int MPIDI_CH3U_Receive_data_unexpected(MPID_Request * rreq, char *buf, MPIDI_msg_sz_t *buflen, int *complete)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
/* FIXME: to improve performance, allocate temporary buffer from a
specialized buffer pool. */
/* FIXME: to avoid memory exhaustion, integrate buffer pool management
with flow control */
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"unexpected request allocated");
rreq->dev.tmpbuf = MPIU_Malloc(rreq->dev.recv_data_sz);
if (!rreq->dev.tmpbuf) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**nomem","**nomem %d",
rreq->dev.recv_data_sz);
}
rreq->dev.tmpbuf_sz = rreq->dev.recv_data_sz;
/* if all of the data has already been received, copy it
now, otherwise build an iov and let the channel copy it */
if (rreq->dev.recv_data_sz <= *buflen)
{
MPIU_Memcpy(rreq->dev.tmpbuf, buf, rreq->dev.recv_data_sz);
*buflen = rreq->dev.recv_data_sz;
rreq->dev.recv_pending_count = 1;
*complete = TRUE;
}
else
{
rreq->dev.iov[0].MPID_IOV_BUF = (MPID_IOV_BUF_CAST)((char *)rreq->dev.tmpbuf);
rreq->dev.iov[0].MPID_IOV_LEN = rreq->dev.recv_data_sz;
rreq->dev.iov_count = 1;
rreq->dev.recv_pending_count = 2;
*buflen = 0;
*complete = FALSE;
}
if (MPIDI_Request_get_msg_type(rreq) == MPIDI_REQUEST_EAGER_MSG)
MPIR_T_ADD(RECVQ_STATISTICS, MPIDI_CH3I_unexpected_recvq_buffer_size, rreq->dev.tmpbuf_sz);
rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackUEBufComplete;
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_RECEIVE_DATA_UNEXPECTED);
return mpi_errno;
}
int MPIDI_CH3_PktHandler_RndvSend( MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt,
MPIDI_msg_sz_t *buflen, MPID_Request **rreqp )
{
MPIDI_CH3_Pkt_rndv_send_t * rs_pkt = &pkt->rndv_send;
int mpi_errno = MPI_SUCCESS;
int complete;
char *data_buf;
MPIDI_msg_sz_t data_len;
MPID_Request *req;
MPIU_DBG_MSG(CH3_OTHER,VERBOSE,"received rndv send (data) pkt");
MPID_Request_get_ptr(rs_pkt->receiver_req_id, req);
data_len = ((*buflen - sizeof(MPIDI_CH3_Pkt_t) >= req->dev.recv_data_sz)
? req->dev.recv_data_sz : *buflen - sizeof(MPIDI_CH3_Pkt_t));
data_buf = (char *)pkt + sizeof(MPIDI_CH3_Pkt_t);
if (req->dev.recv_data_sz == 0) {
*buflen = sizeof(MPIDI_CH3_Pkt_t);
MPIDI_CH3U_Request_complete(req);
*rreqp = NULL;
}
else {
mpi_errno = MPIDI_CH3U_Receive_data_found(req, data_buf, &data_len,
&complete);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**ch3|postrecv",
"**ch3|postrecv %s", "MPIDI_CH3_PKT_RNDV_SEND");
}
*buflen = sizeof(MPIDI_CH3_Pkt_t) + data_len;
if (complete)
{
MPIDI_CH3U_Request_complete(req);
*rreqp = NULL;
}
else
{
*rreqp = req;
}
}
fn_fail:
return mpi_errno;
}
int MPIR_T_cvar_read_impl(MPI_T_cvar_handle handle, void *buf)
{
int mpi_errno = MPI_SUCCESS;
struct MPIR_Param_t *p = handle->p;
switch (p->default_val.type) {
case MPIR_PARAM_TYPE_INT:
{
int *i_buf = buf;
*i_buf = *(int *)p->val_p;
}
break;
case MPIR_PARAM_TYPE_DOUBLE:
{
double *d_buf = buf;
*d_buf = *(double *)p->val_p;
}
break;
case MPIR_PARAM_TYPE_BOOLEAN:
{
int *i_buf = buf;
*i_buf = *(int *)p->val_p;
}
break;
case MPIR_PARAM_TYPE_STRING:
if (*(char **)p->val_p == NULL) {
char *c_buf = buf;
c_buf[0] = '\0';
}
else {
MPIU_Strncpy(buf, *(char **)p->val_p, MPIR_PARAM_MAX_STRLEN);
}
break;
case MPIR_PARAM_TYPE_RANGE:
MPIU_Memcpy(buf, p->val_p, 2*sizeof(int));
break;
default:
/* FIXME the error handling code may not have been setup yet */
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_INTERN, "**intern", "**intern %s", "unexpected parameter type");
break;
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/**
* Get the type of a device.
*
* @param dev the device.
* @param ctx the device context.
* @param hca_type the type (output).
*
* @return MPI_SUCCESS if succeded, MPI_ERR_OTHER if failed
*
* \see HCA_Type
*/
static inline int get_hca_type (struct ibv_device* dev, struct ibv_context* ctx, HCA_Type* hca_type)
{
MPIDI_STATE_DECL(MPID_STATE_GET_HCA_TYPE);
MPIDI_FUNC_ENTER(MPID_STATE_GET_HCA_TYPE);
int ret;
int mpi_errno = MPI_SUCCESS;
struct ibv_device_attr dev_attr;
memset(&dev_attr, 0, sizeof(struct ibv_device_attr));
char* dev_name = (char*) ibv_get_device_name(dev);
if (!dev_name)
{
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**ibv_get_device_name");
}
ret = ibv_query_device(ctx, &dev_attr);
if (ret)
{
MPIU_ERR_SETANDJUMP1(
mpi_errno,
MPI_ERR_OTHER,
"**ibv_query_device",
"**ibv_query_device %s",
dev_name
);
}
if ((mpi_errno = hcaNameToType(dev_name, hca_type)) != MPI_SUCCESS)
{
MPIU_ERR_POP(mpi_errno);
}
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_GET_HCA_TYPE);
return mpi_errno;
}
int MPIDI_Comm_spawn_multiple(int count, char **commands,
char ***argvs, const int *maxprocs,
MPID_Info **info_ptrs, int root,
MPID_Comm *comm_ptr, MPID_Comm
**intercomm, int *errcodes)
{
char port_name[MPI_MAX_PORT_NAME];
int *info_keyval_sizes=0, i, mpi_errno=MPI_SUCCESS;
PMI_keyval_t **info_keyval_vectors=0, preput_keyval_vector;
int *pmi_errcodes = 0, pmi_errno;
int total_num_processes, should_accept = 1;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMM_SPAWN_MULTIPLE);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_COMM_SPAWN_MULTIPLE);
if (comm_ptr->rank == root) {
/* create an array for the pmi error codes */
total_num_processes = 0;
for (i=0; i<count; i++) {
total_num_processes += maxprocs[i];
}
pmi_errcodes = (int*)MPIU_Malloc(sizeof(int) * total_num_processes);
if (pmi_errcodes == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
/* initialize them to 0 */
for (i=0; i<total_num_processes; i++)
pmi_errcodes[i] = 0;
/* Open a port for the spawned processes to connect to */
/* FIXME: info may be needed for port name */
mpi_errno = MPID_Open_port(NULL, port_name);
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
/* --END ERROR HANDLING-- */
/* Spawn the processes */
#ifdef USE_PMI2_API
MPIU_Assert(count > 0);
{
int *argcs = MPIU_Malloc(count*sizeof(int));
struct MPID_Info preput;
struct MPID_Info *preput_p[1] = { &preput };
MPIU_Assert(argcs);
/*
info_keyval_sizes = MPIU_Malloc(count * sizeof(int));
*/
/* FIXME cheating on constness */
preput.key = (char *)PARENT_PORT_KVSKEY;
preput.value = port_name;
preput.next = NULL;
/* compute argcs array */
for (i = 0; i < count; ++i) {
argcs[i] = 0;
if (argvs != NULL && argvs[i] != NULL) {
while (argvs[i][argcs[i]]) {
++argcs[i];
}
}
/* a fib for now */
/*
info_keyval_sizes[i] = 0;
*/
}
/* XXX DJG don't need this, PMI API is thread-safe? */
/*MPIU_THREAD_CS_ENTER(PMI,);*/
/* release the global CS for spawn PMI calls */
MPIU_THREAD_CS_EXIT(ALLFUNC,);
pmi_errno = PMI2_Job_Spawn(count, (const char **)commands,
argcs, (const char ***)argvs,
maxprocs,
info_keyval_sizes, (const MPID_Info **)info_ptrs,
1, (const struct MPID_Info **)preput_p,
NULL, 0,
/*jobId, jobIdSize,*/ /* XXX DJG job stuff? */
pmi_errcodes);
MPIU_THREAD_CS_ENTER(ALLFUNC,);
/*MPIU_THREAD_CS_EXIT(PMI,);*/
MPIU_Free(argcs);
if (pmi_errno != PMI2_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
"**pmi_spawn_multiple", "**pmi_spawn_multiple %d", pmi_errno);
}
}
#else
/* FIXME: This is *really* awkward. We should either
Fix on MPI-style info data structures for PMI (avoid unnecessary
duplication) or add an MPIU_Info_getall(...) that creates
the necessary arrays of key/value pairs */
/* convert the infos into PMI keyvals */
info_keyval_sizes = (int *) MPIU_Malloc(count * sizeof(int));
info_keyval_vectors =
(PMI_keyval_t**) MPIU_Malloc(count * sizeof(PMI_keyval_t*));
if (!info_keyval_sizes || !info_keyval_vectors) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**nomem");
}
if (!info_ptrs) {
for (i=0; i<count; i++) {
info_keyval_vectors[i] = 0;
info_keyval_sizes[i] = 0;
}
}
else {
for (i=0; i<count; i++) {
mpi_errno = mpi_to_pmi_keyvals( info_ptrs[i],
&info_keyval_vectors[i],
&info_keyval_sizes[i] );
if (mpi_errno) { MPIU_ERR_POP(mpi_errno); }
}
}
preput_keyval_vector.key = PARENT_PORT_KVSKEY;
preput_keyval_vector.val = port_name;
MPIU_THREAD_CS_ENTER(PMI,);
pmi_errno = PMI_Spawn_multiple(count, (const char **)
commands,
(const char ***) argvs,
maxprocs, info_keyval_sizes,
(const PMI_keyval_t **)
info_keyval_vectors, 1,
&preput_keyval_vector,
pmi_errcodes);
MPIU_THREAD_CS_EXIT(PMI,);
if (pmi_errno != PMI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER,
"**pmi_spawn_multiple", "**pmi_spawn_multiple %d", pmi_errno);
}
#endif
if (errcodes != MPI_ERRCODES_IGNORE) {
for (i=0; i<total_num_processes; i++) {
/* FIXME: translate the pmi error codes here */
errcodes[i] = pmi_errcodes[i];
/* We want to accept if any of the spawns succeeded.
Alternatively, this is the same as we want to NOT accept if
all of them failed. should_accept = NAND(e_0, ..., e_n)
Remember, success equals false (0). */
should_accept = should_accept && errcodes[i];
}
should_accept = !should_accept; /* the `N' in NAND */
}
}
/**
* Initialize Process Manager Interface and update global_info.
* Called by MPID_nem_ib_init.
*
* -# Initialize the Process Manager Interface;
* -# Set the rank;
* -# Set the progexx group size;
*
* \see MPID_nem_ib_init
*
\todo Need to add more stuff here
Look at InitPG in mvapich2/trunk/src/mpid/ch3/src/mpid_init.c
*/
int MPID_nem_ib_pmi_init()
{
int pmi_errno = 0;
int mpi_errno = MPI_SUCCESS;
int spawned;
/* Process group id size*/
int pg_id_sz, pg_size;
char *pg_id;
MPIDI_PG_t *pg = 0;
assert( global_info!= NULL );
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PMI_INIT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PMI_INIT);
/* Initialize the Process Manager Interface */
pmi_errno = PMI_Init(&spawned);
if (pmi_errno != PMI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**pmi_init",
"**pmi_init %d", pmi_errno);
}
/* Set the rank */
pmi_errno = PMI_Get_rank(&global_info->pg_rank);
if (pmi_errno != PMI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**pmi_get_rank",
"**pmi_get_rank %d", pmi_errno);
}
/* Set the progexx group size */
pmi_errno = PMI_Get_size(&global_info->pg_size);
if (pmi_errno != 0) {
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**pmi_get_size",
"**pmi_get_size %d", pmi_errno);
}
/* -------------------------------------- From InitPG in mvapich2/trunk/src/mpid/ch3/src/mpid_init.c
pmi_errno = PMI_Get_appnum(&appnum);
if (pmi_errno != PMI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_get_appnum",
"**pmi_get_appnum %d", pmi_errno);
}
/ * Note that if pmi is not availble, the value of MPI_APPNUM is
not set * /
if (appnum != -1) {
MPIR_Process.attrs.appnum = appnum;
}
*/
/* Now, initialize the process group information with PMI calls */
/*
* Get the process group id
*/
pmi_errno = PMI_KVS_Get_name_length_max(&pg_id_sz);
if (pmi_errno != PMI_SUCCESS) {
/*
* I don't believe that MPICH2 has updated the error message for this
* yet.
*/
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,
"**pmi_get_id_length_max",
"**pmi_get_id_length_max %d", pmi_errno);
}
/* This memory will be freed by the PG_Destroy if there is an error */
pg_id = MPIU_Malloc(pg_id_sz + 1);
if (pg_id == NULL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**nomem");
}
/* Note in the singleton init case, the pg_id is a dummy.
We'll want to replace this value if we join a
Process manager */
pmi_errno = PMI_KVS_Get_my_name(pg_id, pg_id_sz);
if (pmi_errno != PMI_SUCCESS) {
/*
* I don't believe the MPICH2 team has updated the error message for
* this change yet.
*/
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_get_id",
"**pmi_get_id %d", pmi_errno);
}
/*
* Create a new structure to track the process group for our MPI_COMM_WORLD
*/
mpi_errno = MPIDI_PG_Create(pg_size, pg_id, &pg);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**dev|pg_create");
}
MPIDI_PG_InitConnKVS( pg );
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PMI_INIT);
return mpi_errno;
fn_fail:
if (pg) {
MPIDI_PG_Destroy( pg );
}
goto fn_exit;
}
/*@
MPI_Pack - Packs a datatype into contiguous memory
Input Parameters:
+ inbuf - input buffer start (choice)
. incount - number of input data items (non-negative integer)
. datatype - datatype of each input data item (handle)
. outsize - output buffer size, in bytes (non-negative integer)
- comm - communicator for packed message (handle)
Output Parameters:
. outbuf - output buffer start (choice)
Input/Output Parameters:
. position - current position in buffer, in bytes (integer)
Notes (from the specifications):
The input value of position is the first location in the output buffer to be
used for packing. position is incremented by the size of the packed message,
and the output value of position is the first location in the output buffer
following the locations occupied by the packed message. The comm argument is
the communicator that will be subsequently used for sending the packed
message.
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_ARG
.N MPI_ERR_OTHER
@*/
int MPI_Pack(const void *inbuf,
int incount,
MPI_Datatype datatype,
void *outbuf,
int outsize,
int *position,
MPI_Comm comm)
{
int mpi_errno = MPI_SUCCESS;
MPI_Aint position_x;
MPID_Comm *comm_ptr = NULL;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_PACK);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_PACK);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Comm_get_ptr(comm, comm_ptr);
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COUNT(incount,mpi_errno);
MPIR_ERRTEST_COUNT(outsize,mpi_errno);
/* NOTE: inbuf could be null (MPI_BOTTOM) */
if (incount > 0) {
MPIR_ERRTEST_ARGNULL(outbuf, "output buffer", mpi_errno);
}
MPIR_ERRTEST_ARGNULL(position, "position", mpi_errno);
/* Validate comm_ptr */
/* If comm_ptr is not valid, it will be reset to null */
MPID_Comm_valid_ptr(comm_ptr, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
MPIR_ERRTEST_DATATYPE(datatype, "datatype", mpi_errno);
if (HANDLE_GET_KIND(datatype) != HANDLE_KIND_BUILTIN) {
MPID_Datatype *datatype_ptr = NULL;
MPID_Datatype_get_ptr(datatype, datatype_ptr);
MPID_Datatype_valid_ptr(datatype_ptr, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
MPID_Datatype_committed_ptr(datatype_ptr, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
#ifdef HAVE_ERROR_CHECKING /* IMPLEMENTATION-SPECIFIC ERROR CHECKS */
{
int tmp_sz;
MPID_BEGIN_ERROR_CHECKS;
/* Verify that there is space in the buffer to pack the type */
MPID_Datatype_get_size_macro(datatype, tmp_sz);
if (tmp_sz * incount > outsize - *position) {
if (*position < 0) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_ARG,
"**argposneg","**argposneg %d",
*position);
}
else if (outsize < 0) {
MPIU_ERR_SETANDJUMP2(mpi_errno,MPI_ERR_ARG,"**argneg",
"**argneg %s %d","outsize",outsize);
}
else if (incount < 0) {
MPIU_ERR_SETANDJUMP2(mpi_errno,MPI_ERR_ARG,"**argneg",
"**argneg %s %d","incount",incount);
}
else {
MPIU_ERR_SETANDJUMP2(mpi_errno,MPI_ERR_ARG,"**argpackbuf",
"**argpackbuf %d %d", tmp_sz * incount,
outsize - *position);
}
}
MPID_END_ERROR_CHECKS;
}
#endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
position_x = *position;
mpi_errno = MPIR_Pack_impl(inbuf, incount, datatype, outbuf, outsize, &position_x);
MPIU_Assign_trunc(*position, position_x, int);
if (mpi_errno) goto fn_fail;
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_PACK);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_pack",
"**mpi_pack %p %d %D %p %d %p %C", inbuf, incount, datatype, outbuf, outsize, position, comm);
}
# endif
mpi_errno = MPIR_Err_return_comm(comm_ptr, FCNAME, mpi_errno);
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPIDI_CH3_Connect_to_root(const char* port_name, MPIDI_VC_t** new_vc)
{
int mpi_errno = MPI_SUCCESS;
int str_errno;
char ifname[MAX_HOST_DESCRIPTION_LEN];
MPIDI_VC_t *vc;
MPIDI_CH3_Pkt_cm_establish_t pkt;
MPID_Request * sreq;
int seqnum;
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
*new_vc = NULL;
if (!MPIDI_CH3I_Process.has_dpm)
return MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, FCNAME,
__LINE__, MPI_ERR_OTHER, "**notimpl", 0);
str_errno = MPIU_Str_get_string_arg(port_name,
MPIDI_CH3I_HOST_DESCRIPTION_KEY,
ifname, MAX_HOST_DESCRIPTION_LEN);
if (str_errno != MPIU_STR_SUCCESS) {
/* --BEGIN ERROR HANDLING */
if (str_errno == MPIU_STR_FAIL) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER,"**argstr_missinghost");
}
else {
/* MPIU_STR_TRUNCATED or MPIU_STR_NONEM */
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_hostd");
}
/* --END ERROR HANDLING-- */
}
vc = MPIU_Malloc(sizeof(MPIDI_VC_t));
if (!vc) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**nomem");
}
MPIDI_VC_Init(vc, NULL, 0);
mpi_errno = MPIDI_CH3I_CM_Connect_raw_vc(vc, ifname);
if (mpi_errno) {
MPIU_ERR_POP(mpi_errno);
}
while (vc->ch.state != MPIDI_CH3I_VC_STATE_IDLE) {
mpi_errno = MPID_Progress_test();
/* --BEGIN ERROR HANDLING-- */
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP(mpi_errno);
}
}
/* fprintf(stderr, "[###] vc state to idel, now send cm_establish msg\n") */
/* Now a connection is created, send a cm_establish message */
/* FIXME: vc->mrail.remote_vc_addr is used to find remote vc
* A more elegant way is needed */
MPIDI_Pkt_init(&pkt, MPIDI_CH3_PKT_CM_ESTABLISH);
MPIDI_VC_FAI_send_seqnum(vc, seqnum);
MPIDI_Pkt_set_seqnum(&pkt, seqnum);
pkt.vc_addr = vc->mrail.remote_vc_addr;
mpi_errno = MPIDI_GetTagFromPort(port_name, &pkt.port_name_tag);
if (mpi_errno != MPIU_STR_SUCCESS) {
MPIU_ERR_SETANDJUMP(mpi_errno,MPI_ERR_OTHER, "**argstr_port_name_tag");
}
mpi_errno = MPIDI_CH3_iStartMsg(vc, &pkt, sizeof(pkt), &sreq);
if (mpi_errno != MPI_SUCCESS)
{
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER,"**fail", "**fail %s",
"Failed to send cm establish message");
}
if (sreq != NULL)
{
if (sreq->status.MPI_ERROR != MPI_SUCCESS)
{
mpi_errno = MPIR_Err_create_code(sreq->status.MPI_ERROR,
MPIR_ERR_FATAL, FCNAME, __LINE__,
MPI_ERR_OTHER,
"**fail", 0);
MPID_Request_release(sreq);
goto fn_fail;
}
MPID_Request_release(sreq);
}
*new_vc = vc;
fn_fail:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
return mpi_errno;
}
int MPIDI_PG_SetConnInfo( int rank, const char *connString )
{
#ifdef USE_PMI2_API
int mpi_errno = MPI_SUCCESS;
int len;
char key[PMI2_MAX_KEYLEN];
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PG_SetConnInfo);
len = MPIU_Snprintf(key, sizeof(key), "P%d-businesscard", rank);
MPIU_ERR_CHKANDJUMP1(len < 0 || len > sizeof(key), mpi_errno, MPI_ERR_OTHER, "**snprintf", "**snprintf %d", len);
mpi_errno = PMI2_KVS_Put(key, connString);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
mpi_errno = PMI2_KVS_Fence();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PG_SetConnInfo);
return mpi_errno;
fn_fail:
goto fn_exit;
#else
int mpi_errno = MPI_SUCCESS;
int pmi_errno;
int len;
char key[128];
MPIDI_STATE_DECL(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_PG_SetConnInfo);
MPIU_Assert(pg_world->connData);
len = MPIU_Snprintf(key, sizeof(key), "P%d-businesscard", rank);
if (len < 0 || len > sizeof(key)) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**snprintf",
"**snprintf %d", len);
}
pmi_errno = PMI_KVS_Put(pg_world->connData, key, connString );
if (pmi_errno != PMI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_kvs_put",
"**pmi_kvs_put %d", pmi_errno);
}
pmi_errno = PMI_KVS_Commit(pg_world->connData);
if (pmi_errno != PMI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_kvs_commit",
"**pmi_kvs_commit %d", pmi_errno);
}
pmi_errno = PMI_Barrier();
if (pmi_errno != PMI_SUCCESS) {
MPIU_ERR_SETANDJUMP1(mpi_errno,MPI_ERR_OTHER, "**pmi_barrier",
"**pmi_barrier %d", pmi_errno);
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_PG_SetConnInfo);
return mpi_errno;
fn_fail:
goto fn_exit;
#endif
}
int MPID_nem_lmt_dma_progress(void)
{
int mpi_errno = MPI_SUCCESS;
struct lmt_dma_node *prev = NULL;
struct lmt_dma_node *free_me = NULL;
struct lmt_dma_node *cur = outstanding_head;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_DMA_PROGRESS);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_DMA_PROGRESS);
/* Iterate over a linked-list of (req,status_idx)-tuples looking for
completed/failed requests. Currently knem only provides status to the
receiver, so all of these requests are recv requests. */
while (cur) {
switch (*cur->status_p) {
case KNEM_STATUS_SUCCESS:
{
/* complete the request if all data has been sent, remove it from the list */
int complete = 0;
mpi_errno = check_req_complete(cur->vc, cur->req, &complete);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
free_status_index(cur->status_p - knem_status);
if (complete) {
/* request was completed by the OnDataAvail fn */
MPID_nem_lmt_send_DONE(cur->vc, cur->req); /* tell the other side to complete its request */
MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
}
else {
/* There is more data to send. We must inform the sender that we have
completely received the current batch and that the next batch should
be sent. */
MPID_nem_lmt_send_COOKIE(cur->vc, cur->req, NULL, 0);
}
/* Right now we always free the cur element, even if the
request is incomplete because it simplifies the logic. */
if (cur == outstanding_head) {
outstanding_head = cur->next;
prev = NULL;
free_me = cur;
cur = cur->next;
}
else {
prev->next = cur->next;
free_me = cur;
cur = cur->next;
}
if (free_me) MPIU_Free(free_me);
--MPID_nem_local_lmt_pending;
continue;
}
break;
case KNEM_STATUS_FAILED:
/* set the error status for the request, complete it then dequeue the entry */
cur->req->status.MPI_ERROR = MPI_SUCCESS;
MPIU_ERR_SET1(cur->req->status.MPI_ERROR, MPI_ERR_OTHER, "**recv_status", "**recv_status %d", *cur->status_p);
MPIDI_CH3U_Request_complete(cur->req);
if (cur == outstanding_head) {
outstanding_head = cur->next;
prev = NULL;
free_me = cur;
cur = cur->next;
}
else {
prev->next = cur->next;
free_me = cur;
cur = cur->next;
}
if (free_me) MPIU_Free(free_me);
--MPID_nem_local_lmt_pending;
continue;
break;
case KNEM_STATUS_PENDING:
/* nothing to do here */
break;
default:
MPIU_ERR_SETANDJUMP1(mpi_errno, MPI_ERR_OTHER, "**invalid_knem_status",
"**invalid_knem_status %d", *cur->status_p);
break;
}
prev = cur;
cur = cur->next;
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_DMA_PROGRESS);
return mpi_errno;
fn_fail:
goto fn_exit;
}