int MPID_nem_tcp_get_vc_from_conninfo (char *pg_id, int pg_rank, struct MPIDI_VC **vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_PG_t *pg;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO);
MPIU_DBG_MSG_FMT(NEM_SOCK_DET, VERBOSE, (MPIU_DBG_FDEST, "pg_id=%s pg_rank=%d", pg_id, pg_rank));
mpi_errno = MPIDI_PG_Find (pg_id, &pg);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
MPIU_ERR_CHKINTERNAL(pg == NULL, mpi_errno, "invalid PG");
MPIU_ERR_CHKINTERNAL(pg_rank < 0 || pg_rank > MPIDI_PG_Get_size (pg), mpi_errno, "invalid pg_rank");
MPIDI_PG_Get_vc_set_active (pg, pg_rank, vc);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_GET_VC_FROM_CONNINFO);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPIDI_CH3_Connect_to_root (const char *port_name, MPIDI_VC_t **new_vc)
{
int mpi_errno = MPI_SUCCESS;
MPIDI_VC_t * vc;
MPIU_CHKPMEM_DECL(1);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
*new_vc = NULL; /* so that the err handling knows to cleanup */
MPIU_CHKPMEM_MALLOC (vc, MPIDI_VC_t *, sizeof(MPIDI_VC_t), mpi_errno, "vc");
/* FIXME - where does this vc get freed?
ANSWER (goodell@) - ch3u_port.c FreeNewVC
(but the VC_Destroy is in this file) */
/* init ch3 portion of vc */
MPIDI_VC_Init (vc, NULL, 0);
/* init channel portion of vc */
MPIU_ERR_CHKINTERNAL(!nemesis_initialized, mpi_errno, "Nemesis not initialized");
vc->ch.recv_active = NULL;
MPIDI_CHANGE_VC_STATE(vc, ACTIVE);
*new_vc = vc; /* we now have a valid, disconnected, temp VC */
mpi_errno = MPID_nem_connect_to_root (port_name, vc);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
MPIU_CHKPMEM_COMMIT();
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3_CONNECT_TO_ROOT);
return mpi_errno;
fn_fail:
/* freeing without giving the lower layer a chance to cleanup can lead to
leaks on error */
if (*new_vc)
MPIDI_CH3_VC_Destroy(*new_vc);
MPIU_CHKPMEM_REAP();
goto fn_exit;
}
int MPIDI_Comm_connect(const char *port_name, MPID_Info *info, int root,
MPID_Comm *comm_ptr, MPID_Comm **newcomm)
{
int mpi_errno=MPI_SUCCESS;
int j, i, rank, recv_ints[3], send_ints[3], context_id;
int remote_comm_size=0;
MPID_Comm *tmp_comm = NULL;
MPIDI_VC_t *new_vc = NULL;
int sendtag=100, recvtag=100, n_remote_pgs;
int n_local_pgs=1, local_comm_size;
pg_translation *local_translation = NULL, *remote_translation = NULL;
pg_node *pg_list = NULL;
MPIDI_PG_t **remote_pg = NULL;
MPIR_Context_id_t recvcontext_id = MPIR_INVALID_CONTEXT_ID;
int errflag = FALSE;
MPIU_CHKLMEM_DECL(3);
MPIDI_STATE_DECL(MPID_STATE_MPIDI_COMM_CONNECT);
MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_COMM_CONNECT);
/* Get the context ID here because we need to send it to the remote side */
mpi_errno = MPIR_Get_contextid( comm_ptr, &recvcontext_id );
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
rank = comm_ptr->rank;
local_comm_size = comm_ptr->local_size;
if (rank == root)
{
/* Establish a communicator to communicate with the root on the
other side. */
mpi_errno = MPIDI_Create_inter_root_communicator_connect(
port_name, &tmp_comm, &new_vc);
if (mpi_errno != MPI_SUCCESS) {
MPIU_ERR_POP_LABEL(mpi_errno, no_port);
}
/* Make an array to translate local ranks to process group index
and rank */
MPIU_CHKLMEM_MALLOC(local_translation,pg_translation*,
local_comm_size*sizeof(pg_translation),
mpi_errno,"local_translation");
/* Make a list of the local communicator's process groups and encode
them in strings to be sent to the other side.
The encoded string for each process group contains the process
group id, size and all its KVS values */
mpi_errno = ExtractLocalPGInfo( comm_ptr, local_translation,
&pg_list, &n_local_pgs );
MPIU_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Can't extract local PG info.");
/* Send the remote root: n_local_pgs, local_comm_size,
Recv from the remote root: n_remote_pgs, remote_comm_size,
recvcontext_id for newcomm */
send_ints[0] = n_local_pgs;
send_ints[1] = local_comm_size;
send_ints[2] = recvcontext_id;
MPIU_DBG_MSG_FMT(CH3_CONNECT,VERBOSE,(MPIU_DBG_FDEST,
"sending 3 ints, %d, %d and %d, and receiving 3 ints",
send_ints[0], send_ints[1], send_ints[2]));
mpi_errno = MPIC_Sendrecv(send_ints, 3, MPI_INT, 0,
sendtag++, recv_ints, 3, MPI_INT,
0, recvtag++, tmp_comm->handle,
MPI_STATUS_IGNORE);
if (mpi_errno != MPI_SUCCESS) {
/* this is a no_port error because we may fail to connect
on the send if the port name is invalid */
MPIU_ERR_POP_LABEL(mpi_errno, no_port);
}
}
int MPIR_Bsend_isend(const void *buf, int count, MPI_Datatype dtype,
int dest, int tag, MPID_Comm *comm_ptr,
MPIR_Bsend_kind_t kind, MPID_Request **request )
{
int mpi_errno = MPI_SUCCESS;
MPIR_Bsend_data_t *p;
MPIR_Bsend_msg_t *msg;
int packsize, pass;
/* Find a free segment and copy the data into it. If we could
have, we would already have used tBsend to send the message with
no copying.
We may want to decide here whether we need to pack at all
or if we can just use (a MPIU_Memcpy) of the buffer.
*/
/* We check the active buffer first. This helps avoid storage
fragmentation */
mpi_errno = MPIR_Bsend_check_active();
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
if (dtype != MPI_PACKED)
MPIR_Pack_size_impl( count, dtype, &packsize );
else
packsize = count;
MPIU_DBG_MSG_D(BSEND,TYPICAL,"looking for buffer of size %d", packsize);
/*
* Use two passes. Each pass is the same; between the two passes,
* attempt to complete any active requests, and start any pending
* ones. If the message can be initiated in the first pass,
* do not perform the second pass.
*/
for (pass = 0; pass < 2; pass++) {
p = MPIR_Bsend_find_buffer( packsize );
if (p) {
MPIU_DBG_MSG_FMT(BSEND,TYPICAL,(MPIU_DBG_FDEST,
"found buffer of size %d with address %p",packsize,p));
/* Found a segment */
msg = &p->msg;
/* Pack the data into the buffer */
/* We may want to optimize for the special case of
either primative or contiguous types, and just
use MPIU_Memcpy and the provided datatype */
msg->count = 0;
if (dtype != MPI_PACKED)
{
mpi_errno = MPIR_Pack_impl( buf, count, dtype, p->msg.msgbuf, packsize, &p->msg.count);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
else
{
MPIU_Memcpy(p->msg.msgbuf, buf, count);
p->msg.count = count;
}
/* Try to send the message. We must use MPID_Isend
because this call must not block */
mpi_errno = MPID_Isend(msg->msgbuf, msg->count, MPI_PACKED,
dest, tag, comm_ptr,
MPID_CONTEXT_INTRA_PT2PT, &p->request );
MPIU_ERR_CHKINTERNAL(mpi_errno, mpi_errno, "Bsend internal error: isend returned err");
/* If the error is "request not available", we should
put this on the pending list. This will depend on
how we signal failure to send. */
if (p->request) {
MPIU_DBG_MSG_FMT(BSEND,TYPICAL,
(MPIU_DBG_FDEST,"saving request %p in %p",p->request,p));
/* An optimization is to check to see if the
data has already been sent. The original code
to do this was commented out and probably did not match
the current request internals */
MPIR_Bsend_take_buffer( p, p->msg.count );
p->kind = kind;
*request = p->request;
}
break;
}
/* If we found a buffer or we're in the seccond pass, then break.
Note that the test on phere is redundant, as the code breaks
out of the loop in the test above if a block p is found. */
if (p || pass == 1) break;
MPIU_DBG_MSG(BSEND,TYPICAL,"Could not find storage, checking active");
/* Try to complete some pending bsends */
MPIR_Bsend_check_active( );
/* Give priority to any pending operations */
MPIR_Bsend_retry_pending( );
}
if (!p) {
/* Return error for no buffer space found */
/* Generate a traceback of the allocated space, explaining why
packsize could not be found */
MPIU_DBG_MSG(BSEND,TYPICAL,"Could not find space; dumping arena" );
MPIU_DBG_STMT(BSEND,TYPICAL,MPIR_Bsend_dump());
MPIU_ERR_SETANDJUMP2(mpi_errno, MPI_ERR_BUFFER, "**bufbsend", "**bufbsend %d %d", packsize, BsendBuffer.buffer_size);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
