int MPID_nem_tcp_set_sockopts (int fd)
{
int mpi_errno = MPI_SUCCESS;
int option, flags;
int ret;
socklen_t len;
/* fprintf(stdout, FCNAME " Enter\n"); fflush(stdout); */
/* I heard you have to read the options after setting them in some implementations */
option = 1;
len = sizeof(int);
ret = setsockopt (fd, IPPROTO_TCP, TCP_NODELAY, &option, len);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
ret = getsockopt (fd, IPPROTO_TCP, TCP_NODELAY, &option, &len);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
option = 128*1024;
len = sizeof(int);
ret = setsockopt (fd, SOL_SOCKET, SO_RCVBUF, &option, len);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
ret = getsockopt (fd, SOL_SOCKET, SO_RCVBUF, &option, &len);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
ret = setsockopt (fd, SOL_SOCKET, SO_SNDBUF, &option, len);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
ret = getsockopt (fd, SOL_SOCKET, SO_SNDBUF, &option, &len);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
flags = fcntl(fd, F_GETFL, 0);
MPIU_ERR_CHKANDJUMP2 (flags == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
ret = fcntl(fd, F_SETFL, flags | SO_REUSEADDR);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
flags = fcntl(fd, F_GETFL, 0);
MPIU_ERR_CHKANDJUMP2 (flags == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
ret = fcntl(fd, F_SETFL, flags | O_NONBLOCK);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", MPIU_Strerror (errno), errno);
fn_exit:
/* fprintf(stdout, FCNAME " Exit\n"); fflush(stdout); */
return mpi_errno;
fn_fail:
/* fprintf(stdout, "failure. mpi_errno = %d\n", mpi_errno); */
goto fn_exit;
}
int MPIR_Cart_shift_impl(MPID_Comm *comm_ptr, int direction, int disp, int *rank_source, int *rank_dest)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Topology *cart_ptr;
int i;
int pos[MAX_CART_DIM];
cart_ptr = MPIR_Topology_get( comm_ptr );
MPIU_ERR_CHKANDJUMP((!cart_ptr || cart_ptr->kind != MPI_CART), mpi_errno, MPI_ERR_TOPOLOGY, "**notcarttopo");
MPIU_ERR_CHKANDJUMP((cart_ptr->topo.cart.ndims == 0), mpi_errno, MPI_ERR_TOPOLOGY, "**dimszero");
MPIU_ERR_CHKANDJUMP2((direction >= cart_ptr->topo.cart.ndims), mpi_errno, MPI_ERR_ARG, "**dimsmany",
"**dimsmany %d %d", cart_ptr->topo.cart.ndims, direction);
/* Check for the case of a 0 displacement */
if (disp == 0) {
*rank_source = *rank_dest = comm_ptr->rank;
}
else {
/* To support advanced implementations that support MPI_Cart_create,
we compute the new position and call PMPI_Cart_rank to get the
source and destination. We could bypass that step if we know that
the mapping is trivial. Copy the current position. */
for (i=0; i<cart_ptr->topo.cart.ndims; i++) {
pos[i] = cart_ptr->topo.cart.position[i];
}
/* We must return MPI_PROC_NULL if shifted over the edge of a
non-periodic mesh */
pos[direction] += disp;
if (!cart_ptr->topo.cart.periodic[direction] &&
(pos[direction] >= cart_ptr->topo.cart.dims[direction] ||
pos[direction] < 0)) {
*rank_dest = MPI_PROC_NULL;
}
else {
MPIR_Cart_rank_impl( cart_ptr, pos, rank_dest );
}
pos[direction] = cart_ptr->topo.cart.position[direction] - disp;
if (!cart_ptr->topo.cart.periodic[direction] &&
(pos[direction] >= cart_ptr->topo.cart.dims[direction] ||
pos[direction] < 0)) {
*rank_source = MPI_PROC_NULL;
}
else {
MPIR_Cart_rank_impl( cart_ptr, pos, rank_source );
}
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int do_dma_recv(int iov_n, MPID_IOV iov[], knem_cookie_t s_cookie, int nodma, volatile knem_status_t **status_p_p, knem_status_t *current_status_p)
{
int mpi_errno = MPI_SUCCESS;
int i, err;
#if KNEM_ABI_VERSION < MPICH_NEW_KNEM_ABI_VERSION
struct knem_cmd_init_async_recv_param recvcmd = {0};
#else
struct knem_cmd_inline_copy icopy;
#endif
struct knem_cmd_param_iovec knem_iov[MPID_IOV_LIMIT];
/* FIXME The knem module iovec is potentially different from the system
iovec. This causes all sorts of fun if you don't realize it and use the
system iovec directly instead. Eventually we need to either unify them
or avoid this extra copy. */
for (i = 0; i < iov_n; ++i) {
knem_iov[i].base = (uintptr_t)iov[i] .MPID_IOV_BUF;
knem_iov[i].len = iov[i] .MPID_IOV_LEN;
}
#if KNEM_ABI_VERSION < MPICH_NEW_KNEM_ABI_VERSION
recvcmd.recv_iovec_array = (uintptr_t) &knem_iov[0];
recvcmd.recv_iovec_nr = iov_n;
recvcmd.status_index = alloc_status_index();
recvcmd.send_cookie = s_cookie;
recvcmd.flags = nodma ? 0 : KNEM_FLAG_DMA | KNEM_FLAG_ASYNCDMACOMPLETE;
err = ioctl(knem_fd, KNEM_CMD_INIT_ASYNC_RECV, &recvcmd);
#else
icopy.local_iovec_array = (uintptr_t) &knem_iov[0];
icopy.local_iovec_nr = iov_n;
icopy.remote_cookie = s_cookie;
icopy.remote_offset = 0;
icopy.write = 0;
icopy.async_status_index = alloc_status_index();
icopy.flags = nodma ? 0 : KNEM_FLAG_DMA | KNEM_FLAG_ASYNCDMACOMPLETE;
err = ioctl(knem_fd, KNEM_CMD_INLINE_COPY, &icopy);
#endif
MPIU_ERR_CHKANDJUMP2(err < 0, mpi_errno, MPI_ERR_OTHER, "**ioctl",
"**ioctl %d %s", errno, MPIU_Strerror(errno));
#if KNEM_ABI_VERSION < MPICH_NEW_KNEM_ABI_VERSION
*status_p_p = &knem_status[recvcmd.status_index];
*current_status_p = KNEM_STATUS_PENDING;
#else
*status_p_p = &knem_status[icopy.async_status_index];
*current_status_p = icopy.current_status;
#endif
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int set_sockopts (int fd)
{
int mpi_errno = MPI_SUCCESS;
int option;
int ret;
option = 0;
ret = setsockopt (fd, IPPROTO_TCP, TCP_NODELAY, &option, sizeof(int));
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
option = 128*1024;
setsockopt (fd, SOL_SOCKET, SO_RCVBUF, &option, sizeof(int));
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
setsockopt (fd, SOL_SOCKET, SO_SNDBUF, &option, sizeof(int));
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_lmt_vmsplice_initiate_lmt(MPIDI_VC_t *vc, MPIDI_CH3_Pkt_t *pkt, MPID_Request *sreq)
{
int mpi_errno = MPI_SUCCESS;
MPID_nem_pkt_lmt_rts_t * const rts_pkt = (MPID_nem_pkt_lmt_rts_t *)pkt;
MPIDI_CH3I_VC *vc_ch = &vc->ch;
int complete = 0;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LMT_VMSPLICE_INITIATE_LMT);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LMT_VMSPLICE_INITIATE_LMT);
/* re-use the same pipe per-pair,per-sender */
if (vc_ch->lmt_copy_buf_handle == NULL) {
int err;
char *pipe_name;
MPIDI_CH3I_VC *vc_ch = &vc->ch;
pipe_name = tempnam(NULL, "lmt_");
MPIU_ERR_CHKANDJUMP2(!pipe_name, mpi_errno, MPI_ERR_OTHER, "**tempnam",
"**tempnam %d %s", errno, MPIU_Strerror(errno));
vc_ch->lmt_copy_buf_handle = MPIU_Strdup(pipe_name);
/* XXX DJG hack */
#undef free
free(pipe_name);
err = mkfifo(vc_ch->lmt_copy_buf_handle, 0660);
MPIU_ERR_CHKANDJUMP2(err < 0, mpi_errno, MPI_ERR_OTHER, "**mkfifo",
"**mkfifo %d %s", errno, MPIU_Strerror(errno));
}
/* can't start sending data yet, need full RTS/CTS handshake */
MPID_nem_lmt_send_RTS(vc, rts_pkt, vc_ch->lmt_copy_buf_handle,
strlen(vc_ch->lmt_copy_buf_handle)+1);
fn_fail:
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LMT_VMSPLICE_INITIATE_LMT);
return mpi_errno;
}
int MPID_nem_scif_get_business_card(int my_rank, char **bc_val_p, int *val_max_sz_p)
{
int mpi_errno = MPI_SUCCESS;
int str_errno = MPIU_STR_SUCCESS;
int ret;
char hostname[512];
uint16_t self;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_SCIF_GET_BUSINESS_CARD);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_SCIF_GET_BUSINESS_CARD);
hostname[sizeof(hostname) - 1] = 0;
gethostname(hostname, sizeof(hostname) - 1);
str_errno =
MPIU_Str_add_string_arg(bc_val_p, val_max_sz_p,
MPIDI_CH3I_HOST_DESCRIPTION_KEY, hostname);
if (str_errno) {
MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno,
MPI_ERR_OTHER, "**buscard_len");
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
str_errno = MPIU_Str_add_int_arg(bc_val_p, val_max_sz_p, MPIDI_CH3I_PORT_KEY, listen_port);
if (str_errno) {
MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno,
MPI_ERR_OTHER, "**buscard_len");
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
ret = scif_get_nodeIDs(NULL, 0, &self);
MPIU_ERR_CHKANDJUMP2(ret == -1, mpi_errno, MPI_ERR_OTHER,
"**scif_get_nodeIDs", "**scif_get_nodeIDs %s %d",
MPIU_Strerror(errno), errno);
str_errno = MPIU_Str_add_int_arg(bc_val_p, val_max_sz_p, MPIDI_CH3I_NODE_KEY, self);
if (str_errno) {
MPIU_ERR_CHKANDJUMP(str_errno == MPIU_STR_NOMEM, mpi_errno,
MPI_ERR_OTHER, "**buscard_len");
MPIU_ERR_SETANDJUMP(mpi_errno, MPI_ERR_OTHER, "**buscard");
}
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_SCIF_GET_BUSINESS_CARD);
return mpi_errno;
fn_fail:
goto fn_exit;
}
static int do_dma_send(MPIDI_VC_t *vc, MPID_Request *sreq, int send_iov_n,
MPID_IOV send_iov[], knem_cookie_t *s_cookiep)
{
int mpi_errno = MPI_SUCCESS;
int i, err;
#if KNEM_ABI_VERSION < MPICH_NEW_KNEM_ABI_VERSION
struct knem_cmd_init_send_param sendcmd = {0};
#else
struct knem_cmd_create_region cr;
#endif
struct knem_cmd_param_iovec knem_iov[MPID_IOV_LIMIT];
/* FIXME The knem module iovec is potentially different from the system
iovec. This causes all sorts of fun if you don't realize it and use the
system iovec directly instead. Eventually we need to either unify them
or avoid this extra copy. */
for (i = 0; i < send_iov_n; ++i) {
knem_iov[i].base = (uintptr_t)send_iov[i] .MPID_IOV_BUF;
knem_iov[i].len = send_iov[i] .MPID_IOV_LEN;
}
#if KNEM_ABI_VERSION < MPICH_NEW_KNEM_ABI_VERSION
sendcmd.send_iovec_array = (uintptr_t) &knem_iov[0];
sendcmd.send_iovec_nr = send_iov_n;
sendcmd.flags = 0;
err = ioctl(knem_fd, KNEM_CMD_INIT_SEND, &sendcmd);
#else
cr.iovec_array = (uintptr_t) &knem_iov[0];
cr.iovec_nr = send_iov_n;
cr.flags = KNEM_FLAG_SINGLEUSE;
cr.protection = PROT_READ;
err = ioctl(knem_fd, KNEM_CMD_CREATE_REGION, &cr);
#endif
MPIU_ERR_CHKANDJUMP2(err < 0, mpi_errno, MPI_ERR_OTHER, "**ioctl",
"**ioctl %d %s", errno, MPIU_Strerror(errno));
#if KNEM_ABI_VERSION < MPICH_NEW_KNEM_ABI_VERSION
*s_cookiep = sendcmd.send_cookie;
#else
*s_cookiep = cr.cookie;
#endif
fn_fail:
fn_exit:
return mpi_errno;
}
int MPIR_Graph_neighbors_count_impl(MPID_Comm *comm_ptr, int rank, int *nneighbors)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Topology *graph_ptr;
graph_ptr = MPIR_Topology_get( comm_ptr );
MPIU_ERR_CHKANDJUMP((!graph_ptr || graph_ptr->kind != MPI_GRAPH), mpi_errno, MPI_ERR_TOPOLOGY, "**notgraphtopo");
MPIU_ERR_CHKANDJUMP2((rank < 0 || rank >= graph_ptr->topo.graph.nnodes), mpi_errno, MPI_ERR_RANK, "**rank",
"**rank %d %d", rank, graph_ptr->topo.graph.nnodes );
if ( rank == 0 )
*nneighbors = graph_ptr->topo.graph.index[rank];
else
*nneighbors = graph_ptr->topo.graph.index[rank] -
graph_ptr->topo.graph.index[rank-1];
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/*@
MPI_Cart_shift - Returns the shifted source and destination ranks, given a
shift direction and amount
Input Parameters:
+ comm - communicator with cartesian structure (handle)
. direction - coordinate dimension of shift (integer)
- displ - displacement (> 0: upwards shift, < 0: downwards shift) (integer)
Output Parameters:
+ source - rank of source process (integer)
- dest - rank of destination process (integer)
Notes:
The 'direction' argument is in the range '[0,n-1]' for an n-dimensional
Cartesian mesh.
.N SignalSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_TOPOLOGY
.N MPI_ERR_COMM
.N MPI_ERR_ARG
@*/
int MPI_Cart_shift(MPI_Comm comm, int direction, int displ, int *source,
int *dest)
{
int mpi_errno = MPI_SUCCESS;
MPID_Comm *comm_ptr = NULL;
MPIR_Topology *cart_ptr;
int i;
int pos[MAX_CART_DIM];
int rank;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_CART_SHIFT);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_CART_SHIFT);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Comm_get_ptr( comm, comm_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate comm_ptr */
MPID_Comm_valid_ptr( comm_ptr, mpi_errno );
/* If comm_ptr is not valid, it will be reset to null */
MPIR_ERRTEST_ARGNULL( source, "source", mpi_errno );
MPIR_ERRTEST_ARGNULL( dest, "dest", mpi_errno );
MPIR_ERRTEST_ARGNEG( direction, "direction", mpi_errno );
/* Nothing in the standard indicates that a zero displacement
is not valid, so we don't check for a zero shift */
if (mpi_errno) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
cart_ptr = MPIR_Topology_get( comm_ptr );
MPIU_ERR_CHKANDJUMP((!cart_ptr || cart_ptr->kind != MPI_CART), mpi_errno, MPI_ERR_TOPOLOGY, "**notcarttopo");
MPIU_ERR_CHKANDJUMP((cart_ptr->topo.cart.ndims == 0), mpi_errno, MPI_ERR_TOPOLOGY, "**dimszero");
MPIU_ERR_CHKANDJUMP2((direction >= cart_ptr->topo.cart.ndims), mpi_errno, MPI_ERR_ARG, "**dimsmany",
"**dimsmany %d %d", cart_ptr->topo.cart.ndims, direction);
/* Check for the case of a 0 displacement */
rank = comm_ptr->rank;
if (displ == 0) {
*source = *dest = rank;
}
else {
/* To support advanced implementations that support MPI_Cart_create,
we compute the new position and call PMPI_Cart_rank to get the
source and destination. We could bypass that step if we know that
the mapping is trivial. Copy the current position. */
for (i=0; i<cart_ptr->topo.cart.ndims; i++) {
pos[i] = cart_ptr->topo.cart.position[i];
}
/* We must return MPI_PROC_NULL if shifted over the edge of a
non-periodic mesh */
pos[direction] += displ;
if (!cart_ptr->topo.cart.periodic[direction] &&
(pos[direction] >= cart_ptr->topo.cart.dims[direction] ||
pos[direction] < 0)) {
*dest = MPI_PROC_NULL;
}
else {
MPIR_Cart_rank_impl( cart_ptr, pos, dest );
}
pos[direction] = cart_ptr->topo.cart.position[direction] - displ;
if (!cart_ptr->topo.cart.periodic[direction] &&
(pos[direction] >= cart_ptr->topo.cart.dims[direction] ||
pos[direction] < 0)) {
*source = MPI_PROC_NULL;
}
else {
MPIR_Cart_rank_impl( cart_ptr, pos, source );
}
}
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_CART_SHIFT);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_cart_shift",
"**mpi_cart_shift %C %d %d %p %p", comm, direction, displ, source, dest);
}
# endif
mpi_errno = MPIR_Err_return_comm( comm_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*@
MPI_Comm_join - Create a communicator by joining two processes connected by
a socket.
Input Parameter:
. fd - socket file descriptor
Output Parameter:
. intercomm - new intercommunicator (handle)
Notes:
The socket must be quiescent before 'MPI_COMM_JOIN' is called and after
'MPI_COMM_JOIN' returns. More specifically, on entry to 'MPI_COMM_JOIN', a
read on the socket will not read any data that was written to the socket
before the remote process called 'MPI_COMM_JOIN'.
.N ThreadSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_ARG
@*/
int MPI_Comm_join(int fd, MPI_Comm *intercomm)
{
static const char FCNAME[] = "MPI_Comm_join";
int mpi_errno = MPI_SUCCESS, err;
MPID_Comm *intercomm_ptr;
char *local_port, *remote_port;
MPIU_CHKLMEM_DECL(2);
MPID_MPI_STATE_DECL(MPID_STATE_MPI_COMM_JOIN);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPIU_THREAD_CS_ENTER(ALLFUNC,);
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_COMM_JOIN);
/* ... body of routine ... */
MPIU_CHKLMEM_MALLOC(local_port, char *, MPI_MAX_PORT_NAME, mpi_errno, "local port name");
MPIU_CHKLMEM_MALLOC(remote_port, char *, MPI_MAX_PORT_NAME, mpi_errno, "remote port name");
mpi_errno = MPIR_Open_port_impl(NULL, local_port);
MPIU_ERR_CHKANDJUMP((mpi_errno != MPI_SUCCESS), mpi_errno, MPI_ERR_OTHER, "**openportfailed");
err = MPIR_fd_send(fd, local_port, MPI_MAX_PORT_NAME);
MPIU_ERR_CHKANDJUMP1((err != 0), mpi_errno, MPI_ERR_INTERN, "**join_send", "**join_send %d", err);
err = MPIR_fd_recv(fd, remote_port, MPI_MAX_PORT_NAME);
MPIU_ERR_CHKANDJUMP1((err != 0), mpi_errno, MPI_ERR_INTERN, "**join_recv", "**join_recv %d", err);
MPIU_ERR_CHKANDJUMP2((strcmp(local_port, remote_port) == 0), mpi_errno, MPI_ERR_INTERN, "**join_portname",
"**join_portname %s %s", local_port, remote_port);
if (strcmp(local_port, remote_port) < 0) {
MPID_Comm *comm_self_ptr;
MPID_Comm_get_ptr( MPI_COMM_SELF, comm_self_ptr );
mpi_errno = MPIR_Comm_accept_impl(local_port, NULL, 0, comm_self_ptr, &intercomm_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
} else {
MPID_Comm *comm_self_ptr;
MPID_Comm_get_ptr( MPI_COMM_SELF, comm_self_ptr );
mpi_errno = MPIR_Comm_connect_impl(remote_port, NULL, 0, comm_self_ptr, &intercomm_ptr);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
}
mpi_errno = MPIR_Close_port_impl(local_port);
if (mpi_errno) MPIU_ERR_POP(mpi_errno);
MPIU_OBJ_PUBLISH_HANDLE(*intercomm, intercomm_ptr->handle);
/* ... end of body of routine ... */
fn_exit:
MPIU_CHKLMEM_FREEALL();
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_COMM_JOIN);
MPIU_THREAD_CS_EXIT(ALLFUNC,);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_comm_join",
"**mpi_comm_join %d %p", fd, intercomm);
}
# endif
mpi_errno = MPIR_Err_return_comm( NULL, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
/*@
MPI_Cart_coords - Determines process coords in cartesian topology given
rank in group
Input Parameters:
+ comm - communicator with cartesian structure (handle)
. rank - rank of a process within group of 'comm' (integer)
- maxdims - length of vector 'coords' in the calling program (integer)
Output Parameter:
. coords - integer array (of size 'ndims') containing the Cartesian
coordinates of specified process (integer)
.N SignalSafe
.N Fortran
.N Errors
.N MPI_SUCCESS
.N MPI_ERR_TOPOLOGY
.N MPI_ERR_RANK
.N MPI_ERR_DIMS
.N MPI_ERR_ARG
@*/
int MPI_Cart_coords(MPI_Comm comm, int rank, int maxdims, int *coords)
{
static const char FCNAME[] = "MPI_Cart_coords";
int mpi_errno = MPI_SUCCESS;
MPID_Comm *comm_ptr = NULL;
MPIR_Topology *cart_ptr;
int i, nnodes;
MPID_MPI_STATE_DECL(MPID_STATE_MPI_CART_COORDS);
MPIR_ERRTEST_INITIALIZED_ORDIE();
MPID_MPI_FUNC_ENTER(MPID_STATE_MPI_CART_COORDS);
/* Validate parameters, especially handles needing to be converted */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIR_ERRTEST_COMM(comm, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif
/* Convert MPI object handles to object pointers */
MPID_Comm_get_ptr( comm, comm_ptr );
/* Validate parameters and objects (post conversion) */
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
/* Validate comm_ptr */
MPID_Comm_valid_ptr( comm_ptr, mpi_errno );
/* If comm_ptr is not valid, it will be reset to null */
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
MPIR_ERRTEST_RANK(comm_ptr, rank, mpi_errno);
if (mpi_errno != MPI_SUCCESS) goto fn_fail;
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
cart_ptr = MPIR_Topology_get( comm_ptr );
# ifdef HAVE_ERROR_CHECKING
{
MPID_BEGIN_ERROR_CHECKS;
{
MPIU_ERR_CHKANDJUMP((!cart_ptr || cart_ptr->kind != MPI_CART), mpi_errno, MPI_ERR_TOPOLOGY, "**notcarttopo");
MPIU_ERR_CHKANDJUMP2((cart_ptr->topo.cart.ndims > maxdims), mpi_errno, MPI_ERR_ARG, "**dimsmany",
"**dimsmany %d %d", cart_ptr->topo.cart.ndims, maxdims);
if (cart_ptr->topo.cart.ndims) {
MPIR_ERRTEST_ARGNULL(coords,"coords",mpi_errno);
if (mpi_errno) goto fn_fail;
}
}
MPID_END_ERROR_CHECKS;
}
# endif /* HAVE_ERROR_CHECKING */
/* ... body of routine ... */
/* Calculate coords */
nnodes = cart_ptr->topo.cart.nnodes;
for ( i=0; i < cart_ptr->topo.cart.ndims; i++ ) {
nnodes = nnodes / cart_ptr->topo.cart.dims[i];
coords[i] = rank / nnodes;
rank = rank % nnodes;
}
/* ... end of body of routine ... */
fn_exit:
MPID_MPI_FUNC_EXIT(MPID_STATE_MPI_CART_COORDS);
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
# ifdef HAVE_ERROR_CHECKING
{
mpi_errno = MPIR_Err_create_code(
mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER, "**mpi_cart_coords",
"**mpi_cart_coords %C %d %d %p", comm, rank, maxdims, coords);
}
# endif
mpi_errno = MPIR_Err_return_comm( comm_ptr, FCNAME, mpi_errno );
goto fn_exit;
/* --END ERROR HANDLING-- */
}
int MPID_nem_tcp_module_lmt_pre_recv (MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV s_cookie, MPID_IOV *r_cookie, int *send_cts)
{
int mpi_errno = MPI_SUCCESS;
int ret;
unsigned int len;
struct sockaddr_in saddr;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_RECV);
mpi_errno = read_s_cookie (s_cookie, &vc_ch->net.tcp.lmt_s_len);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
memset (&saddr, sizeof(saddr), 0);
if (!vc_ch->net.tcp.lmt_connected)
{
vc_ch->net.tcp.lmt_desc = socket (AF_INET, SOCK_STREAM, 0);
MPIU_ERR_CHKANDJUMP2 (vc_ch->net.tcp.lmt_desc == -1, mpi_errno, MPI_ERR_OTHER, "**sock_create", "**sock_create %s %d", strerror (errno), errno);
// ret = fcntl (vc_ch->net.tcp.lmt_desc, F_SETFL, O_NONBLOCK);
// MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
saddr.sin_family = AF_INET;
saddr.sin_addr.s_addr = htonl (INADDR_ANY);
saddr.sin_port = htons (0);
ret = bind (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, sizeof (saddr));
MPIU_ERR_CHKANDJUMP3 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**sock|poll|bind", "**sock|poll|bind %d %d %s", ntohs (saddr.sin_port), errno, strerror (errno));
len = sizeof (saddr);
ret = getsockname (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, &len);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
set_sockopts (vc_ch->net.tcp.lmt_desc);
ret = listen (vc_ch->net.tcp.lmt_desc, SOMAXCONN);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**listen", "**listen %s %d", errno, strerror (errno));
}
MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
mpi_errno = create_r_cookie (MPID_nem_hostname, ntohs (saddr.sin_port), data_sz, &vc_ch->net.tcp.lmt_cookie, &len);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
r_cookie->MPID_IOV_BUF = vc_ch->net.tcp.lmt_cookie;
r_cookie->MPID_IOV_LEN = len;
*send_cts = 1;
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_PRE_RECV);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_tcp_module_lmt_start_recv (MPIDI_VC_t *vc, MPID_Request *req)
{
int mpi_errno = MPI_SUCCESS;
int ret;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPIDI_msg_sz_t last;
int nb;
int r_len;
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_RECV);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_RECV);
free_cookie (vc_ch->net.tcp.lmt_cookie);
if (!vc_ch->net.tcp.lmt_connected)
{
int len;
struct sockaddr_in saddr;
int connfd;
len = sizeof (saddr);
connfd = accept (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, &len);
MPIU_ERR_CHKANDJUMP2 (connfd == -1, mpi_errno, MPI_ERR_OTHER, "**sock|poll|accept", "**sock|poll|accept %d %s", errno, strerror (errno));
/* close listen fd */
do
ret = close (vc_ch->net.tcp.lmt_desc);
while (ret == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**closesocket", "**closesocket %s %d", strerror (errno), errno);
/* set lmt_desc to new connected fd */
vc_ch->net.tcp.lmt_desc = connfd;
vc_ch->net.tcp.lmt_connected = 1;
// ret = fcntl (vc_ch->net.tcp.lmt_desc, F_SETFL, O_NONBLOCK);
// MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
}
MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
if (data_sz > vc_ch->net.tcp.lmt_s_len)
{
data_sz = vc_ch->net.tcp.lmt_s_len;
}
else if (data_sz < vc_ch->net.tcp.lmt_s_len)
{
/* message will be truncated */
r_len = data_sz;
req->status.MPI_ERROR = MPIU_ERR_SET2 (mpi_errno, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", vc_ch->net.tcp.lmt_s_len, r_len);
}
MPID_Segment_init (req->dev.user_buf, req->dev.user_count, req->dev.datatype, &req->dev.segment, 0);
req->dev.segment_first = 0;
req->dev.segment_size = data_sz;
req->dev.iov_count = MPID_IOV_LIMIT;
req->dev.iov_offset = 0;
last = data_sz;
do
{
int iov_offset;
int left_to_recv;
MPID_Segment_unpack_vector (&req->dev.segment, req->dev.segment_first, &last, req->dev.iov, &req->dev.iov_count);
left_to_recv = last - req->dev.segment_first;
iov_offset = 0;
#ifdef TESTING_CHUNKING
{
char *buf = req->dev.iov[0].MPID_IOV_BUF;
int l;
while (left_to_recv)
{
if (left_to_recv > CHUNK)
l = CHUNK;
else
l = left_to_recv;
do
nb = read (vc_ch->net.tcp.lmt_desc, buf, l);
while (nb == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
left_to_recv -= nb;
buf += nb;
}
MPIDI_CH3U_Request_complete (req);
goto fn_exit;
}
#endif
do
nb = readv (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
while (nb == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP2 (nb == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
MPIU_ERR_CHKANDJUMP (nb == 0, mpi_errno, MPI_ERR_OTHER, "**fail");
left_to_recv -= nb;
while (left_to_recv)
{ /* recv rest of iov */
while (nb >= req->dev.iov[iov_offset].MPID_IOV_LEN)
{ /* update iov to reflect sent bytes */
nb -= req->dev.iov[iov_offset].MPID_IOV_LEN;
++iov_offset;
}
req->dev.iov[iov_offset].MPID_IOV_BUF = (char *)req->dev.iov[iov_offset].MPID_IOV_BUF + nb;
req->dev.iov[iov_offset].MPID_IOV_LEN -= nb;
do
nb = readv (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
while (nb == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP2 (nb == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
MPIU_ERR_CHKANDJUMP (nb == 0, mpi_errno, MPI_ERR_OTHER, "**fail");
left_to_recv -= nb;
}
}
while (last < data_sz);
MPIDI_CH3U_Request_complete (req);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_RECV);
return mpi_errno;
fn_fail:
goto fn_exit;
}
int MPID_nem_tcp_module_lmt_start_send (MPIDI_VC_t *vc, MPID_Request *req, MPID_IOV r_cookie)
{
int mpi_errno = MPI_SUCCESS;
int ret;
MPIDI_msg_sz_t data_sz;
int dt_contig;
MPI_Aint dt_true_lb;
MPID_Datatype * dt_ptr;
MPIDI_msg_sz_t last;
int nb;
int s_len = 0;
int r_len;
int r_port;
char *r_hostname;
MPIDI_CH3I_VC *vc_ch = (MPIDI_CH3I_VC *)vc->channel_private;
MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_SEND);
MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_SEND);
mpi_errno = read_r_cookie (r_cookie, &r_hostname, &r_port, &r_len);
if (mpi_errno) MPIU_ERR_POP (mpi_errno);
free_cookie (vc_ch->net.tcp.lmt_cookie);
if (!vc_ch->net.tcp.lmt_connected)
{
struct sockaddr_in saddr;
struct hostent *hp;
vc_ch->net.tcp.lmt_desc = socket (AF_INET, SOCK_STREAM, 0);
MPIU_ERR_CHKANDJUMP2 (vc_ch->net.tcp.lmt_desc == -1, mpi_errno, MPI_ERR_OTHER, "**sock_create", "**sock_create %s %d", strerror (errno), errno);
// ret = fcntl (vc_ch->net.tcp.lmt_desc, F_SETFL, O_NONBLOCK);
// MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
hp = gethostbyname (r_hostname);
MPIU_ERR_CHKANDJUMP2 (hp == NULL, mpi_errno, MPI_ERR_OTHER, "**gethostbyname", "**gethostbyname %s %d", hstrerror (h_errno), h_errno);
memset (&saddr, sizeof(saddr), 0);
saddr.sin_family = AF_INET;
saddr.sin_port = htons (r_port);
MPIU_Memcpy (&saddr.sin_addr, hp->h_addr, hp->h_length);
set_sockopts (vc_ch->net.tcp.lmt_desc);
ret = connect (vc_ch->net.tcp.lmt_desc, (struct sockaddr *)&saddr, sizeof(saddr));
MPIU_ERR_CHKANDJUMP2 (ret == -1, mpi_errno, MPI_ERR_OTHER, "**fail", "**fail %s %d", strerror (errno), errno);
vc_ch->net.tcp.lmt_connected = 1;
}
MPIDI_Datatype_get_info (req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr, dt_true_lb);
if (r_len < data_sz)
{
/* message will be truncated */
s_len = data_sz;
data_sz = r_len;
req->status.MPI_ERROR = MPIU_ERR_SET2 (mpi_errno, MPI_ERR_TRUNCATE, "**truncate", "**truncate %d %d", s_len, r_len);
}
MPID_Segment_init (req->dev.user_buf, req->dev.user_count, req->dev.datatype, &req->dev.segment, 0);
req->dev.segment_first = 0;
req->dev.segment_size = data_sz;
req->dev.iov_count = MPID_IOV_LIMIT;
req->dev.iov_offset = 0;
last = data_sz;
do
{
int iov_offset;
int left_to_send;
MPID_Segment_pack_vector (&req->dev.segment, req->dev.segment_first, &last, req->dev.iov, &req->dev.iov_count);
left_to_send = last - req->dev.segment_first;
iov_offset = 0;
#ifdef TESTING_CHUNKING
{
char *buf = req->dev.iov[0].MPID_IOV_BUF;
int l;
while (left_to_send)
{
if (left_to_send > CHUNK)
l = CHUNK;
else
l = left_to_send;
do
nb = write (vc_ch->net.tcp.lmt_desc, buf, l);
while (nb == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
left_to_send -= nb;
buf += nb;
}
MPIDI_CH3U_Request_complete (req);
goto fn_exit;
}
#endif
do
nb = writev (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
while (nb == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
left_to_send -= nb;
while (left_to_send)
{ /* send rest of iov */
while (nb >= req->dev.iov[iov_offset].MPID_IOV_LEN)
{ /* update iov to reflect sent bytes */
nb -= req->dev.iov[iov_offset].MPID_IOV_LEN;
++iov_offset;
}
req->dev.iov[iov_offset].MPID_IOV_BUF = (char *)req->dev.iov[iov_offset].MPID_IOV_BUF + nb;
req->dev.iov[iov_offset].MPID_IOV_LEN -= nb;
do
nb = writev (vc_ch->net.tcp.lmt_desc, &req->dev.iov[iov_offset], req->dev.iov_count - iov_offset);
while (nb == -1 && errno == EINTR);
MPIU_ERR_CHKANDJUMP (nb == -1, mpi_errno, MPI_ERR_OTHER, "**sock_writev");
left_to_send -= nb;
}
}
while (last < data_sz);
MPIDI_CH3U_Request_complete (req);
fn_exit:
MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_TCP_MODULE_LMT_START_SEND);
return mpi_errno;
fn_fail:
goto fn_exit;
}
