int main(int argc, char *argv[]) { int rank, size; int lsize, rsize; int grank, gsize; MPI_Comm parent, global; MPI_Init(&argc, &argv); // Locat info MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); // Global info MPI_Comm_get_parent(&parent); if (parent == MPI_COMM_NULL) error("No parent!"); MPI_Comm_remote_size(parent, &size); MPI_Comm_size(parent, &lsize); MPI_Comm_remote_size(parent, &rsize); MPI_Intercomm_merge(parent, 1, &global); MPI_Comm_rank(global, &grank); MPI_Comm_size(global, &gsize); printf("child %d: lsize=%d, rsize=%d, grank=%d, gsize=%d\n", rank, lsize, rsize, grank, gsize); MPI_Barrier(global); printf("%d: after Barrier\n", grank); MPI_Comm_free(&global); MPI_Finalize(); return 0; }
static void emulate_armci_init_clusinfo() { int psize; MPI_Comm_remote_size(MPI_COMM_SERVER2CLIENT, &psize); /* server id (i.e. server's armci_me) is derived from node master's id. Similar to armci_create_server_process() to set SERVER_CONTEXT */ armci_me = SOFFSET - armci_client_first; armci_nproc = psize; armci_usr_tid = THREAD_ID_SELF(); /*remember the main user thread id */ armci_master = armci_client_first; /* ***** emulate armci_init_clusinfo() ***** */ armci_clus_me = armci_server_me; armci_nclus = armci_nserver; armci_clus_first = armci_clus_info[armci_clus_me].master; armci_clus_last = (armci_clus_first + armci_clus_info[armci_clus_me].nslave - 1); if(armci_clus_first != armci_client_first || armci_nclients != armci_clus_info[armci_clus_me].nslave) { armci_mpi2_server_debug(armci_server_me, "armci_clus_first=%d, armci_clus_last=%d\n", armci_clus_first, armci_clus_last); armci_die("mpi2_server: armci_clus_info is incorrect.", 0); } }
int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, res, rsize, peer; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Comm_remote_size(comm, &rsize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_remote_size() (%i)\n", res); return res; } handle->tmpbuf=(void*)malloc(2*sizeof(char)); schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc()\n"); return res; } res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; } if (0 == rank) { for (peer = 1 ; peer < rsize ; ++peer) { res = NBC_Sched_recv (0, true, 1, MPI_BYTE, peer, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } } } /* synchronize with the remote root */ res = NBC_Sched_recv (0, true, 1, MPI_BYTE, 0, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } res = NBC_Sched_send (0, true, 1, MPI_BYTE, 0, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } if (0 == rank) { /* wait for the remote root */ res = NBC_Sched_barrier(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; } /* inform remote peers that all local peers have entered the barrier */ for (peer = 0 ; peer < rsize ; ++peer) { res = NBC_Sched_send (0, true, 1, MPI_BYTE, peer, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } } } res = NBC_Sched_commit(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; } return NBC_OK; }
FORTRAN_API void FORT_CALL mpi_comm_remote_size_ ( MPI_Fint *comm, MPI_Fint *size, MPI_Fint *__ierr ) { int l_size; *__ierr = MPI_Comm_remote_size( MPI_Comm_f2c(*comm), &l_size); *size = l_size; }
int MPIPortsCommunication:: getRemoteCommunicatorSize() { preciceTrace ( "getRemoteCommunicatorSize()" ); assertion ( _isConnection ); int remoteSize = 0; MPI_Comm_remote_size ( communicator(), &remoteSize ); return remoteSize; }
/* simple linear Alltoallv */ int ompi_coll_libnbc_ialltoallv_inter (void* sendbuf, int *sendcounts, int *sdispls, MPI_Datatype sendtype, void* recvbuf, int *recvcounts, int *rdispls, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_0_0_t *module) { int rank, res, i, rsize; MPI_Aint sndext, rcvext; NBC_Schedule *schedule; char *rbuf, *sbuf; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Type_extent(sendtype, &sndext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } res = MPI_Type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } MPI_Comm_remote_size (comm, &rsize); schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; } handle->tmpbuf=NULL; res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; } for (i = 0; i < rsize; i++) { /* post all sends */ if(sendcounts[i] != 0) { sbuf = ((char *) sendbuf) + (sdispls[i] * sndext); res = NBC_Sched_send(sbuf, false, sendcounts[i], sendtype, i, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } } /* post all receives */ if(recvcounts[i] != 0) { rbuf = ((char *) recvbuf) + (rdispls[i] * rcvext); res = NBC_Sched_recv(rbuf, false, recvcounts[i], recvtype, i, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } } } /*NBC_PRINT_SCHED(*schedule);*/ res = NBC_Sched_commit(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; } return NBC_OK; }
int CWorkerGroup::nbworkers () const { int size; // use macro? MPI_Comm_remote_size(m_comm, &size); return size; }
int main(int argc, char *argv[]) { MPI_Status status; MPI_Comm comm, scomm; int a[10], b[10]; int buf[BUFSIZE], *bptr, bl, i, j, rank, size, color, errs = 0; MTest_Init(0, 0); MPI_Comm_rank(MPI_COMM_WORLD, &rank); color = rank % 2; MPI_Comm_split(MPI_COMM_WORLD, color, rank, &scomm); MPI_Intercomm_create(scomm, 0, MPI_COMM_WORLD, 1 - color, 52, &comm); MPI_Comm_rank(comm, &rank); MPI_Comm_remote_size(comm, &size); MPI_Buffer_attach(buf, BUFSIZE); for (j = 0; j < 10; j++) { for (i = 0; i < 10; i++) { a[i] = (rank + 10 * j) * size + i; } MPI_Bsend(a, 10, MPI_INT, 0, 27 + j, comm); } if (rank == 0) { for (i = 0; i < size; i++) { for (j = 0; j < 10; j++) { int k; status.MPI_TAG = -10; status.MPI_SOURCE = -20; MPI_Recv(b, 10, MPI_INT, i, 27 + j, comm, &status); if (status.MPI_TAG != 27 + j) { errs++; printf("Wrong tag = %d\n", status.MPI_TAG); } if (status.MPI_SOURCE != i) { errs++; printf("Wrong source = %d\n", status.MPI_SOURCE); } for (k = 0; k < 10; k++) { if (b[k] != (i + 10 * j) * size + k) { errs++; printf("received b[%d] = %d from %d tag %d\n", k, b[k], i, 27 + j); } } } } } MPI_Buffer_detach(&bptr, &bl); MPI_Comm_free(&scomm); MPI_Comm_free(&comm); MTest_Finalize(errs); return MTestReturnValue(errs); }
/*! \param [in] parent Pointer to context on client side \param [in] intraComm_ communicator of group client \param [in] interComm_ communicator of group server \cxtSer [in] cxtSer Pointer to context of server side. (It is only used on case of attached mode) */ CContextClient::CContextClient(CContext* parent, MPI_Comm intraComm_, MPI_Comm interComm_, CContext* cxtSer) : mapBufferSize_(), parentServer(cxtSer), maxBufferedEvents(4) { context = parent; intraComm = intraComm_; interComm = interComm_; MPI_Comm_rank(intraComm, &clientRank); MPI_Comm_size(intraComm, &clientSize); int flag; MPI_Comm_test_inter(interComm, &flag); if (flag) MPI_Comm_remote_size(interComm, &serverSize); else MPI_Comm_size(interComm, &serverSize); if (clientSize < serverSize) { int serverByClient = serverSize / clientSize; int remain = serverSize % clientSize; int rankStart = serverByClient * clientRank; if (clientRank < remain) { serverByClient++; rankStart += clientRank; } else rankStart += remain; for (int i = 0; i < serverByClient; i++) ranksServerLeader.push_back(rankStart + i); ranksServerNotLeader.resize(0); } else { int clientByServer = clientSize / serverSize; int remain = clientSize % serverSize; if (clientRank < (clientByServer + 1) * remain) { if (clientRank % (clientByServer + 1) == 0) ranksServerLeader.push_back(clientRank / (clientByServer + 1)); else ranksServerNotLeader.push_back(clientRank / (clientByServer + 1)); } else { int rank = clientRank - (clientByServer + 1) * remain; if (rank % clientByServer == 0) ranksServerLeader.push_back(remain + rank / clientByServer); else ranksServerNotLeader.push_back(remain + rank / clientByServer); } } timeLine = 0; }
int ompi_coll_libnbc_igatherv_inter (void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int *recvcounts, int *displs, MPI_Datatype recvtype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_0_0_t *module) { int rank, p, res, i, rsize; MPI_Aint rcvext; NBC_Schedule *schedule; char *rbuf; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Comm_size(comm, &p); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_size() (%i)\n", res); return res; } res = MPI_Comm_remote_size (comm, &rsize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_remote_size() (%i)\n", res); return res; } if (MPI_ROOT == root) { res = MPI_Type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } } handle->tmpbuf = NULL; schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; } res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; } /* send to root */ if (MPI_ROOT != root && MPI_PROC_NULL != root) { /* send msg to root */ res = NBC_Sched_send(sendbuf, false, sendcount, sendtype, root, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } } else if (MPI_ROOT == root) { for (i = 0 ; i < rsize ; ++i) { rbuf = ((char *)recvbuf) + (displs[i]*rcvext); /* root receives message to the right buffer */ res = NBC_Sched_recv(rbuf, false, recvcounts[i], recvtype, i, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } } } res = NBC_Sched_commit(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; } return NBC_OK; }
int ompi_coll_libnbc_iallgatherv_inter(void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int *recvcounts, int *displs, MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_1_0_t *module) { int rank, res, r, rsize; MPI_Aint rcvext; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Comm_remote_size(comm, &rsize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_remote_size() (%i)\n", res); return res; } res = MPI_Type_extent(recvtype, &rcvext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc() (%i)\n", res); return res; } handle->tmpbuf=NULL; res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create, (%i)\n", res); return res; } /* do rsize rounds */ for (r = 0 ; r < rsize ; ++r) { char *rbuf = ((char *)recvbuf) + (displs[r]*rcvext); if (recvcounts[r]) { res = NBC_Sched_recv(rbuf, false, recvcounts[r], recvtype, r, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } } } if (sendcount) { for (r = 0 ; r < rsize ; ++r) { res = NBC_Sched_send(sendbuf, false, sendcount, sendtype, r, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } } } res = NBC_Sched_commit(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; } return NBC_OK; }
int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_0_0_t *module) { int rank, p, res, size, peer; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Comm_size(comm, &p); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Type_size(datatype, &size); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; } handle->tmpbuf=NULL; schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create, res = %i\n", res); return res; } if(root != MPI_PROC_NULL) { /* send to all others */ if(root == MPI_ROOT) { int remsize; res = MPI_Comm_remote_size(comm, &remsize); if(MPI_SUCCESS != res) { printf("MPI_Comm_remote_size() failed\n"); return res; } for (peer=0;peer<remsize;peer++) { /* send msg to peer */ res = NBC_Sched_send(buffer, false, count, datatype, peer, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } } } else { /* recv msg from root */ res = NBC_Sched_recv(buffer, false, count, datatype, root, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } } } res = NBC_Sched_commit(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if (NBC_OK != res) { printf("Error in NBC_Start() (%i)\n", res); return res; } return NBC_OK; }
void Cache::orient(int *argc_ptr, char ***argv_ptr) { MPI_ASSERT(argc_ptr != NULL); MPI_ASSERT(argv_ptr != NULL); int argc = *argc_ptr; char **argv = *argv_ptr; MPI_ASSERT(argc >= 3); #ifdef DEBUG // Print the argv list for reference printf("job_node argc: %d\n", argc); for (int i = 0; i < argc; ++i) { printf("argv[%d]: %s\n", i, argv[i]); } // #endif // Get the job_num for this job. char *endptr; job_num = strtol(argv[1], &endptr, 10); // Grab the job to cache node pairings list. std::string mapping(argv[2]); stringlist_to_vector(job_to_cache, mapping); // Update argc and argv so things are transparent to the caller. // TODO: Ensure this is working properly argc_ptr -= 1; std::string exec_name(argv[0]); memcpy(argv[2], exec_name.c_str(), exec_name.size()); *argv_ptr = *(argv_ptr + 2); // Get details on the world this node lives in. MPI_Comm_size(MPI_COMM_WORLD, &local_size); MPI_Comm_rank(MPI_COMM_WORLD, &local_rank); MPI_Comm_get_parent(&parent_comm); MPI_Comm_remote_size(parent_comm, &parent_size); MPI_Comm_rank(parent_comm, &parent_rank); // Get coordinator cache node's rank for this job node. coord_rank = job_to_cache[local_rank]; #ifdef DEBUG printf("Job node: local rank - %d/%d parent rank - %d/%d\n", local_rank, local_size, parent_rank, parent_size); printf("Job Num: %d Job node %d: team cache node: %d\n", job_num, local_rank, coord_rank); #endif }
void ompi_comm_remote_size_f(MPI_Fint *comm, MPI_Fint *size, MPI_Fint *ierr) { int c_ierr; MPI_Comm c_comm = MPI_Comm_f2c ( *comm ); OMPI_SINGLE_NAME_DECL(size); c_ierr = MPI_Comm_remote_size ( c_comm, OMPI_SINGLE_NAME_CONVERT(size )); if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr); if (MPI_SUCCESS == c_ierr) { OMPI_SINGLE_INT_2_FINT(size); } }
int main(int argc, char *argv[]) { int rank, size; int universe_size, *universe_sizep, flag; int lsize, rsize; int grank, gsize; MPI_Comm everyone, global; /* intercommunicator */ char worker_program[100]; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_get_attr(MPI_COMM_WORLD, MPI_UNIVERSE_SIZE, &universe_sizep, &flag); if (!flag) { universe_size = 8; } else universe_size = *universe_sizep; if( rank == 0 ) { printf("univ size = %d\n", universe_size); } sprintf(worker_program, "./slave"); MPI_Comm_spawn(worker_program, MPI_ARGV_NULL, 6, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &everyone, MPI_ERRCODES_IGNORE); MPI_Comm_size(everyone, &lsize); MPI_Comm_remote_size(everyone, &rsize); MPI_Intercomm_merge(everyone, 1, &global); MPI_Comm_rank(global, &grank); MPI_Comm_size(global, &gsize); printf("parent %d: lsize=%d, rsize=%d, grank=%d, gsize=%d\n", rank, lsize, rsize, grank, gsize); MPI_Barrier(global); printf("%d: after Barrier\n", grank); MPI_Comm_free(&global); MPI_Finalize(); return 0; }
void CWorkerGroup::signal_solve( Common::SignalArgs & args) { std::string str; char * buffer; int remote_size; SignalFrame frame("solve", uri(), "//Root/Worker"); to_string( *frame.xml_doc, str); buffer = new char[ str.length() + 1 ]; std::strcpy( buffer, str.c_str() ); MPI_Comm_remote_size(m_comm, &remote_size); for(int i = 0 ; i < remote_size ; ++i) MPI_Send( buffer, str.length() + 1, MPI_CHAR, i, 0, m_comm ); }
void LeaderNode::spawn_cache_nodes(uint32_t job_num, MPI_Comm *comm, uint16_t count) { MPI_ASSERT(comm != NULL); #ifdef DEBUG printf("LeaderNode sending SPAWN_CACHE of size %d\n", count); #endif int result; int comm_size; MPI_Comm_remote_size(*comm, &comm_size); SpawnNodesTemplate *format = (SpawnNodesTemplate *)buf; format->job_num = job_num; format->count = count; std::vector<uint32_t> vec = job_to_swing[job_num]; std::string mapping; vector_to_stringlist(vec, mapping); format->mapping_size = (uint16_t)mapping.size(); memcpy(format->mapping, mapping.c_str(), mapping.size()); MPI_ASSERT(mapping.size() <= MAX_MAPPING_SIZE); int msg_size = sizeof(SpawnNodesTemplate); #ifdef DEBUG printf("job_num: %d\n", job_num); printf("count: %d\n", count); printf("spawn_cache_msg_size: %d\n", msg_size); printf("job_num: %d\ncount: %d\nmapping_size: %d\nmapping: %s\n", format->job_num, format->count, format->mapping_size, format->mapping); #endif // TODO: Look into MPI_Comm_Idup and perhaps MPI_Bcast for sending out this // spawn request to all nodes efficiently and having them all handle it // efficiently. // Have all swing nodes collectively spawn the cache nodes. for (uint32_t i = 0; i < comm_size; ++i) { #ifdef DEBUG printf("Leader sending spawn cache msg to swing node %d\n", i); #endif result = send_msg(buf, msg_size, MPI_UINT8_T, i, SPAWN_CACHE, *comm, &request); MPI_ASSERT(result == MPI_SUCCESS); } }
int ompi_coll_libnbc_iallreduce_inter(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_0_0_t *module) { int rank, res, size, rsize; MPI_Aint ext; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Comm_remote_size(comm, &rsize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_remote_size() (%i)\n", res); return res; } res = MPI_Type_extent(datatype, &ext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } res = MPI_Type_size(datatype, &size); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_size() (%i)\n", res); return res; } handle->tmpbuf = malloc(ext*count); if(handle->tmpbuf == NULL) { printf("Error in malloc() (%i)\n", res); return NBC_OOR; } schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc()\n"); return res; } res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; } res = allred_sched_linear(rank, rsize, sendbuf, recvbuf, count, datatype, op, ext, size, schedule, handle); if (NBC_OK != res) { printf("Error in Schedule creation() (%i)\n", res); return res; } res = NBC_Sched_commit(schedule); if(res != NBC_OK) { free(handle->tmpbuf); printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if(res != NBC_OK) { free(handle->tmpbuf); printf("Error in NBC_Start() (%i)\n", res); return res; } /* tmpbuf is freed with the handle */ return NBC_OK; }
// TODO: REMOVE THIS METHOD (It is just for testing functionality). void LeaderNode::create_test_job() { MPI_Comm swing_comm; // TODO: This is a simple place holder for swing node spawning, // will want to make this more flexible later. spawn_swing_nodes(MPI_COMM_WORLD, &swing_comm, 2); #ifdef DEBUG // TODO: REMOVE int size; MPI_Comm_remote_size(swing_comm, &size); printf("leader after --- swing node count: %d\n", size); // #endif // TODO: Make a better way of adding mappings for coordinator nodes. // Use this bandaid to get off the ground for now. int job_num = next_job_num++; job_to_comms[job_num].swing = swing_comm; std::vector<uint32_t> temp_vec; temp_vec.push_back(0); temp_vec.push_back(0); temp_vec.push_back(1); temp_vec.push_back(1); job_to_swing[job_num] = temp_vec; // TODO: This is a simple place holder for cache node spawning, // will want to make this more flexible later. spawn_cache_nodes(job_num, &swing_comm, 4); // TODO: Make a better way of adding mappings for team nodes. // Use this bandaid to get off the ground for now. temp_vec.clear(); temp_vec.push_back(0); temp_vec.push_back(1); temp_vec.push_back(2); temp_vec.push_back(3); job_to_cache[job_num] = temp_vec; // TODO: This is a simple place holder for job node spawning, // will want to make this more flexible later. spawn_job_nodes(job_num, job_exec, &swing_comm, 4); }
/** * Main loop of the function. */ int main(int argc, char **argv) { int rank, size; MPI_Comm parent; MPI_Init(&argc, &argv); MPI_Comm_get_parent(&parent); if (parent == MPI_COMM_NULL) error("No parent."); MPI_Comm_remote_size(parent, &size); if (size != 1) error("Something wrong with parent."); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); printf("I am a worker number %d of %d.\n", rank, size); MPI_Finalize(); return 0; }
int ompi_coll_libnbc_ireduce_scatter_inter(void* sendbuf, void* recvbuf, int *recvcounts, MPI_Datatype datatype, MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, struct mca_coll_base_module_2_0_0_t *module) { int peer, rank, r, res, count, rsize, offset; MPI_Aint ext; NBC_Schedule *schedule; NBC_Handle *handle; ompi_coll_libnbc_request_t **coll_req = (ompi_coll_libnbc_request_t**) request; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; res = NBC_Init_handle(comm, coll_req, libnbc_module); if(res != NBC_OK) { printf("Error in NBC_Init_handle(%i)\n", res); return res; } handle = (*coll_req); res = MPI_Comm_rank(comm, &rank); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_rank() (%i)\n", res); return res; } res = MPI_Comm_remote_size(comm, &rsize); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Comm_remote_size() (%i)\n", res); return res; } MPI_Type_extent(datatype, &ext); if (MPI_SUCCESS != res) { printf("MPI Error in MPI_Type_extent() (%i)\n", res); return res; } schedule = (NBC_Schedule*)malloc(sizeof(NBC_Schedule)); if (NULL == schedule) { printf("Error in malloc()\n"); return NBC_OOR; } res = NBC_Sched_create(schedule); if(res != NBC_OK) { printf("Error in NBC_Sched_create (%i)\n", res); return res; } count = 0; for (r = 0 ; r < rsize ; ++r) count += recvcounts[r]; handle->tmpbuf = malloc(2 * ext * count); if(handle->tmpbuf == NULL) { printf("Error in malloc()\n"); return NBC_OOR; } /* send my data to the remote root */ res = NBC_Sched_send(sendbuf, false, count, datatype, 0, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } if (0 == rank) { res = NBC_Sched_recv((void *) 0, true, count, datatype, 0, schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } res = NBC_Sched_barrier(schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; } for (peer = 1 ; peer < rsize ; ++peer) { res = NBC_Sched_recv((void *)(ext * count), true, count, datatype, peer, schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } res = NBC_Sched_barrier(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; } res = NBC_Sched_op((void *) 0, true, (void *)(ext * count), true, (void *) 0, true, count, datatype, op, schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_op() (%i)\n", res); return res; } res = NBC_Sched_barrier(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; } } /* exchange data with remote root for scatter phase (we *could* use the local communicator to do the scatter) */ res = NBC_Sched_recv((void *)(ext * count), true, count, datatype, 0, schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } res = NBC_Sched_send((void *) 0, true, count, datatype, 0, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } res = NBC_Sched_barrier(schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_barrier() (%i)\n", res); return res; } /* scatter */ for (peer = 0, offset = ext * count ; peer < rsize ; ++peer) { res = NBC_Sched_send((void *)(uintptr_t) offset, true, recvcounts[peer], datatype, peer, schedule); if (NBC_OK != res) { printf("Error in NBC_Sched_send() (%i)\n", res); return res; } offset += recvcounts[peer] * ext; } } /* receive my block */ res = NBC_Sched_recv(recvbuf, false, recvcounts[rank], datatype, 0, schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_recv() (%i)\n", res); return res; } /*NBC_PRINT_SCHED(*schedule);*/ res = NBC_Sched_commit(schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Sched_commit() (%i)\n", res); return res; } res = NBC_Start(handle, schedule); if (NBC_OK != res) { free(handle->tmpbuf); printf("Error in NBC_Start() (%i)\n", res); return res; } /* tmpbuf is freed with the handle */ return NBC_OK; }
int main( int argc, char *argv[] ) { int errs = 0, err; int *sendbuf = 0, *recvbuf = 0; int leftGroup, i, j, idx, count, rrank, rsize; MPI_Comm comm; MPI_Datatype datatype; MTest_Init( &argc, &argv ); datatype = MPI_INT; while (MTestGetIntercomm( &comm, &leftGroup, 4 )) { if (comm == MPI_COMM_NULL) continue; for (count = 1; count < 66000; count = 2 * count) { /* Get an intercommunicator */ MPI_Comm_remote_size( comm, &rsize ); MPI_Comm_rank( comm, &rrank ); sendbuf = (int *)malloc( rsize * count * sizeof(int) ); recvbuf = (int *)malloc( rsize * count * sizeof(int) ); for (i=0; i<rsize*count; i++) recvbuf[i] = -1; if (leftGroup) { idx = 0; for (j=0; j<rsize; j++) { for (i=0; i<count; i++) { sendbuf[idx++] = i + rrank; } } err = MPI_Alltoall( sendbuf, count, datatype, NULL, 0, datatype, comm ); if (err) { errs++; MTestPrintError( err ); } } else { int rank, size; MPI_Comm_rank( comm, &rank ); MPI_Comm_size( comm, &size ); /* In the right group */ err = MPI_Alltoall( NULL, 0, datatype, recvbuf, count, datatype, comm ); if (err) { errs++; MTestPrintError( err ); } /* Check that we have received the correct data */ idx = 0; for (j=0; j<rsize; j++) { for (i=0; i<count; i++) { if (recvbuf[idx++] != i + j) { errs++; if (errs < 10) fprintf( stderr, "buf[%d] = %d on %d\n", i, recvbuf[i], rank ); } } } } free( recvbuf ); free( sendbuf ); } MTestFreeComm( &comm ); } MTest_Finalize( errs ); MPI_Finalize(); return 0; }
int main( int argc, char *argv[] ) { int errs = 0, err; int rank, size, rsize, i; int np = 2; int errcodes[2]; MPI_Comm parentcomm, intercomm, intracomm, intracomm2, intracomm3; int isChild = 0; MPI_Status status; MTest_Init( &argc, &argv ); MPI_Comm_get_parent( &parentcomm ); if (parentcomm == MPI_COMM_NULL) { /* Create 2 more processes */ MPI_Comm_spawn( (char*)"./spawnintra", MPI_ARGV_NULL, np, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm, errcodes ); } else intercomm = parentcomm; /* We now have a valid intercomm */ MPI_Comm_remote_size( intercomm, &rsize ); MPI_Comm_size( intercomm, &size ); MPI_Comm_rank( intercomm, &rank ); if (parentcomm == MPI_COMM_NULL) { /* Master */ if (rsize != np) { errs++; printf( "Did not create %d processes (got %d)\n", np, rsize ); } if (rank == 0) { for (i=0; i<rsize; i++) { MPI_Send( &i, 1, MPI_INT, i, 0, intercomm ); } } } else { /* Child */ isChild = 1; if (size != np) { errs++; printf( "(Child) Did not create %d processes (got %d)\n", np, size ); } MPI_Recv( &i, 1, MPI_INT, 0, 0, intercomm, &status ); if (i != rank) { errs++; printf( "Unexpected rank on child %d (%d)\n", rank, i ); } } /* At this point, try to form the intracommunicator */ MPI_Intercomm_merge( intercomm, isChild, &intracomm ); /* Check on the intra comm */ { int icsize, icrank, wrank; MPI_Comm_size( intracomm, &icsize ); MPI_Comm_rank( intracomm, &icrank ); MPI_Comm_rank( MPI_COMM_WORLD, &wrank ); if (icsize != rsize + size) { errs++; printf( "Intracomm rank %d thinks size is %d, not %d\n", icrank, icsize, rsize + size ); } /* Make sure that the processes are ordered correctly */ if (isChild) { int psize; MPI_Comm_remote_size( parentcomm, &psize ); if (icrank != psize + wrank ) { errs++; printf( "Intracomm rank %d (from child) should have rank %d\n", icrank, psize + wrank ); } } else { if (icrank != wrank) { errs++; printf( "Intracomm rank %d (from parent) should have rank %d\n", icrank, wrank ); } } } /* At this point, try to form the intracommunicator, with the other processes first */ MPI_Intercomm_merge( intercomm, !isChild, &intracomm2 ); /* Check on the intra comm */ { int icsize, icrank, wrank; MPI_Comm_size( intracomm2, &icsize ); MPI_Comm_rank( intracomm2, &icrank ); MPI_Comm_rank( MPI_COMM_WORLD, &wrank ); if (icsize != rsize + size) { errs++; printf( "(2)Intracomm rank %d thinks size is %d, not %d\n", icrank, icsize, rsize + size ); } /* Make sure that the processes are ordered correctly */ if (isChild) { if (icrank != wrank ) { errs++; printf( "(2)Intracomm rank %d (from child) should have rank %d\n", icrank, wrank ); } } else { int csize; MPI_Comm_remote_size( intercomm, &csize ); if (icrank != wrank + csize) { errs++; printf( "(2)Intracomm rank %d (from parent) should have rank %d\n", icrank, wrank + csize ); } } } /* At this point, try to form the intracommunicator, with an arbitrary choice for the first group of processes */ MPI_Intercomm_merge( intercomm, 0, &intracomm3 ); /* Check on the intra comm */ { int icsize, icrank, wrank; MPI_Comm_size( intracomm3, &icsize ); MPI_Comm_rank( intracomm3, &icrank ); MPI_Comm_rank( MPI_COMM_WORLD, &wrank ); if (icsize != rsize + size) { errs++; printf( "(3)Intracomm rank %d thinks size is %d, not %d\n", icrank, icsize, rsize + size ); } /* Eventually, we should test that the processes are ordered correctly, by groups (must be one of the two cases above) */ } /* Update error count */ if (isChild) { /* Send the errs back to the master process */ MPI_Ssend( &errs, 1, MPI_INT, 0, 1, intercomm ); } else { if (rank == 0) { /* We could use intercomm reduce to get the errors from the children, but we'll use a simpler loop to make sure that we get valid data */ for (i=0; i<rsize; i++) { MPI_Recv( &err, 1, MPI_INT, i, 1, intercomm, MPI_STATUS_IGNORE ); errs += err; } } } /* It isn't necessary to free the intracomms, but it should not hurt */ MPI_Comm_free( &intracomm ); MPI_Comm_free( &intracomm2 ); MPI_Comm_free( &intracomm3 ); /* It isn't necessary to free the intercomm, but it should not hurt */ MPI_Comm_free( &intercomm ); /* Note that the MTest_Finalize get errs only over COMM_WORLD */ /* Note also that both the parent and child will generate "No Errors" if both call MTest_Finalize */ if (parentcomm == MPI_COMM_NULL) { MTest_Finalize( errs ); } MPI_Finalize(); return 0; }
/* create a loop between all the elements types */ int mpi_lsa_create_intercoms(com_lsa * com){ int prev, next,flag; int prev_size,next_size,size; /* create first connection between intracommunicators thanks to an intercommunicator */ /* one way */ MPI_Barrier(MPI_COMM_WORLD); if(com->rank_world==0)printf("]> Creating intercommunicators\n-One Way :\n"); MPI_Barrier(MPI_COMM_WORLD); printf("\t *> %d -> %d ",com->rank_world,com->master.com[4-((com->color_group)+1)]); MPI_Barrier(MPI_COMM_WORLD); MPI_Intercomm_create(com->com_group,0, MPI_COMM_WORLD,com->master.com[4-((com->color_group)+1)], com->rank_group, &(com->inter.com[4-((com->color_group)+1)])); MPI_Barrier(MPI_COMM_WORLD); if(com->rank_world==0)printf("\n]> The Other : \n"); MPI_Barrier(MPI_COMM_WORLD); printf("\t *> %d -> %d ",(com->color_group),(4-((com->color_group)-1)%4)%4); MPI_Barrier(MPI_COMM_WORLD); if(com->rank_world==0)printf("\n"); MPI_Barrier(MPI_COMM_WORLD); /* the other */ MPI_Intercomm_create(com->com_group,0, com->com_world,com->master.com[(4-((com->color_group)-1)%4)%4], com->rank_group, &(com->inter.com[(4-((com->color_group)-1)%4)%4])); /// WHY THIS ???????? if((4-((com->color_group)-1)%4)%4>com->color_group){ next=(4-(com->color_group-1)%4)%4; prev=4-((com->color_group)+1); } else { prev=(4-(com->color_group-1)%4)%4; next=4-((com->color_group)+1); } /* set the in and out communicators */ com->out_com=com->inter.com[next]; MPI_Comm_test_inter(com->inter.com[next],&flag); if(!flag){ mpi_lsa_print("\n\n\n\nproblem with inter.[next]\n\n\n\n\n", com); } com->in_com=com->inter.com[prev]; MPI_Comm_test_inter(com->inter.com[prev],&flag); if(!flag){ mpi_lsa_print("\n\n\n\n\nproblem with inter.[prev]\n\n\n\n", com); } MPI_Comm_remote_size(com->out_com,&next_size); MPI_Comm_remote_size(com->in_com,&prev_size); MPI_Comm_size(com->com_group,&size); if(com->color_group==0) printf("GMRES 1: my intercomm with LS %d \n",com->in_com); if(com->rank_world==0) printf("]> In and Out communicators : \n"); MPI_Barrier(MPI_COMM_WORLD); if(com->color_group==0) printf("GMRES : "); else if(com->color_group==1) printf("MAIN : "); else if(com->color_group==2) printf("ARNOLDI : "); else if(com->color_group==3) printf("LS : "); printf("%d: %d (%d) -> %d (%d) -> %d (%d) in_com: %d, out_com: %d\n",com->rank_world,com->master.com[prev],prev_size,com->color_group,size,com->master.com[next],next_size, com->in_com, com->out_com); return 0; }
int main( int argc, char *argv[] ) { int errs = 0; int size, isLeft; MPI_Comm intercomm, newcomm; MTest_Init( &argc, &argv ); MPI_Comm_size( MPI_COMM_WORLD, &size ); if (size < 4) { printf( "This test requires at least 4 processes\n" ); MPI_Abort( MPI_COMM_WORLD, 1 ); } while (MTestGetIntercomm( &intercomm, &isLeft, 2 )) { int key, color; if (intercomm == MPI_COMM_NULL) continue; /* Split this intercomm. The new intercomms contain the processes that had odd (resp even) rank in their local group in the original intercomm */ MTestPrintfMsg( 1, "Created intercomm %s\n", MTestGetIntercommName() ); MPI_Comm_rank( intercomm, &key ); color = (key % 2); MPI_Comm_split( intercomm, color, key, &newcomm ); /* Make sure that the new communicator has the appropriate pieces */ if (newcomm != MPI_COMM_NULL) { int orig_rsize, orig_size, new_rsize, new_size; int predicted_size, flag, commok=1; MPI_Comm_test_inter( intercomm, &flag ); if (!flag) { errs++; printf( "Output communicator is not an intercomm\n" ); commok = 0; } MPI_Comm_remote_size( intercomm, &orig_rsize ); MPI_Comm_remote_size( newcomm, &new_rsize ); MPI_Comm_size( intercomm, &orig_size ); MPI_Comm_size( newcomm, &new_size ); /* The local size is 1/2 the original size, +1 if the size was odd and the color was even. More precisely, let n be the orig_size. Then color 0 color 1 orig size even n/2 n/2 orig size odd (n+1)/2 n/2 However, since these are integer valued, if n is even, then (n+1)/2 = n/2, so this table is much simpler: color 0 color 1 orig size even (n+1)/2 n/2 orig size odd (n+1)/2 n/2 */ predicted_size = (orig_size + !color) / 2; if (predicted_size != new_size) { errs++; printf( "Predicted size = %d but found %d for %s (%d,%d)\n", predicted_size, new_size, MTestGetIntercommName(), orig_size, orig_rsize ); commok = 0; } predicted_size = (orig_rsize + !color) / 2; if (predicted_size != new_rsize) { errs++; printf( "Predicted remote size = %d but found %d for %s (%d,%d)\n", predicted_size, new_rsize, MTestGetIntercommName(), orig_size, orig_rsize ); commok = 0; } /* ... more to do */ if (commok) { errs += TestIntercomm( newcomm ); } } else { int orig_rsize; /* If the newcomm is null, then this means that remote group for this color is of size zero (since all processes in this test have been given colors other than MPI_UNDEFINED). Confirm that here */ /* FIXME: ToDo */ MPI_Comm_remote_size( intercomm, &orig_rsize ); if (orig_rsize == 1) { if (color == 0) { errs++; printf( "Returned null intercomm when non-null expected\n" ); } } } if (newcomm != MPI_COMM_NULL) MPI_Comm_free( &newcomm ); MPI_Comm_free( &intercomm ); } MTest_Finalize(errs); MPI_Finalize(); return 0; }
/* * Return an intercomm; set isLeftGroup to 1 if the calling process is * a member of the "left" group. */ int MTestGetIntercomm(MPI_Comm * comm, int *isLeftGroup, int min_size) { int size, rank, remsize, merr; int done = 0; MPI_Comm mcomm = MPI_COMM_NULL; MPI_Comm mcomm2 = MPI_COMM_NULL; int rleader; /* The while loop allows us to skip communicators that are too small. * MPI_COMM_NULL is always considered large enough. The size is * the sum of the sizes of the local and remote groups */ while (!done) { *comm = MPI_COMM_NULL; *isLeftGroup = 0; interCommName = "MPI_COMM_NULL"; switch (interCommIdx) { case 0: /* Split comm world in half */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD"; } else *comm = MPI_COMM_NULL; break; case 1: /* Split comm world in to 1 and the rest */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, rank == 0, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = 1; } else if (rank == 1) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank == 0; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12346, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD into 1, rest"; } else *comm = MPI_COMM_NULL; break; case 2: /* Split comm world in to 2 and the rest */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 3) { merr = MPI_Comm_split(MPI_COMM_WORLD, rank < 2, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = 2; } else if (rank == 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12347, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD into 2, rest"; } else *comm = MPI_COMM_NULL; break; case 3: /* Split comm world in half, then dup */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); /* avoid leaking after assignment below */ merr = MPI_Comm_free(&mcomm); if (merr) MTestPrintError(merr); /* now dup, some bugs only occur for dup's of intercomms */ mcomm = *comm; merr = MPI_Comm_dup(mcomm, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD then dup'ing"; } else *comm = MPI_COMM_NULL; break; case 4: /* Split comm world in half, form intercomm, then split that intercomm */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); /* avoid leaking after assignment below */ merr = MPI_Comm_free(&mcomm); if (merr) MTestPrintError(merr); /* now split, some bugs only occur for splits of intercomms */ mcomm = *comm; merr = MPI_Comm_rank(mcomm, &rank); if (merr) MTestPrintError(merr); /* this split is effectively a dup but tests the split code paths */ merr = MPI_Comm_split(mcomm, 0, rank, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD then then splitting again"; } else *comm = MPI_COMM_NULL; break; case 5: /* split comm world in half discarding rank 0 on the "left" * communicator, then form them into an intercommunicator */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size >= 4) { int color = (rank < size / 2 ? 0 : 1); if (rank == 0) color = MPI_UNDEFINED; merr = MPI_Comm_split(MPI_COMM_WORLD, color, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 1) { rleader = size / 2; } else if (rank == (size / 2)) { rleader = 1; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; if (rank != 0) { /* 0's mcomm is MPI_COMM_NULL */ merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); } interCommName = "Intercomm by splitting MPI_COMM_WORLD (discarding rank 0 in the left group) then MPI_Intercomm_create'ing"; } else { *comm = MPI_COMM_NULL; } break; case 6: /* Split comm world in half then form them into an * intercommunicator. Then discard rank 0 from each group of the * intercomm via MPI_Comm_create. */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size >= 4) { MPI_Group oldgroup, newgroup; int ranks[1]; int color = (rank < size / 2 ? 0 : 1); merr = MPI_Comm_split(MPI_COMM_WORLD, color, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == (size / 2)) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, &mcomm2); if (merr) MTestPrintError(merr); /* We have an intercomm between the two halves of comm world. Now create * a new intercomm that removes rank 0 on each side. */ merr = MPI_Comm_group(mcomm2, &oldgroup); if (merr) MTestPrintError(merr); ranks[0] = 0; merr = MPI_Group_excl(oldgroup, 1, ranks, &newgroup); if (merr) MTestPrintError(merr); merr = MPI_Comm_create(mcomm2, newgroup, comm); if (merr) MTestPrintError(merr); merr = MPI_Group_free(&oldgroup); if (merr) MTestPrintError(merr); merr = MPI_Group_free(&newgroup); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD then discarding 0 ranks with MPI_Comm_create"; } else { *comm = MPI_COMM_NULL; } break; default: *comm = MPI_COMM_NULL; interCommIdx = -1; break; } if (*comm != MPI_COMM_NULL) { merr = MPI_Comm_size(*comm, &size); if (merr) MTestPrintError(merr); merr = MPI_Comm_remote_size(*comm, &remsize); if (merr) MTestPrintError(merr); if (size + remsize >= min_size) done = 1; } else { interCommName = "MPI_COMM_NULL"; done = 1; } /* we are only done if all processes are done */ MPI_Allreduce(MPI_IN_PLACE, &done, 1, MPI_INT, MPI_LAND, MPI_COMM_WORLD); /* Advance the comm index whether we are done or not, otherwise we could * spin forever trying to allocate a too-small communicator over and * over again. */ interCommIdx++; if (!done && *comm != MPI_COMM_NULL) { /* avoid leaking communicators */ merr = MPI_Comm_free(comm); if (merr) MTestPrintError(merr); } /* cleanup for common temp objects */ if (mcomm != MPI_COMM_NULL) { merr = MPI_Comm_free(&mcomm); if (merr) MTestPrintError(merr); } if (mcomm2 != MPI_COMM_NULL) { merr = MPI_Comm_free(&mcomm2); if (merr) MTestPrintError(merr); } } return interCommIdx; }
int main( int argc, char *argv[] ) { int errs = 0, err; int *buf = 0; int *recvcounts; int *recvdispls; int leftGroup, i, count, rank, rsize, size; MPI_Comm comm; MPI_Datatype datatype; MTest_Init( &argc, &argv ); datatype = MPI_INT; while (MTestGetIntercomm( &comm, &leftGroup, 4 )) { if (comm == MPI_COMM_NULL) continue; MPI_Comm_rank( comm, &rank ); MPI_Comm_remote_size( comm, &rsize ); MPI_Comm_size( comm, &size ); /* To improve reporting of problems about operations, we change the error handler to errors return */ MPI_Comm_set_errhandler( comm, MPI_ERRORS_RETURN ); for (count = 1; count < 65000; count = 2 * count) { /* Get an intercommunicator */ recvcounts = (int *)malloc( rsize * sizeof(int) ); recvdispls = (int *)malloc( rsize * sizeof(int) ); /* This simple test duplicates the Gather test, using the same lengths for all messages */ for (i=0; i<rsize; i++) { recvcounts[i] = count; recvdispls[i] = count * i; } if (leftGroup) { buf = (int *)malloc( count * rsize * sizeof(int) ); for (i=0; i<count*rsize; i++) buf[i] = -1; err = MPI_Gatherv( NULL, 0, datatype, buf, recvcounts, recvdispls, datatype, (rank == 0) ? MPI_ROOT : MPI_PROC_NULL, comm ); if (err) { errs++; MTestPrintError( err ); } /* Test that no other process in this group received the broadcast */ if (rank != 0) { for (i=0; i<count; i++) { if (buf[i] != -1) { errs++; } } } else { /* Check for the correct data */ for (i=0; i<count*rsize; i++) { if (buf[i] != i) { errs++; } } } } else { /* In the right group */ buf = (int *)malloc( count * sizeof(int) ); for (i=0; i<count; i++) buf[i] = rank * count + i; err = MPI_Gatherv( buf, count, datatype, NULL, 0, 0, datatype, 0, comm ); if (err) { errs++; MTestPrintError( err ); } } free( buf ); free( recvcounts ); free( recvdispls ); } MTestFreeComm( &comm ); } MTest_Finalize( errs ); MPI_Finalize(); return 0; }
int main( int argc, char **argv ) { int err = 0; int *recvcounts; int size, rsize, rank, i; int recvcount, /* Each process receives this much data */ sendcount, /* Each process contributes this much data */ basecount; /* Unit of elements - basecount *rsize is recvcount, etc. */ int isLeftGroup; long long *sendbuf, *recvbuf; long long sumval; MPI_Comm comm; MTest_Init( &argc, &argv ); comm = MPI_COMM_WORLD; basecount = 1024; while (MTestGetIntercomm( &comm, &isLeftGroup, 2 )) { if (comm == MPI_COMM_NULL) continue; MPI_Comm_remote_size( comm, &rsize ); MPI_Comm_size( comm, &size ); MPI_Comm_rank( comm, &rank ); if (0) { printf( "[%d] %s (%d,%d) remote %d\n", rank, isLeftGroup ? "L" : "R", rank, size, rsize ); } recvcount = basecount * rsize; sendcount = basecount * rsize * size; recvcounts = (int *)malloc( size * sizeof(int) ); if (!recvcounts) { fprintf( stderr, "Could not allocate %d int for recvcounts\n", size ); MPI_Abort( MPI_COMM_WORLD, 1 ); } for (i=0; i<size; i++) recvcounts[i] = recvcount; sendbuf = (long long *) malloc( sendcount * sizeof(long long) ); if (!sendbuf) { fprintf( stderr, "Could not allocate %d ints for sendbuf\n", sendcount ); MPI_Abort( MPI_COMM_WORLD, 1 ); } for (i=0; i<sendcount; i++) { sendbuf[i] = (long long)(rank*sendcount + i); } recvbuf = (long long *)malloc( recvcount * sizeof(long long) ); if (!recvbuf) { fprintf( stderr, "Could not allocate %d ints for recvbuf\n", recvcount ); MPI_Abort( MPI_COMM_WORLD, 1 ); } for (i=0; i<recvcount; i++) { recvbuf[i] = (long long)(-i); } MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_LONG_LONG, MPI_SUM, comm ); /* Check received data */ for (i=0; i<recvcount; i++) { sumval = (long long)(sendcount) * (long long)((rsize * (rsize-1))/2) + (long long)(i + rank * rsize * basecount) * (long long)rsize; if (recvbuf[i] != sumval) { err++; if (err < 4) { fprintf( stdout, "Did not get expected value for reduce scatter\n" ); fprintf( stdout, "[%d] %s recvbuf[%d] = %lld, expected %lld\n", rank, isLeftGroup ? "L" : "R", i, recvbuf[i], sumval ); } } } free(sendbuf); free(recvbuf); free(recvcounts); MTestFreeComm( &comm ); } MTest_Finalize( err ); MPI_Finalize( ); return 0; }
int main(int argc, char *argv[]) { int errs = 0; int rank, size, rsize; int np = 3; MPI_Comm parentcomm, intercomm; int verbose = 0; char *env; int can_spawn; env = getenv("MPITEST_VERBOSE"); if (env) { if (*env != '0') verbose = 1; } MTest_Init(&argc, &argv); errs += MTestSpawnPossible(&can_spawn); if (can_spawn) { MPI_Comm_get_parent(&parentcomm); if (parentcomm == MPI_COMM_NULL) { IF_VERBOSE(("spawning %d processes\n", np)); /* Create 3 more processes */ MPI_Comm_spawn((char *) "./disconnect", MPI_ARGV_NULL, np, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm, MPI_ERRCODES_IGNORE); } else { intercomm = parentcomm; } /* We now have a valid intercomm */ MPI_Comm_remote_size(intercomm, &rsize); MPI_Comm_size(intercomm, &size); MPI_Comm_rank(intercomm, &rank); if (parentcomm == MPI_COMM_NULL) { IF_VERBOSE(("parent rank %d alive.\n", rank)); /* Parent */ if (rsize != np) { errs++; printf("Did not create %d processes (got %d)\n", np, rsize); fflush(stdout); } IF_VERBOSE(("disconnecting child communicator\n")); MPI_Comm_disconnect(&intercomm); /* Errors cannot be sent back to the parent because there is no * communicator connected to the children * for (i=0; i<rsize; i++) * { * MPI_Recv(&err, 1, MPI_INT, i, 1, intercomm, MPI_STATUS_IGNORE); * errs += err; * } */ } else { IF_VERBOSE(("child rank %d alive.\n", rank)); /* Child */ if (size != np) { errs++; printf("(Child) Did not create %d processes (got %d)\n", np, size); fflush(stdout); } IF_VERBOSE(("disconnecting communicator\n")); MPI_Comm_disconnect(&intercomm); /* Send the errs back to the master process */ /* Errors cannot be sent back to the parent because there is no * communicator connected to the parent */ /*MPI_Ssend(&errs, 1, MPI_INT, 0, 1, intercomm); */ } /* Note that the MTest_Finalize get errs only over COMM_WORLD */ /* Note also that both the parent and child will generate "No Errors" * if both call MTest_Finalize */ if (parentcomm == MPI_COMM_NULL) { MTest_Finalize(errs); } else { MPI_Finalize(); } } else { MTest_Finalize(errs); } IF_VERBOSE(("calling finalize\n")); return MTestReturnValue(errs); }
/* FIXME: This is copied from iccreate. It should be in one place */ int TestIntercomm( MPI_Comm comm ) { int local_size, remote_size, rank, **bufs, *bufmem, rbuf[2], j; int errs = 0, wrank, nsize; char commname[MPI_MAX_OBJECT_NAME+1]; MPI_Request *reqs; MPI_Comm_rank( MPI_COMM_WORLD, &wrank ); MPI_Comm_size( comm, &local_size ); MPI_Comm_remote_size( comm, &remote_size ); MPI_Comm_rank( comm, &rank ); MPI_Comm_get_name( comm, commname, &nsize ); MTestPrintfMsg( 1, "Testing communication on intercomm %s\n", commname ); reqs = (MPI_Request *)malloc( remote_size * sizeof(MPI_Request) ); if (!reqs) { printf( "[%d] Unable to allocated %d requests for testing intercomm %s\n", wrank, remote_size, commname ); errs++; return errs; } bufs = (int **) malloc( remote_size * sizeof(int *) ); if (!bufs) { printf( "[%d] Unable to allocated %d int pointers for testing intercomm %s\n", wrank, remote_size, commname ); errs++; return errs; } bufmem = (int *) malloc( remote_size * 2 * sizeof(int) ); if (!bufmem) { printf( "[%d] Unable to allocated %d int data for testing intercomm %s\n", wrank, 2*remote_size, commname ); errs++; return errs; } /* Each process sends a message containing its own rank and the rank of the destination with a nonblocking send. Because we're using nonblocking sends, we need to use different buffers for each isend */ for (j=0; j<remote_size; j++) { bufs[j] = &bufmem[2*j]; bufs[j][0] = rank; bufs[j][1] = j; MPI_Isend( bufs[j], 2, MPI_INT, j, 0, comm, &reqs[j] ); } for (j=0; j<remote_size; j++) { MPI_Recv( rbuf, 2, MPI_INT, j, 0, comm, MPI_STATUS_IGNORE ); if (rbuf[0] != j) { printf( "[%d] Expected rank %d but saw %d in %s\n", wrank, j, rbuf[0], commname ); errs++; } if (rbuf[1] != rank) { printf( "[%d] Expected target rank %d but saw %d from %d in %s\n", wrank, rank, rbuf[1], j, commname ); errs++; } } if (errs) fflush(stdout); MPI_Waitall( remote_size, reqs, MPI_STATUSES_IGNORE ); free( reqs ); free( bufs ); free( bufmem ); return errs; }