int parclient ( int me, int nproc, char *processor_name, int clients, int servers, int file_size, int a_size, int fs ) { double start_time; double used_time; double us_rate; int j ; MPI_Comm world, clients_world; MPI_Group world_group, client_group; long ranks[32]; printf("Client %d; total %d is alive on %s\n",me,nproc,processor_name); start_time = MPI_Wtime(); read_data(servers, clients, file_size, a_size); used_time = (MPI_Wtime() - start_time); us_rate = (double)(( file_size )/ (used_time*(double)1000000)); printf("FS=%d KB, time= %f sec., rate=%f Mbytes/sec\n", fs*KB, used_time, us_rate); world = MPI_COMM_WORLD; MPI_Comm_group(world, &world_group); for (j = 0; j < servers; j++) ranks[j] = j; MPI_Group_excl(world_group, servers, ranks, &client_group); MPI_Comm_create(world, client_group, &clients_world); MPI_Barrier(clients_world); printf("Client %d finished\n", me); return 1 ; }
static void serverWinCreate(void) { int ranks[1], modelID; MPI_Comm commCalc = commInqCommCalc (); MPI_Group groupCalc; int nProcsModel = commInqNProcsModel (); MPI_Info no_locks_info; xmpi(MPI_Info_create(&no_locks_info)); xmpi(MPI_Info_set(no_locks_info, "no_locks", "true")); xmpi(MPI_Win_create(MPI_BOTTOM, 0, 1, no_locks_info, commCalc, &getWin)); /* target group */ ranks[0] = nProcsModel; xmpi ( MPI_Comm_group ( commCalc, &groupCalc )); xmpi ( MPI_Group_excl ( groupCalc, 1, ranks, &groupModel )); rxWin = xcalloc((size_t)nProcsModel, sizeof (rxWin[0])); size_t totalBufferSize = collDefBufferSizes(); rxWin[0].buffer = (unsigned char*) xmalloc(totalBufferSize); size_t ofs = 0; for ( modelID = 1; modelID < nProcsModel; modelID++ ) { ofs += rxWin[modelID - 1].size; rxWin[modelID].buffer = rxWin[0].buffer + ofs; } xmpi(MPI_Info_free(&no_locks_info)); xdebug("%s", "created mpi_win, allocated getBuffer"); }
int main (int argc, char **argv) { int num, i, rank, localRank; MPI_Group all, odd, even; MPI_Comm oddComm, evenComm; char mess[11]; MPI_Init (&argc, &argv); // copy all the processes in group "all" MPI_Comm_group (MPI_COMM_WORLD, &all); MPI_Comm_size (MPI_COMM_WORLD, &num); MPI_Comm_rank (MPI_COMM_WORLD, &rank); int grN = 0; int ranks[num / 2]; for (i = 0; i < num; i += 2) ranks[grN++] = i; // extract from "all" only the odd ones MPI_Group_excl (all, grN, ranks, &odd); // sutract odd group from all to get the even ones MPI_Group_difference (all, odd, &even); MPI_Comm_create (MPI_COMM_WORLD, odd, &oddComm); MPI_Comm_create (MPI_COMM_WORLD, even, &evenComm); // check group membership MPI_Group_rank (odd, &localRank); if (localRank != MPI_UNDEFINED) { if (localRank == 0) // local group root, sets-up message strcpy (mess, "ODD GROUP"); MPI_Bcast (mess, 11, MPI_CHAR, 0, oddComm); MPI_Comm_free (&oddComm); // free communicator in processes where it is valid } else { MPI_Comm_rank (evenComm, &localRank); if (localRank == 0) // local group root, sets-up message strcpy (mess, "EVEN GROUP"); MPI_Bcast (mess, 11, MPI_CHAR, 0, evenComm); MPI_Comm_free (&evenComm); } printf ("Process %i with local rank %i received %s\n", rank, localRank, mess); // free up memory MPI_Group_free (&all); MPI_Group_free (&odd); MPI_Group_free (&even); MPI_Finalize (); return 0; }
value caml_mpi_group_excl(value group, value vranks) { MPI_Group newgroup; int n = Wosize_val(vranks); int * ranks = stat_alloc(n * sizeof(int)); int i; for (i = 0; i < n; i++) ranks[i] = Int_val(Field(vranks, i)); MPI_Group_excl(Group_val(group), n, ranks, &newgroup); stat_free(ranks); return caml_mpi_alloc_group(newgroup); }
JNIEXPORT jlong JNICALL Java_mpi_Group_excl( JNIEnv *env, jobject jthis, jlong group, jintArray ranks) { jsize n = (*env)->GetArrayLength(env, ranks); jint *jRanks; int *cRanks; ompi_java_getIntArray(env, ranks, &jRanks, &cRanks); MPI_Group newGroup; int rc = MPI_Group_excl((MPI_Group)group, n, cRanks, &newGroup); ompi_java_exceptionCheck(env, rc); ompi_java_forgetIntArray(env, ranks, jRanks, cRanks); return (jlong)newGroup; }
int main(int argc, char *argv[]) { int size, rank, i, *excl; MPI_Group world_group, even_group; MPI_Comm __attribute__((unused)) even_comm; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (size % 2) { fprintf(stderr, "this program requires a multiple of 2 number of processes\n"); MPI_Abort(MPI_COMM_WORLD, 1); exit(1); } excl = malloc((size / 2) * sizeof(int)); assert(excl); /* exclude the odd ranks */ for (i = 0; i < size / 2; i++) excl[i] = (2 * i) + 1; /* Create some groups */ MPI_Comm_group(MPI_COMM_WORLD, &world_group); MPI_Group_excl(world_group, size / 2, excl, &even_group); MPI_Group_free(&world_group); #if !defined(USE_STRICT_MPI) && defined(MPICH) if (rank % 2 == 0) { /* Even processes create a group for themselves */ MPI_Comm_create_group(MPI_COMM_WORLD, even_group, 0, &even_comm); MPI_Barrier(even_comm); MPI_Comm_free(&even_comm); } #endif /* USE_STRICT_MPI */ MPI_Group_free(&even_group); MPI_Barrier(MPI_COMM_WORLD); if (rank == 0) printf(" No errors\n"); MPI_Finalize(); return 0; }
/* * Class: mpi_Group * Method: excl * Signature: ([I)J */ JNIEXPORT jlong JNICALL Java_mpi_Group_excl(JNIEnv *env, jobject jthis, jintArray ranks) { int n; jint *rks; jboolean isCopy=JNI_TRUE; MPI_Group newgroup; ompi_java_clearFreeList(env) ; n=(*env)->GetArrayLength(env,ranks); rks=(*env)->GetIntArrayElements(env,ranks,&isCopy); MPI_Group_excl((MPI_Group)((*env)->GetLongField(env,jthis,ompi_java.GrouphandleID)), n, (int*)rks, &newgroup); (*env)->ReleaseIntArrayElements(env,ranks,rks,0); return (jlong)newgroup; }
int main (int argc, char **argv) { int num, i, rank; MPI_Group all, odd, even; MPI_Init (&argc, &argv); // copy all the processes in group "all" MPI_Comm_group (MPI_COMM_WORLD, &all); MPI_Comm_size (MPI_COMM_WORLD, &num); MPI_Comm_rank (MPI_COMM_WORLD, &rank); int grN = 0; int ranks[num / 2]; for (i = 0; i < num; i += 2) ranks[grN++] = i; // extract from "all" only the odd ones MPI_Group_excl (all, grN, ranks, &odd); // sutract odd group from all to get the even ones MPI_Group_difference (all, odd, &even); // print group sizes if (rank == 0) { MPI_Group_size (odd, &i); printf ("Odd group has %i processes\n", i); MPI_Group_size (even, &i); printf ("Even group has %i processes\n", i); } // check group membership MPI_Group_rank (odd, &i); if (i == MPI_UNDEFINED) printf ("Process %i belongs to even group\n", rank); else printf ("Process %i belongs to odd group\n", rank); // free up memory MPI_Group_free (&all); MPI_Group_free (&odd); MPI_Group_free (&even); MPI_Finalize (); return 0; }
/** * this is based on the MPI Forum decision that MPI_COMM_WORLD is a C constant */ void make_tcgmsg_comm() { extern int single_cluster(); #ifdef NXTVAL_SERVER if( SR_parallel ){ /* data server for a single process */ int server; MPI_Group MPI_GROUP_WORLD, tcgmsg_grp; MPI_Comm_size(MPI_COMM_WORLD, &server); server --; /* the highest numbered process will be excluded */ MPI_Comm_group(MPI_COMM_WORLD, &MPI_GROUP_WORLD); MPI_Group_excl(MPI_GROUP_WORLD, 1, &server, &tcgmsg_grp); MPI_Comm_create(MPI_COMM_WORLD, tcgmsg_grp, &TCGMSG_Comm); }else #endif TCGMSG_Comm = MPI_COMM_WORLD; }
static VALUE group_excl(VALUE self, VALUE ary) { int rv, i, len, *ranks; MPI_Group *grp, *newgrp; Data_Get_Struct(self, MPI_Group, grp); newgrp = ALLOC(MPI_Group); len = RARRAY(ary)->len; ranks = ALLOCA_N(int, len); for (i = 0; i < len; i++) ranks[i] = FIX2INT(rb_ary_entry(ary, i)); rv = MPI_Group_excl(*grp, len, ranks, newgrp); mpi_exception(rv); return group_new(newgrp); }
void ompi_group_excl_f(MPI_Fint *group, MPI_Fint *n, MPI_Fint *ranks, MPI_Fint *newgroup, MPI_Fint *ierr) { int c_ierr; ompi_group_t *c_group, *c_newgroup; OMPI_ARRAY_NAME_DECL(ranks); /* Make the fortran to c representation conversion */ c_group = MPI_Group_f2c(*group); OMPI_ARRAY_FINT_2_INT(ranks, *n); c_ierr = MPI_Group_excl(c_group, OMPI_FINT_2_INT(*n), OMPI_ARRAY_NAME_CONVERT(ranks), &c_newgroup); if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr); /* translate the results from c to fortran */ if (MPI_SUCCESS == c_ierr) { *newgroup = c_newgroup->grp_f_to_c_index; } }
/* * This test attempts collective communication after a process in * the communicator has failed. */ int main(int argc, char **argv) { int rank, size, i, rc, errclass, toterrs, errs = 0; char rbuf[100000]; char *sendbuf; int deadprocs[1] = {1}; MPI_Group world, newgroup; MPI_Comm newcomm; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); if (size < 3) { fprintf( stderr, "Must run with at least 3 processes\n" ); MPI_Abort( MPI_COMM_WORLD, 1 ); } if (rank == 1) { exit(EXIT_FAILURE); } /* try a small send first */ sendbuf = (char *)malloc(10*size*sizeof(char)); if (rank == 0) { for (i=0;i<size;i++) { strcpy(sendbuf + i*10, "No Errors"); } } rc = MPI_Scatter(sendbuf, 10, MPI_CHAR, rbuf, 10, MPI_CHAR, 0, MPI_COMM_WORLD); #if defined (MPICH) && (MPICH_NUMVERSION >= 30100102) MPI_Error_class(rc, &errclass); if ((rc) && (errclass != MPIX_ERR_PROC_FAILED)) { fprintf(stderr, "Wrong error code (%d) returned. Expected MPIX_ERR_PROC_FAILED\n", errclass); errs++; } #endif /* reset the buffers and try a larger scatter */ free(sendbuf); memset(rbuf, 0, sizeof(rbuf)); sendbuf = (char *)malloc(100000*size*sizeof(char)); if (rank == 0) { for (i=0;i<size;i++) { strcpy(sendbuf + i*100000, "No Errors"); } } rc = MPI_Scatter(sendbuf, 100000, MPI_CHAR, rbuf, 100000, MPI_CHAR, 0, MPI_COMM_WORLD); #if defined (MPICH) && (MPICH_NUMVERSION >= 30100102) MPI_Error_class(rc, &errclass); if ((rc) && (errclass != MPIX_ERR_PROC_FAILED)) { fprintf(stderr, "Wrong error code (%d) returned. Expected MPIX_ERR_PROC_FAILED\n", errclass); errs++; } #endif MPI_Comm_group(MPI_COMM_WORLD, &world); MPI_Group_excl(world, 1, deadprocs, &newgroup); MPI_Comm_create_group(MPI_COMM_WORLD, newgroup, 0, &newcomm); rc = MPI_Reduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, 0, newcomm); if(rc) fprintf(stderr, "Failed to get errors from other processes\n"); if (rank == 0) { if (toterrs) { printf( " Found %d errors\n", toterrs ); } else { printf( " No Errors\n" ); } fflush(stdout); } free(sendbuf); MPI_Finalize(); return 0; }
void declareBindings (void) { /* === Point-to-point === */ void* buf; int count; MPI_Datatype datatype; int dest; int tag; MPI_Comm comm; MPI_Send (buf, count, datatype, dest, tag, comm); // L12 int source; MPI_Status status; MPI_Recv (buf, count, datatype, source, tag, comm, &status); // L15 MPI_Get_count (&status, datatype, &count); MPI_Bsend (buf, count, datatype, dest, tag, comm); MPI_Ssend (buf, count, datatype, dest, tag, comm); MPI_Rsend (buf, count, datatype, dest, tag, comm); void* buffer; int size; MPI_Buffer_attach (buffer, size); // L22 MPI_Buffer_detach (buffer, &size); MPI_Request request; MPI_Isend (buf, count, datatype, dest, tag, comm, &request); // L25 MPI_Ibsend (buf, count, datatype, dest, tag, comm, &request); MPI_Issend (buf, count, datatype, dest, tag, comm, &request); MPI_Irsend (buf, count, datatype, dest, tag, comm, &request); MPI_Irecv (buf, count, datatype, source, tag, comm, &request); MPI_Wait (&request, &status); int flag; MPI_Test (&request, &flag, &status); // L32 MPI_Request_free (&request); MPI_Request* array_of_requests; int index; MPI_Waitany (count, array_of_requests, &index, &status); // L36 MPI_Testany (count, array_of_requests, &index, &flag, &status); MPI_Status* array_of_statuses; MPI_Waitall (count, array_of_requests, array_of_statuses); // L39 MPI_Testall (count, array_of_requests, &flag, array_of_statuses); int incount; int outcount; int* array_of_indices; MPI_Waitsome (incount, array_of_requests, &outcount, array_of_indices, array_of_statuses); // L44--45 MPI_Testsome (incount, array_of_requests, &outcount, array_of_indices, array_of_statuses); // L46--47 MPI_Iprobe (source, tag, comm, &flag, &status); // L48 MPI_Probe (source, tag, comm, &status); MPI_Cancel (&request); MPI_Test_cancelled (&status, &flag); MPI_Send_init (buf, count, datatype, dest, tag, comm, &request); MPI_Bsend_init (buf, count, datatype, dest, tag, comm, &request); MPI_Ssend_init (buf, count, datatype, dest, tag, comm, &request); MPI_Rsend_init (buf, count, datatype, dest, tag, comm, &request); MPI_Recv_init (buf, count, datatype, source, tag, comm, &request); MPI_Start (&request); MPI_Startall (count, array_of_requests); void* sendbuf; int sendcount; MPI_Datatype sendtype; int sendtag; void* recvbuf; int recvcount; MPI_Datatype recvtype; MPI_Datatype recvtag; MPI_Sendrecv (sendbuf, sendcount, sendtype, dest, sendtag, recvbuf, recvcount, recvtype, source, recvtag, comm, &status); // L67--69 MPI_Sendrecv_replace (buf, count, datatype, dest, sendtag, source, recvtag, comm, &status); // L70--71 MPI_Datatype oldtype; MPI_Datatype newtype; MPI_Type_contiguous (count, oldtype, &newtype); // L74 int blocklength; { int stride; MPI_Type_vector (count, blocklength, stride, oldtype, &newtype); // L78 } { MPI_Aint stride; MPI_Type_hvector (count, blocklength, stride, oldtype, &newtype); // L82 } int* array_of_blocklengths; { int* array_of_displacements; MPI_Type_indexed (count, array_of_blocklengths, array_of_displacements, oldtype, &newtype); // L87--88 } { MPI_Aint* array_of_displacements; MPI_Type_hindexed (count, array_of_blocklengths, array_of_displacements, oldtype, &newtype); // L92--93 MPI_Datatype* array_of_types; MPI_Type_struct (count, array_of_blocklengths, array_of_displacements, array_of_types, &newtype); // L95--96 } void* location; MPI_Aint address; MPI_Address (location, &address); // L100 MPI_Aint extent; MPI_Type_extent (datatype, &extent); // L102 MPI_Type_size (datatype, &size); MPI_Aint displacement; MPI_Type_lb (datatype, &displacement); // L105 MPI_Type_ub (datatype, &displacement); MPI_Type_commit (&datatype); MPI_Type_free (&datatype); MPI_Get_elements (&status, datatype, &count); void* inbuf; void* outbuf; int outsize; int position; MPI_Pack (inbuf, incount, datatype, outbuf, outsize, &position, comm); // L114 int insize; MPI_Unpack (inbuf, insize, &position, outbuf, outcount, datatype, comm); // L116--117 MPI_Pack_size (incount, datatype, comm, &size); /* === Collectives === */ MPI_Barrier (comm); // L121 int root; MPI_Bcast (buffer, count, datatype, root, comm); // L123 MPI_Gather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); // L124--125 int* recvcounts; int* displs; MPI_Gatherv (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm); // L128--130 MPI_Scatter (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); // L131--132 int* sendcounts; MPI_Scatterv (sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm); // L134--135 MPI_Allgather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); // L136--137 MPI_Allgatherv (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm); // L138--140 MPI_Alltoall (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); // L141--142 int* sdispls; int* rdispls; MPI_Alltoallv (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm); // L145--147 MPI_Op op; MPI_Reduce (sendbuf, recvbuf, count, datatype, op, root, comm); // L149 #if 0 MPI_User_function function; int commute; MPI_Op_create (function, commute, &op); // L153 #endif MPI_Op_free (&op); // L155 MPI_Allreduce (sendbuf, recvbuf, count, datatype, op, comm); MPI_Reduce_scatter (sendbuf, recvbuf, recvcounts, datatype, op, comm); MPI_Scan (sendbuf, recvbuf, count, datatype, op, comm); /* === Groups, contexts, and communicators === */ MPI_Group group; MPI_Group_size (group, &size); // L162 int rank; MPI_Group_rank (group, &rank); // L164 MPI_Group group1; int n; int* ranks1; MPI_Group group2; int* ranks2; MPI_Group_translate_ranks (group1, n, ranks1, group2, ranks2); // L170 int result; MPI_Group_compare (group1, group2, &result); // L172 MPI_Group newgroup; MPI_Group_union (group1, group2, &newgroup); // L174 MPI_Group_intersection (group1, group2, &newgroup); MPI_Group_difference (group1, group2, &newgroup); int* ranks; MPI_Group_incl (group, n, ranks, &newgroup); // L178 MPI_Group_excl (group, n, ranks, &newgroup); extern int ranges[][3]; MPI_Group_range_incl (group, n, ranges, &newgroup); // L181 MPI_Group_range_excl (group, n, ranges, &newgroup); MPI_Group_free (&group); MPI_Comm_size (comm, &size); MPI_Comm_rank (comm, &rank); MPI_Comm comm1; MPI_Comm comm2; MPI_Comm_compare (comm1, comm2, &result); MPI_Comm newcomm; MPI_Comm_dup (comm, &newcomm); MPI_Comm_create (comm, group, &newcomm); int color; int key; MPI_Comm_split (comm, color, key, &newcomm); // L194 MPI_Comm_free (&comm); MPI_Comm_test_inter (comm, &flag); MPI_Comm_remote_size (comm, &size); MPI_Comm_remote_group (comm, &group); MPI_Comm local_comm; int local_leader; MPI_Comm peer_comm; int remote_leader; MPI_Comm newintercomm; MPI_Intercomm_create (local_comm, local_leader, peer_comm, remote_leader, tag, &newintercomm); // L204--205 MPI_Comm intercomm; MPI_Comm newintracomm; int high; MPI_Intercomm_merge (intercomm, high, &newintracomm); // L209 int keyval; #if 0 MPI_Copy_function copy_fn; MPI_Delete_function delete_fn; void* extra_state; MPI_Keyval_create (copy_fn, delete_fn, &keyval, extra_state); // L215 #endif MPI_Keyval_free (&keyval); // L217 void* attribute_val; MPI_Attr_put (comm, keyval, attribute_val); // L219 MPI_Attr_get (comm, keyval, attribute_val, &flag); MPI_Attr_delete (comm, keyval); /* === Environmental inquiry === */ char* name; int resultlen; MPI_Get_processor_name (name, &resultlen); // L226 MPI_Errhandler errhandler; #if 0 MPI_Handler_function function; MPI_Errhandler_create (function, &errhandler); // L230 #endif MPI_Errhandler_set (comm, errhandler); // L232 MPI_Errhandler_get (comm, &errhandler); MPI_Errhandler_free (&errhandler); int errorcode; char* string; MPI_Error_string (errorcode, string, &resultlen); // L237 int errorclass; MPI_Error_class (errorcode, &errorclass); // L239 MPI_Wtime (); MPI_Wtick (); int argc; char** argv; MPI_Init (&argc, &argv); // L244 MPI_Finalize (); MPI_Initialized (&flag); MPI_Abort (comm, errorcode); }
//main program int main(int argc, char *argv[]) { MPI_Init(&argc, &argv); int size, rank; long t1, t2; static int ranks[1] = { 0 }; MPI_Request request1, request2, request3, request4; MPI_Status status, status1, status2, status3, status4; MPI_Group MPI_GROUP_WORLD, grprem; MPI_Comm commslave, newcomm; MPI_Comm commsp; MPI_Comm_group(MPI_COMM_WORLD, &MPI_GROUP_WORLD); MPI_Group_excl(MPI_GROUP_WORLD, 1, ranks, &grprem); MPI_Comm_create(MPI_COMM_WORLD, grprem, &commslave); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); printf("Node %d in %d is ready\n", rank, size); //Initialize double *grid0 = creat_grid(x_size, y_size, z_size, size, rank, GRID0); double *grid1 = creat_grid(x_size, y_size, z_size, size, rank, GRID1); MPI_Barrier(MPI_COMM_WORLD); if (size != 1) { if (rank == 0) { for (int i = 1; i < size; i++) { int len = (i == size - 1) ? (x_size / (size - 1)) : (x_size / (size - 1) + x_size % (size - 1)); MPI_Ssend(grid0 + (i - 1)*x_size / (size - 1), len*y_size*z_size, MPI_DOUBLE, i, i, MPI_COMM_WORLD); } } else { for (int i = 1; i < size; i++) { if (rank == i) { int len = (i == size - 1) ? (x_size / (size - 1)) : (x_size / (size - 1) + x_size % (size - 1)); MPI_Recv(grid0 + y_size*z_size, len*y_size*z_size, MPI_DOUBLE, 0, i, MPI_COMM_WORLD, &status); } } } } //Compute if (rank == 0) printf("Start computing...\n"); if (rank != 0 && size > 1) { for (int t = 0; t < stepnum; t++) { compute(grid0, grid1, rank, size); //send right slice of data to next node, then receieve right slice of data form next node if (rank < size - 1) { MPI_Ssend(grid1 + (1 + x_size / (size - 1))*y_size*z_size, y_size*z_size, MPI_DOUBLE, rank + 1, rank, MPI_COMM_WORLD); MPI_Recv(grid1 + (1 + x_size / (size - 1))*y_size*z_size, y_size*z_size, MPI_DOUBLE, rank + 1, rank + 1, MPI_COMM_WORLD, &status1); //MPI_Wait(&request1, &status1); //MPI_Wait(&request2, &status2); } //receieve left slice of data from perior node, then send left slice of data to perior node if (rank > 1) { MPI_Recv(grid1, y_size*z_size, MPI_DOUBLE, rank - 1, rank - 1, MPI_COMM_WORLD, &status2); MPI_Ssend(grid1, y_size*z_size, MPI_DOUBLE, rank - 1, rank, MPI_COMM_WORLD); //MPI_Wait(&request3, &status3); //MPI_Wait(&request4, &status4); } double *temp; temp = grid0; grid0 = grid1; grid1 = temp; MPI_Barrier(commslave); } } else if (size == 1) { for (int t = 0; t < stepnum; t++) { compute(grid0, grid1, rank, size); double *temp; temp = grid0; grid0 = grid1; grid1 = temp; } } else { ; } MPI_Barrier(MPI_COMM_WORLD); printf("Rank %d finished computing!\n", rank); //Gather data form nodes to host if (size != 1) { if (stepnum % 2) { double *temp; temp = grid0; grid0 = grid1; grid1 = temp; } for (int i = 1; i < size; i++) { if (rank == i) { int len = (i == size - 1) ? (x_size / (size - 1)) : (x_size / (size - 1) + x_size % (size - 1)); MPI_Ssend(grid0 + y_size*z_size, len*y_size*z_size, MPI_DOUBLE, 0, i, MPI_COMM_WORLD); //MPI_Wait(&request2, &status2); } } if (rank == 0) { for (int i = 1; i < size; i++) { int len = (i == size - 1) ? (x_size / (size - 1)) : (x_size / (size - 1) + x_size % (size - 1)); MPI_Recv(grid1, len*y_size*z_size, MPI_DOUBLE, i, i, MPI_COMM_WORLD, &status3); //MPI_Wait(&request1, &status1); } } } MPI_Barrier(MPI_COMM_WORLD); if (rank == 0) printf("All work complete\n"); MPI_Finalize(); return 0; }
int main (int argc, char *argv[]) { int iter; int in, out, i, ierr, ranks[1], done; double x, y, Pi, error, epsilon; double start_time, end_time; int numprocs, master, myrank, workerrank; int totalin, totalout; long max, total; int rands[CHUNKSIZE], request; MPI_Comm world, workers; // two communicators MPI_Group world_group, worker_group; // two groups MPI_Status status; if (MPI_Init(&argc, &argv) != MPI_SUCCESS) { fprintf(stderr, "MPI initialization error\n"); exit(-1); } // world is the MPI_COMM_WORLD communicator world = MPI_COMM_WORLD; // get the processors within world communicator MPI_Comm_size(world, &numprocs); // assign the rank for each processor in the communicator MPI_Comm_rank(world, &myrank); if (argc <= 1) { fprintf(stderr, "Usage: mpirun -np number_of_process ./mcpi number_of_epsilon number_of_iterations\n"); MPI_Finalize(); exit(-1); } // assign to master the rank of the last process in the world communicator master = numprocs - 1; // if it is the first process, read the input from console if(myrank == 0) { sscanf(argv[1], "%lf", &epsilon); //sscanf(argv[2], "%ld", &total); } // then send the epsilon to all the worker process MPI_Bcast(&epsilon, 1, MPI_DOUBLE, 0, world); //MPI_Bcast(&total, 1, MPI_INT, 0, MPI_COMM_WORLD); // create the world group associated with world communicator MPI_Comm_group(world, &world_group); ranks[0] = master; // create the worker group from world group, but exclude the master process MPI_Group_excl(world_group, 1, ranks, &worker_group); // create a new communicator from existing communicator and group // this is the communicator we will use when we wish to do collective operations // that don not involve the master process MPI_Comm_create(world, worker_group, &workers); // free the two groups, we don't need them any more, // left work will be handled by communicators MPI_Group_free(&worker_group); MPI_Group_free(&world_group); // record the start time start_time = MPI_Wtime(); // the master process is in charge of producing the random points, // then send the points to worker processes. if(myrank == master) { #if RANDOM_SEED struct timeval time; gettimeofday(&time, 0); // initialize the random number generator srandom((int)(time.tv_usec * 1000000 + time.tv_sec)); #endif do { // check whether the worker processes need new random points MPI_Recv(&request, 1, MPI_INT, MPI_ANY_SOURCE, REQUEST_TAG, world, &status); if(request) { for(i = 0; i < CHUNKSIZE;) { rands[i] = random(); if (rands[i] <= INT_MAX) i++; } MPI_Send(rands, CHUNKSIZE, MPI_INT, status.MPI_SOURCE, REPLY_TAG, world); } } while (request > 0); } // if it is a worker process else { // if request is 1, the worker process ask master process for random points request = 1; done = in = out = 0; max = (1 << 31) - 1; // max int, for normalization MPI_Send(&request, 1, MPI_INT, master, REQUEST_TAG, world); // get the rank of current process in worker communicator MPI_Comm_rank(workers, &workerrank); // iter = 0; while (!done) { iter++; request = 1; MPI_Recv(rands, CHUNKSIZE, MPI_INT, master, REPLY_TAG, world, &status); for (i = 0; i < CHUNKSIZE-1;) { #if QUARTER_CIRCLE x = ((double) rands[i++])/max; y = ((double) rands[i++])/max; #else // avoid x and y have the same number x = (((double) rands[i++])/max) * 2 - 1; y = (((double) rands[i++])/max) * 2 - 1; #endif // if the point is in the circle, increase in if(x*x + y*y < 1.0) { in++; } // if the point is out of the circle, increase out else { out++; } } // combine the result from each worker process, // and then send back to each worker process MPI_Allreduce(&in, &totalin, 1, MPI_INT, MPI_SUM, workers); MPI_Allreduce(&out, &totalout, 1, MPI_INT, MPI_SUM, workers); // if stop condition is not satisfied, // ask the master process to send points again if(myrank == 0) { Pi = (4.0 * totalin)/(totalin + totalout); error = fabs(Pi - M_PI); // if the error is acceptable or the total points is larger than 1000000, // stop the process done = (error < epsilon || (totalin + totalout) > TOTAL); request = (done) ? 0 : 1; MPI_Send(&request, 1, MPI_INT, master, REQUEST_TAG, world); MPI_Bcast(&done, 1, MPI_INT, 0, workers); } else { MPI_Bcast(&done, 1, MPI_INT, 0, workers); if(request) { MPI_Send(&request, 1, MPI_INT, master, REQUEST_TAG, world); } } } MPI_Comm_free(&workers); } end_time = MPI_Wtime(); if(myrank == 0) { printf("pi is: %23.20f\n", Pi); printf("\ntotal points: %d\nin: %d, out: %d, <ret> to exit\n", totalin+totalout, totalin, totalout); printf("It took %f seconds\n", end_time - start_time); getchar(); } // MPE_Close_graphics(&graph); MPI_Finalize(); return 0; }
/* * Return an intercomm; set isLeftGroup to 1 if the calling process is * a member of the "left" group. */ int MTestGetIntercomm(MPI_Comm * comm, int *isLeftGroup, int min_size) { int size, rank, remsize, merr; int done = 0; MPI_Comm mcomm = MPI_COMM_NULL; MPI_Comm mcomm2 = MPI_COMM_NULL; int rleader; /* The while loop allows us to skip communicators that are too small. * MPI_COMM_NULL is always considered large enough. The size is * the sum of the sizes of the local and remote groups */ while (!done) { *comm = MPI_COMM_NULL; *isLeftGroup = 0; interCommName = "MPI_COMM_NULL"; switch (interCommIdx) { case 0: /* Split comm world in half */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD"; } else *comm = MPI_COMM_NULL; break; case 1: /* Split comm world in to 1 and the rest */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, rank == 0, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = 1; } else if (rank == 1) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank == 0; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12346, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD into 1, rest"; } else *comm = MPI_COMM_NULL; break; case 2: /* Split comm world in to 2 and the rest */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 3) { merr = MPI_Comm_split(MPI_COMM_WORLD, rank < 2, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = 2; } else if (rank == 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12347, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD into 2, rest"; } else *comm = MPI_COMM_NULL; break; case 3: /* Split comm world in half, then dup */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); /* avoid leaking after assignment below */ merr = MPI_Comm_free(&mcomm); if (merr) MTestPrintError(merr); /* now dup, some bugs only occur for dup's of intercomms */ mcomm = *comm; merr = MPI_Comm_dup(mcomm, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD then dup'ing"; } else *comm = MPI_COMM_NULL; break; case 4: /* Split comm world in half, form intercomm, then split that intercomm */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); /* avoid leaking after assignment below */ merr = MPI_Comm_free(&mcomm); if (merr) MTestPrintError(merr); /* now split, some bugs only occur for splits of intercomms */ mcomm = *comm; merr = MPI_Comm_rank(mcomm, &rank); if (merr) MTestPrintError(merr); /* this split is effectively a dup but tests the split code paths */ merr = MPI_Comm_split(mcomm, 0, rank, comm); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD then then splitting again"; } else *comm = MPI_COMM_NULL; break; case 5: /* split comm world in half discarding rank 0 on the "left" * communicator, then form them into an intercommunicator */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size >= 4) { int color = (rank < size / 2 ? 0 : 1); if (rank == 0) color = MPI_UNDEFINED; merr = MPI_Comm_split(MPI_COMM_WORLD, color, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 1) { rleader = size / 2; } else if (rank == (size / 2)) { rleader = 1; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; if (rank != 0) { /* 0's mcomm is MPI_COMM_NULL */ merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, comm); if (merr) MTestPrintError(merr); } interCommName = "Intercomm by splitting MPI_COMM_WORLD (discarding rank 0 in the left group) then MPI_Intercomm_create'ing"; } else { *comm = MPI_COMM_NULL; } break; case 6: /* Split comm world in half then form them into an * intercommunicator. Then discard rank 0 from each group of the * intercomm via MPI_Comm_create. */ merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); if (size >= 4) { MPI_Group oldgroup, newgroup; int ranks[1]; int color = (rank < size / 2 ? 0 : 1); merr = MPI_Comm_split(MPI_COMM_WORLD, color, rank, &mcomm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == (size / 2)) { rleader = 0; } else { /* Remote leader is signficant only for the processes * designated local leaders */ rleader = -1; } *isLeftGroup = rank < size / 2; merr = MPI_Intercomm_create(mcomm, 0, MPI_COMM_WORLD, rleader, 12345, &mcomm2); if (merr) MTestPrintError(merr); /* We have an intercomm between the two halves of comm world. Now create * a new intercomm that removes rank 0 on each side. */ merr = MPI_Comm_group(mcomm2, &oldgroup); if (merr) MTestPrintError(merr); ranks[0] = 0; merr = MPI_Group_excl(oldgroup, 1, ranks, &newgroup); if (merr) MTestPrintError(merr); merr = MPI_Comm_create(mcomm2, newgroup, comm); if (merr) MTestPrintError(merr); merr = MPI_Group_free(&oldgroup); if (merr) MTestPrintError(merr); merr = MPI_Group_free(&newgroup); if (merr) MTestPrintError(merr); interCommName = "Intercomm by splitting MPI_COMM_WORLD then discarding 0 ranks with MPI_Comm_create"; } else { *comm = MPI_COMM_NULL; } break; default: *comm = MPI_COMM_NULL; interCommIdx = -1; break; } if (*comm != MPI_COMM_NULL) { merr = MPI_Comm_size(*comm, &size); if (merr) MTestPrintError(merr); merr = MPI_Comm_remote_size(*comm, &remsize); if (merr) MTestPrintError(merr); if (size + remsize >= min_size) done = 1; } else { interCommName = "MPI_COMM_NULL"; done = 1; } /* we are only done if all processes are done */ MPI_Allreduce(MPI_IN_PLACE, &done, 1, MPI_INT, MPI_LAND, MPI_COMM_WORLD); /* Advance the comm index whether we are done or not, otherwise we could * spin forever trying to allocate a too-small communicator over and * over again. */ interCommIdx++; if (!done && *comm != MPI_COMM_NULL) { /* avoid leaking communicators */ merr = MPI_Comm_free(comm); if (merr) MTestPrintError(merr); } /* cleanup for common temp objects */ if (mcomm != MPI_COMM_NULL) { merr = MPI_Comm_free(&mcomm); if (merr) MTestPrintError(merr); } if (mcomm2 != MPI_COMM_NULL) { merr = MPI_Comm_free(&mcomm2); if (merr) MTestPrintError(merr); } } return interCommIdx; }
int main( int argc, char **argv ) { int errs=0, toterr; MPI_Group basegroup; MPI_Group g1, g2, g3, g4, g5, g6, g7, g8, g9, g10; MPI_Group g3a, g3b; MPI_Comm comm, newcomm, splitcomm, dupcomm; int i, grp_rank, rank, grp_size, size, result; int nranks, *ranks, *ranks_out; int range[1][3]; int worldrank; MPI_Init( &argc, &argv ); MPI_Comm_rank( MPI_COMM_WORLD, &worldrank ); comm = MPI_COMM_WORLD; MPI_Comm_group( comm, &basegroup ); MPI_Comm_rank( comm, &rank ); MPI_Comm_size( comm, &size ); /* Get the basic information on this group */ MPI_Group_rank( basegroup, &grp_rank ); if (grp_rank != rank) { errs++; fprintf( stdout, "group rank %d != comm rank %d\n", grp_rank, rank ); } MPI_Group_size( basegroup, &grp_size ); if (grp_size != size) { errs++; fprintf( stdout, "group size %d != comm size %d\n", grp_size, size ); } /* Form a new communicator with inverted ranking */ MPI_Comm_split( comm, 0, size - rank, &newcomm ); MPI_Comm_group( newcomm, &g1 ); ranks = (int *)malloc( size * sizeof(int) ); ranks_out = (int *)malloc( size * sizeof(int) ); for (i=0; i<size; i++) ranks[i] = i; nranks = size; MPI_Group_translate_ranks( g1, nranks, ranks, basegroup, ranks_out ); for (i=0; i<size; i++) { if (ranks_out[i] != (size - 1) - i) { errs++; fprintf( stdout, "Translate ranks got %d expected %d\n", ranks_out[i], (size - 1) - i ); } } /* Check Compare */ MPI_Group_compare( basegroup, g1, &result ); if (result != MPI_SIMILAR) { errs++; fprintf( stdout, "Group compare should have been similar, was %d\n", result ); } MPI_Comm_dup( comm, &dupcomm ); MPI_Comm_group( dupcomm, &g2 ); MPI_Group_compare( basegroup, g2, &result ); if (result != MPI_IDENT) { errs++; fprintf( stdout, "Group compare should have been ident, was %d\n", result ); } MPI_Comm_split( comm, rank < size/2, rank, &splitcomm ); MPI_Comm_group( splitcomm, &g3 ); MPI_Group_compare( basegroup, g3, &result ); if (result != MPI_UNEQUAL) { errs++; fprintf( stdout, "Group compare should have been unequal, was %d\n", result ); } /* Build two groups that have this process and one other, but do not have the same processes */ ranks[0] = rank; ranks[1] = (rank + 1) % size; MPI_Group_incl( basegroup, 2, ranks, &g3a ); ranks[1] = (rank + size - 1) % size; MPI_Group_incl( basegroup, 2, ranks, &g3b ); MPI_Group_compare( g3a, g3b, &result ); if (result != MPI_UNEQUAL) { errs++; fprintf( stdout, "Group compare of equal sized but different groups should have been unequal, was %d\n", result ); } /* Build two new groups by excluding members; use Union to put them together again */ /* Exclude 0 */ for (i=0; i<size; i++) ranks[i] = i; MPI_Group_excl( basegroup, 1, ranks, &g4 ); /* Exclude 1-(size-1) */ MPI_Group_excl( basegroup, size-1, ranks+1, &g5 ); MPI_Group_union( g5, g4, &g6 ); MPI_Group_compare( basegroup, g6, &result ); if (result != MPI_IDENT) { int usize; errs++; /* See ordering requirements on union */ fprintf( stdout, "Group excl and union did not give ident groups\n" ); fprintf( stdout, "[%d] result of compare was %d\n", rank, result ); MPI_Group_size( g6, &usize ); fprintf( stdout, "Size of union is %d, should be %d\n", usize, size ); } MPI_Group_union( basegroup, g4, &g7 ); MPI_Group_compare( basegroup, g7, &result ); if (result != MPI_IDENT) { int usize; errs++; fprintf( stdout, "Group union of overlapping groups failed\n" ); fprintf( stdout, "[%d] result of compare was %d\n", rank, result ); MPI_Group_size( g7, &usize ); fprintf( stdout, "Size of union is %d, should be %d\n", usize, size ); } /* Use range_excl instead of ranks */ /* printf ("range excl\n" ); fflush( stdout ); */ range[0][0] = 1; range[0][1] = size-1; range[0][2] = 1; MPI_Group_range_excl( basegroup, 1, range, &g8 ); /* printf( "out of range excl\n" ); fflush( stdout ); */ MPI_Group_compare( g5, g8, &result ); /* printf( "out of compare\n" ); fflush( stdout ); */ if (result != MPI_IDENT) { errs++; fprintf( stdout, "Group range excl did not give ident groups\n" ); } /* printf( "intersection\n" ); fflush( stdout ); */ MPI_Group_intersection( basegroup, g4, &g9 ); MPI_Group_compare( g9, g4, &result ); if (result != MPI_IDENT) { errs++; fprintf( stdout, "Group intersection did not give ident groups\n" ); } /* Exclude EVERYTHING and check against MPI_GROUP_EMPTY */ /* printf( "range excl all\n" ); fflush( stdout ); */ range[0][0] = 0; range[0][1] = size-1; range[0][2] = 1; MPI_Group_range_excl( basegroup, 1, range, &g10 ); /* printf( "done range excl all\n" ); fflush(stdout); */ MPI_Group_compare( g10, MPI_GROUP_EMPTY, &result ); /* printf( "done compare to MPI_GROUP_EMPTY\n" ); fflush(stdout); */ if (result != MPI_IDENT) { errs++; fprintf( stdout, "MPI_GROUP_EMPTY didn't compare against empty group\n"); } /* printf( "freeing groups\n" ); fflush( stdout ); */ MPI_Group_free( &basegroup ); MPI_Group_free( &g1 ); MPI_Group_free( &g2 ); MPI_Group_free( &g3 ); MPI_Group_free( &g3a ); MPI_Group_free( &g3b ); MPI_Group_free( &g4 ); MPI_Group_free( &g5 ); MPI_Group_free( &g6 ); MPI_Group_free( &g7 ); MPI_Group_free( &g8 ); MPI_Group_free( &g9 ); MPI_Group_free( &g10 ); MPI_Comm_free( &dupcomm ); MPI_Comm_free( &splitcomm ); MPI_Comm_free( &newcomm ); MPI_Allreduce( &errs, &toterr, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); if (worldrank == 0) { if (toterr == 0) printf( " No Errors\n" ); else printf( "Found %d errors in MPI Group routines\n", toterr ); } MPI_Finalize(); return toterr; }
int main(int argc, char *argv[]) { double t1,t2,duration; if (argc > 1){ Size = atoi(argv[argc-1]); } MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &ProcNum);//Number of processes MPI_Comm_rank(MPI_COMM_WORLD, &ProcRank);//Rank of process t1 = MPI_Wtime(); MPI_Group WorldGroup, CalculatorGroup; MPI_Comm Calculators; int ranks[1]; ranks[0] = ProcNum-1; MPI_Comm_group(MPI_COMM_WORLD, &WorldGroup); MPI_Group_excl(WorldGroup, 1, ranks, &CalculatorGroup); MPI_Comm_create(MPI_COMM_WORLD,CalculatorGroup,&Calculators); int GridSize=sqrt((double)(ProcNum-1));//size of virtual topology(Grid) N=Size/GridSize+2;//N-size of subsystem; +2 - for boundary condition assert(GridSize*GridSize+1 == ProcNum); if (ProcRank!=ProcNum-1) { MPI_Datatype column; MPI_Type_vector(N,1,N,MPI_DOUBLE,&column); MPI_Type_commit(&column); //creating datatype for square MPI_Datatype square; MPI_Type_vector(N-2,N-2,N,MPI_DOUBLE,&square); MPI_Type_commit(&square); MPI_Comm GridComm; CreateGrid(GridSize, &GridComm, &Calculators); //Latt_Init1(N,"200last_state_gk_d0p1.bin"); LattInit(N); SolveEquations(250, GridComm, GridSize, column, square); MPI_Type_free(&square); MPI_Type_free(&column); MPI_Group_free(&CalculatorGroup); MPI_Comm_free(&Calculators); delete[] V; delete[] Cell; delete[] I_ext; } else { convert_buf = new short[(N-2)*(N-2)*ProcNum]; double* V_all = new double[(N-2)*(N-2)*ProcNum]; double* V_temp = new double[(N-2)*(N-2)]; short* V_save = new short[(N-2)*(N-2)*ProcNum]; int ii; //printf("Total number of frames: %i\n", DrawNum); #ifdef OS_WINDOWS int fd = open("200snapshots_gk_d0p1.bin",O_RDWR|O_CREAT | O_BINARY,S_IREAD|S_IWRITE); #else int fd = open("snapshots.bin",O_RDWR|O_CREAT ,S_IREAD|S_IWRITE); #endif for (int i=0;i<DrawNum*2;i++) { MPI_Gather(V_temp,(N-2)*(N-2),MPI_DOUBLE,V_all,(N-2)*(N-2),MPI_DOUBLE,ProcNum-1,MPI_COMM_WORLD); for (ii=0;ii<(N-2)*(N-2)*(ProcNum-1);ii++) { V_save[ii]=short(V_all[ii]*250.); } convertRst(V_save,convert_buf,Size,GridSize); save(convert_buf,Size*Size,fd); } close(fd); MPI_Gather(V_temp,(N-2)*(N-2),MPI_DOUBLE,V_all,(N-2)*(N-2),MPI_DOUBLE,ProcNum-1,MPI_COMM_WORLD); for (ii=0;ii<(N-2)*(N-2)*(ProcNum-1);ii++) { V_save[ii]=short(V_all[ii]*250.); } #ifdef OS_WINDOWS fd = open("200last_V_gk_d0p1.bin",O_RDWR|O_CREAT | O_BINARY,S_IREAD|S_IWRITE); #else fd = open("lastV.bin",O_RDWR|O_CREAT,S_IREAD|S_IWRITE); #endif save(V_save,(N-2)*(N-2)*(ProcNum-1),fd); close(fd); #ifdef OS_WINDOWS fd = open("200last_state_gk_d0p1.bin",O_RDWR|O_CREAT | O_BINARY,S_IREAD|S_IWRITE); #else fd = open("state.bin",O_RDWR|O_CREAT ,S_IREAD|S_IWRITE); #endif for (int i=0; i<9; i++) { MPI_Gather(V_temp,(N-2)*(N-2),MPI_DOUBLE,V_all,(N-2)*(N-2),MPI_DOUBLE,ProcNum-1,MPI_COMM_WORLD); save_double(V_all,(N-2)*(N-2)*(ProcNum-1),fd); } close(fd); delete[] V_all; delete[] V_temp; delete[] V_save; delete[] convert_buf; } t2 = MPI_Wtime(); if (ProcRank==0) { printf("Experiment duration: %f seconds\n",t2-t1); } MPI_Finalize(); return 0; }
int main (int argc, char **argv) { int nprocs = -1; int rank = -1; int comm = MPI_COMM_WORLD; char processor_name[128]; int namelen = 128; int i; int ranks[2], ranges[1][3]; MPI_Group newgroup[GROUP_CONSTRUCTOR_COUNT]; MPI_Group newgroup2[GROUP_CONSTRUCTOR_COUNT]; MPI_Comm temp; MPI_Comm intercomm = MPI_COMM_NULL; /* init */ MPI_Init (&argc, &argv); MPI_Comm_size (comm, &nprocs); MPI_Comm_rank (comm, &rank); MPI_Get_processor_name (processor_name, &namelen); printf ("(%d) is alive on %s\n", rank, processor_name); fflush (stdout); ranks[0] = 0; ranks[1] = 1; ranges[0][0] = 0; ranges[0][1] = 2; ranges[0][2] = 2; MPI_Barrier (comm); if (nprocs < 3) { printf ("requires at least 3 tasks\n"); } else { /* create the groups */ if (GROUP_CONSTRUCTOR_COUNT > 0) MPI_Comm_group (MPI_COMM_WORLD, &newgroup[0]); if (GROUP_CONSTRUCTOR_COUNT > 1) MPI_Group_incl (newgroup[0], 2, ranks, &newgroup[1]); if (GROUP_CONSTRUCTOR_COUNT > 2) MPI_Group_excl (newgroup[0], 2, ranks, &newgroup[2]); if (GROUP_CONSTRUCTOR_COUNT > 3) MPI_Group_range_incl (newgroup[0], 1, ranges, &newgroup[3]); if (GROUP_CONSTRUCTOR_COUNT > 4) MPI_Group_range_excl (newgroup[0], 1, ranges, &newgroup[4]); if (GROUP_CONSTRUCTOR_COUNT > 5) MPI_Group_union (newgroup[1], newgroup[3], &newgroup[5]); if (GROUP_CONSTRUCTOR_COUNT > 6) MPI_Group_intersection (newgroup[5], newgroup[2], &newgroup[6]); if (GROUP_CONSTRUCTOR_COUNT > 7) MPI_Group_difference (newgroup[5], newgroup[2], &newgroup[7]); if (GROUP_CONSTRUCTOR_COUNT > 8) { /* need lots of stuff for this constructor... */ MPI_Comm_split (MPI_COMM_WORLD, rank % 3, nprocs - rank, &temp); if (rank % 3) { MPI_Intercomm_create (temp, 0, MPI_COMM_WORLD, (((nprocs % 3) == 2) && ((rank % 3) == 2)) ? nprocs - 1 : nprocs - (rank % 3) - (nprocs % 3), INTERCOMM_CREATE_TAG, &intercomm); MPI_Comm_remote_group (intercomm, &newgroup[8]); MPI_Comm_free (&intercomm); } else { MPI_Comm_group (temp, &newgroup[8]); } MPI_Comm_free (&temp); } } MPI_Barrier (comm); printf ("(%d) Finished normally\n", rank); MPI_Finalize (); }
int main( int argc, char *argv[] ) { int iter; int in, out, i, iters, max, ix, iy, ranks[1], done, temp; double x, y, Pi, error, epsilon; int numprocs, myid, server, totalin, totalout, workerid; int rands[CHUNKSIZE], request; MPI_Comm world, workers; MPI_Group world_group, worker_group; MPI_Status status; MPI_Init(&argc,&argv); world = MPI_COMM_WORLD; MPI_Comm_size(world,&numprocs); MPI_Comm_rank(world,&myid); server = numprocs-1; /* last proc is server */ if (myid == 0) sscanf( argv[1], "%lf", &epsilon ); MPI_Bcast( &epsilon, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD ); MPI_Comm_group( world, &world_group ); ranks[0] = server; MPI_Group_excl( world_group, 1, ranks, &worker_group ); MPI_Comm_create( world, worker_group, &workers ); MPI_Group_free(&worker_group); if (myid == server) { /* I am the rand server */ do { MPI_Recv(&request, 1, MPI_INT, MPI_ANY_SOURCE, REQUEST, world, &status); if (request) { assert(request>0); for (i = 0; i < CHUNKSIZE; ) { rands[i] = random(); if (rands[i] <= INT_MAX) i++; } MPI_Send(rands, CHUNKSIZE, MPI_INT, status.MPI_SOURCE, REPLY, world); } } while( request>0 ); } else { /* I am a worker process */ request = 1; done = in = out = 0; max = INT_MAX; /* max int, for normalization */ MPI_Send( &request, 1, MPI_INT, server, REQUEST, world ); MPI_Comm_rank( workers, &workerid ); iter = 0; while (!done) { iter++; request = 1; MPI_Recv( rands, CHUNKSIZE, MPI_INT, server, REPLY, world, &status ); for (i=0; i<CHUNKSIZE-1; ) { x = (((double) rands[i++])/max) * 2 - 1; y = (((double) rands[i++])/max) * 2 - 1; if (x*x + y*y < 1.0) in++; else out++; } MPI_Allreduce(&in, &totalin, 1, MPI_INT, MPI_SUM, workers); MPI_Allreduce(&out, &totalout, 1, MPI_INT, MPI_SUM, workers); Pi = (4.0*totalin)/(totalin + totalout); error = fabs( Pi-3.141592653589793238462643); done = (error < epsilon || (totalin+totalout) > 1000000); request = (done) ? 0 : 1; if (myid == 0) { //printf( "\rpi = %23.20f", Pi ); MPI_Send( &request, 1, MPI_INT, server, REQUEST, world ); } else { if (request) { assert(request>0); MPI_Send(&request, 1, MPI_INT, server, REQUEST, world); } } } MPI_Comm_free(&workers); } if (myid == 0) { //printf( "\npoints: %d\nin: %d, out: %d, <ret> to exit\n", // totalin+totalout, totalin, totalout ); } MPI_Finalize(); }
bool pRPL::Process:: grouping(int nGroups, bool incldMaster, Process *pGrpedPrc, Process *pGrpMaster) const { if(!initialized()) { cerr << __FILE__ << " " << __FUNCTION__ \ << " Error: Process has NOT been initialized," \ << " unable to be grouped" << endl; return false; } if(!active()) { cerr << __FILE__ << " " << __FUNCTION__ \ << " Error: inactive Process," \ << " unable to group a Null communicator." \ << " id = " << _id << " nTotPrcs = " << _nTotalPrcs << endl; return false; } if(nGroups <= 0 || nGroups > _nTotalPrcs) { cerr << __FILE__ << " " << __FUNCTION__ \ << " Error: invalid number of groups (" \ << nGroups << ") as the total number of processes is " \ << _nTotalPrcs << endl; return false; } if(!incldMaster && _nTotalPrcs <= 1) { cerr << __FILE__ << " " << __FUNCTION__ \ << " Error: " << _nTotalPrcs << " processes can NOT" \ << " be grouped without the master process" << endl; return false; } MPI_Group glbGrp; MPI_Comm glbComm = _comm; MPI_Comm_group(glbComm, &glbGrp); int myID = -1; int grpID = -1; MPI_Comm grpComm = MPI_COMM_NULL; if(incldMaster) { myID = _id; grpID = myID % nGroups; MPI_Comm_split(glbComm, grpID, myID, &grpComm); if(!pGrpedPrc->set(grpComm, _hasWriter, grpID)) { return false; } if(pGrpMaster != NULL) { MPI_Group masterGrp= MPI_GROUP_NULL; MPI_Comm masterComm = MPI_COMM_NULL; int grpMasterRange[1][3] = {{0, nGroups-1, 1}}; MPI_Group_range_incl(glbGrp, 1, grpMasterRange, &masterGrp); MPI_Comm_create(glbComm, masterGrp, &masterComm); if(!pGrpMaster->set(masterComm)) { return false; } } } else { int excldRanks[1] = {0}; MPI_Group glbGrp2 = MPI_GROUP_NULL; MPI_Group_excl(glbGrp, 1, excldRanks, &glbGrp2); MPI_Comm_create(_comm, glbGrp2, &glbComm); glbGrp = glbGrp2; if(!isMaster()) { MPI_Comm_rank(glbComm, &myID); grpID = myID % nGroups; MPI_Comm_split(glbComm, grpID, myID, &grpComm); if(!pGrpedPrc->set(grpComm, _hasWriter, grpID)) { return false; } if(pGrpMaster != NULL) { MPI_Group masterGrp= MPI_GROUP_NULL; MPI_Comm masterComm = MPI_COMM_NULL; int grpMasterRange[1][3] = {{0, nGroups-1, 1}}; MPI_Group_range_incl(glbGrp, 1, grpMasterRange, &masterGrp); MPI_Comm_create(glbComm, masterGrp, &masterComm); if(!pGrpMaster->set(masterComm)) { return false; } } } } return true; }
MTEST_THREAD_RETURN_TYPE test_idup(void *arg) { int i; int size, rank; int ranges[1][3]; int rleader, isLeft; int *excl = NULL; int tid = *(int *) arg; MPI_Group ingroup, high_group, even_group; MPI_Comm local_comm, inter_comm; MPI_Comm idupcomms[NUM_IDUPS]; MPI_Request reqs[NUM_IDUPS]; MPI_Comm outcomm; MPI_Comm incomm = comms[tid]; MPI_Comm_size(incomm, &size); MPI_Comm_rank(incomm, &rank); MPI_Comm_group(incomm, &ingroup); /* Idup incomm multiple times */ for (i = 0; i < NUM_IDUPS; i++) { MPI_Comm_idup(incomm, &idupcomms[i], &reqs[i]); } /* Overlap pending idups with various comm generation functions */ /* Comm_dup */ MPI_Comm_dup(incomm, &outcomm); errs[tid] += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Comm_split */ MPI_Comm_split(incomm, rank % 2, size - rank, &outcomm); errs[tid] += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Comm_create, high half of incomm */ ranges[0][0] = size / 2; ranges[0][1] = size - 1; ranges[0][2] = 1; MPI_Group_range_incl(ingroup, 1, ranges, &high_group); MPI_Comm_create(incomm, high_group, &outcomm); MPI_Group_free(&high_group); errs[tid] += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Comm_create_group, even ranks of incomm */ /* exclude the odd ranks */ excl = malloc((size / 2) * sizeof(int)); for (i = 0; i < size / 2; i++) excl[i] = (2 * i) + 1; MPI_Group_excl(ingroup, size / 2, excl, &even_group); free(excl); if (rank % 2 == 0) { MPI_Comm_create_group(incomm, even_group, 0, &outcomm); } else { outcomm = MPI_COMM_NULL; } MPI_Group_free(&even_group); errs[tid] += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Intercomm_create & Intercomm_merge */ MPI_Comm_split(incomm, (rank < size / 2), rank, &local_comm); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { rleader = -1; } isLeft = rank < size / 2; MPI_Intercomm_create(local_comm, 0, incomm, rleader, 99, &inter_comm); MPI_Intercomm_merge(inter_comm, isLeft, &outcomm); MPI_Comm_free(&local_comm); errs[tid] += MTestTestComm(inter_comm); MTestFreeComm(&inter_comm); errs[tid] += MTestTestComm(outcomm); MTestFreeComm(&outcomm); MPI_Waitall(NUM_IDUPS, reqs, MPI_STATUSES_IGNORE); for (i = 0; i < NUM_IDUPS; i++) { errs[tid] += MTestTestComm(idupcomms[i]); MPI_Comm_free(&idupcomms[i]); } MPI_Group_free(&ingroup); return NULL; }
int main(int argc, char **argv) { int errs = 0; int i; int rank, size; int *excl; int ranges[1][3]; int isLeft, rleader; MPI_Group world_group, high_group, even_group; MPI_Comm local_comm, inter_comm, test_comm, outcomm; MPI_Comm idupcomms[NUM_IDUPS]; MPI_Request reqs[NUM_IDUPS]; MTest_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_group(MPI_COMM_WORLD, &world_group); if (size < 2) { printf("this test requires at least 2 processes\n"); MPI_Abort(MPI_COMM_WORLD, 1); } /* Idup MPI_COMM_WORLD multiple times */ for (i = 0; i < NUM_IDUPS; i++) { MPI_Comm_idup(MPI_COMM_WORLD, &idupcomms[i], &reqs[i]); } /* Overlap pending idups with various comm generation functions */ /* Comm_dup */ MPI_Comm_dup(MPI_COMM_WORLD, &outcomm); errs += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Comm_split */ MPI_Comm_split(MPI_COMM_WORLD, rank % 2, size - rank, &outcomm); errs += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Comm_create, high half of MPI_COMM_WORLD */ ranges[0][0] = size / 2; ranges[0][1] = size - 1; ranges[0][2] = 1; MPI_Group_range_incl(world_group, 1, ranges, &high_group); MPI_Comm_create(MPI_COMM_WORLD, high_group, &outcomm); MPI_Group_free(&high_group); errs += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Comm_create_group, even ranks of MPI_COMM_WORLD */ /* exclude the odd ranks */ excl = malloc((size / 2) * sizeof(int)); for (i = 0; i < size / 2; i++) excl[i] = (2 * i) + 1; MPI_Group_excl(world_group, size / 2, excl, &even_group); free(excl); if (rank % 2 == 0) { MPI_Comm_create_group(MPI_COMM_WORLD, even_group, 0, &outcomm); } else { outcomm = MPI_COMM_NULL; } MPI_Group_free(&even_group); errs += MTestTestComm(outcomm); MTestFreeComm(&outcomm); /* Intercomm_create & Intercomm_merge */ MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &local_comm); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { rleader = -1; } isLeft = rank < size / 2; MPI_Intercomm_create(local_comm, 0, MPI_COMM_WORLD, rleader, 99, &inter_comm); MPI_Intercomm_merge(inter_comm, isLeft, &outcomm); MPI_Comm_free(&local_comm); errs += MTestTestComm(inter_comm); MTestFreeComm(&inter_comm); errs += MTestTestComm(outcomm); MTestFreeComm(&outcomm); MPI_Waitall(NUM_IDUPS, reqs, MPI_STATUSES_IGNORE); for (i = 0; i < NUM_IDUPS; i++) { errs += MTestTestComm(idupcomms[i]); MPI_Comm_free(&idupcomms[i]); } MPI_Group_free(&world_group); MTest_Finalize(errs); return MTestReturnValue(errs); }
/* * This test attempts collective bcast communication after a process in * the communicator has failed. */ int main(int argc, char **argv) { int rank, size, rc, errclass, toterrs, errs = 0; int deadprocs[] = { 1 }; char buf[100000]; MPI_Group world, newgroup; MPI_Comm newcomm; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); MPI_Comm_group(MPI_COMM_WORLD, &world); MPI_Group_excl(world, 1, deadprocs, &newgroup); MPI_Comm_create_group(MPI_COMM_WORLD, newgroup, 0, &newcomm); if (size < 3) { fprintf(stderr, "Must run with at least 3 processes\n"); MPI_Abort(MPI_COMM_WORLD, 1); } if (rank == 1) { exit(EXIT_FAILURE); } if (rank == 0) { strcpy(buf, "No Errors"); } /* do a small bcast first */ rc = MPI_Bcast(buf, 10, MPI_CHAR, 0, MPI_COMM_WORLD); #if defined (MPICH) && (MPICH_NUMVERSION >= 30100102) MPI_Error_class(rc, &errclass); if ((rc) && (errclass != MPIX_ERR_PROC_FAILED)) { fprintf(stderr, "Wrong error code (%d) returned. Expected MPIX_ERR_PROC_FAILED\n", errclass); errs++; } #endif /* reset the non-root buffers */ if (rank != 0) memset(buf, 0, sizeof(buf)); /* do a larger bcast */ rc = MPI_Bcast(buf, 100000, MPI_CHAR, 0, MPI_COMM_WORLD); #if defined (MPICH) && (MPICH_NUMVERSION >= 30100102) MPI_Error_class(rc, &errclass); if ((rc) && (errclass != MPIX_ERR_PROC_FAILED)) { fprintf(stderr, "Wrong error code (%d) returned. Expected MPIX_ERR_PROC_FAILED\n", errclass); errs++; } #endif rc = MPI_Reduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, 0, newcomm); if (rc) fprintf(stderr, "Failed to get errors from other processes\n"); if (rank == 0) { if (toterrs) { printf(" Found %d errors\n", toterrs); } else { printf(" No Errors\n"); } fflush(stdout); } MPI_Group_free(&world); MPI_Group_free(&newgroup); MPI_Comm_free(&newcomm); MPI_Finalize(); return 0; }
XdmfHDF5ControllerDSM::XdmfHDF5ControllerDSM(const std::string & hdf5FilePath, const std::string & dataSetPath, const shared_ptr<const XdmfArrayType> type, const std::vector<unsigned int> & start, const std::vector<unsigned int> & stride, const std::vector<unsigned int> & dimensions, const std::vector<unsigned int> & dataspaceDimensions, MPI_Comm comm, unsigned int bufferSize, int startCoreIndex, int endCoreIndex) : XdmfHDF5Controller(hdf5FilePath, dataSetPath, type, start, stride, dimensions, dataspaceDimensions), #ifdef XDMF_BUILD_DSM_THREADS mDSMBuffer(NULL), mDSMManager(NULL), #endif mServerMode(true) { int rank, size; MPI_Comm_size(comm, &size); MPI_Comm_rank(comm, &rank); // Negative values will be changed to maximum range if (startCoreIndex < 0) { startCoreIndex = 0; } if (endCoreIndex < 0) { endCoreIndex = size - 1; } // Ensure start index is less than end index if (startCoreIndex > endCoreIndex) { int tempholder = startCoreIndex; startCoreIndex = endCoreIndex; endCoreIndex = tempholder; } MPI_Comm serverComm; MPI_Group workers, dsmgroup, serversplit, servergroup; int * ServerIds = (int *)calloc((3), sizeof(int)); unsigned int index = 0; for(int i=startCoreIndex ; i <= endCoreIndex ; ++i) { ServerIds[index++] = i; } MPI_Comm_group(comm, &serversplit); MPI_Group_incl(serversplit, index, ServerIds, &servergroup); MPI_Comm_create(comm, servergroup, &serverComm); MPI_Comm_group(comm, &dsmgroup); MPI_Group_excl(dsmgroup, index, ServerIds, &workers); MPI_Comm_create(comm, workers, &mWorkerComm); cfree(ServerIds); // Create the manager mDSMServerManager = new XdmfDSMManager(); mDSMServerManager->SetLocalBufferSizeMBytes(bufferSize); mDSMServerManager->SetInterCommType(XDMF_DSM_COMM_MPI); if (rank >= startCoreIndex && rank <= endCoreIndex) { mDSMServerManager->SetMpiComm(serverComm); mDSMServerManager->Create(); } else { mDSMServerManager->SetMpiComm(mWorkerComm); mDSMServerManager->SetIsServer(false); mDSMServerManager->Create(startCoreIndex, endCoreIndex); } XDMF_dsm_set_manager(mDSMServerManager); mDSMServerBuffer = mDSMServerManager->GetDsmBuffer(); mDSMServerBuffer->GetComm()->DupInterComm(comm); mDSMServerBuffer->SetIsConnected(true); if (startCoreIndex < size) { if (rank >= startCoreIndex && rank <= endCoreIndex) { mDSMServerManager->GetDsmBuffer()->ReceiveInfo(); } else { mDSMServerManager->GetDsmBuffer()->SendInfo(); } } MPI_Barrier(comm); // Loop needs to be started before anything can be done to the file // since the service is what sets up the file if (rank < startCoreIndex || rank > endCoreIndex) { // Turn off the server designation mDSMServerBuffer->SetIsServer(false); // If this is set to false then the buffer will attempt to // connect to the intercomm for DSM stuff mDSMServerManager->SetIsServer(false); } else { // On cores where memory is set up, start the service loop // This should iterate infinitely until a value to end the loop is passed int returnOpCode; mDSMServerBuffer->BufferServiceLoop(&returnOpCode); } }
/* * Get an intracommunicator with at least min_size members. If "allowSmaller" * is true, allow the communicator to be smaller than MPI_COMM_WORLD and * for this routine to return MPI_COMM_NULL for some values. Returns 0 if * no more communicators are available. */ int MTestGetIntracommGeneral(MPI_Comm * comm, int min_size, int allowSmaller) { int size, rank, merr; int done = 0; int isBasic = 0; /* The while loop allows us to skip communicators that are too small. * MPI_COMM_NULL is always considered large enough */ while (!done) { isBasic = 0; intraCommName = ""; switch (intraCommIdx) { case 0: *comm = MPI_COMM_WORLD; isBasic = 1; intraCommName = "MPI_COMM_WORLD"; break; case 1: /* dup of world */ merr = MPI_Comm_dup(MPI_COMM_WORLD, comm); if (merr) MTestPrintError(merr); intraCommName = "Dup of MPI_COMM_WORLD"; break; case 2: /* reverse ranks */ merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_split(MPI_COMM_WORLD, 0, size - rank, comm); if (merr) MTestPrintError(merr); intraCommName = "Rank reverse of MPI_COMM_WORLD"; break; case 3: /* subset of world, with reversed ranks */ merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_split(MPI_COMM_WORLD, ((rank < size / 2) ? 1 : MPI_UNDEFINED), size - rank, comm); if (merr) MTestPrintError(merr); intraCommName = "Rank reverse of half of MPI_COMM_WORLD"; break; case 4: *comm = MPI_COMM_SELF; isBasic = 1; intraCommName = "MPI_COMM_SELF"; break; case 5: { #if MTEST_HAVE_MIN_MPI_VERSION(3,0) /* Dup of the world using MPI_Intercomm_merge */ int rleader, isLeft; MPI_Comm local_comm, inter_comm; MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (size > 1) { merr = MPI_Comm_split(MPI_COMM_WORLD, (rank < size / 2), rank, &local_comm); if (merr) MTestPrintError(merr); if (rank == 0) { rleader = size / 2; } else if (rank == size / 2) { rleader = 0; } else { rleader = -1; } isLeft = rank < size / 2; merr = MPI_Intercomm_create(local_comm, 0, MPI_COMM_WORLD, rleader, 99, &inter_comm); if (merr) MTestPrintError(merr); merr = MPI_Intercomm_merge(inter_comm, isLeft, comm); if (merr) MTestPrintError(merr); MPI_Comm_free(&inter_comm); MPI_Comm_free(&local_comm); intraCommName = "Dup of WORLD created by MPI_Intercomm_merge"; } else { *comm = MPI_COMM_NULL; } } break; case 6: { /* Even of the world using MPI_Comm_create_group */ int i; MPI_Group world_group, even_group; int *excl = NULL; MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (allowSmaller && (size + 1) / 2 >= min_size) { /* exclude the odd ranks */ excl = malloc((size / 2) * sizeof(int)); for (i = 0; i < size / 2; i++) excl[i] = (2 * i) + 1; MPI_Comm_group(MPI_COMM_WORLD, &world_group); MPI_Group_excl(world_group, size / 2, excl, &even_group); MPI_Group_free(&world_group); free(excl); if (rank % 2 == 0) { /* Even processes create a comm. for themselves */ MPI_Comm_create_group(MPI_COMM_WORLD, even_group, 0, comm); intraCommName = "Even of WORLD created by MPI_Comm_create_group"; } else { *comm = MPI_COMM_NULL; } MPI_Group_free(&even_group); } else { *comm = MPI_COMM_NULL; } #else *comm = MPI_COMM_NULL; #endif } break; case 7: { /* High half of the world using MPI_Comm_create */ int ranges[1][3]; MPI_Group world_group, high_group; MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &rank); ranges[0][0] = size / 2; ranges[0][1] = size - 1; ranges[0][2] = 1; if (allowSmaller && (size + 1) / 2 >= min_size) { MPI_Comm_group(MPI_COMM_WORLD, &world_group); merr = MPI_Group_range_incl(world_group, 1, ranges, &high_group); if (merr) MTestPrintError(merr); merr = MPI_Comm_create(MPI_COMM_WORLD, high_group, comm); if (merr) MTestPrintError(merr); MPI_Group_free(&world_group); MPI_Group_free(&high_group); intraCommName = "High half of WORLD created by MPI_Comm_create"; } else { *comm = MPI_COMM_NULL; } } break; /* These next cases are communicators that include some * but not all of the processes */ case 8: case 9: case 10: case 11: { int newsize; merr = MPI_Comm_size(MPI_COMM_WORLD, &size); if (merr) MTestPrintError(merr); newsize = size - (intraCommIdx - 7); if (allowSmaller && newsize >= min_size) { merr = MPI_Comm_rank(MPI_COMM_WORLD, &rank); if (merr) MTestPrintError(merr); merr = MPI_Comm_split(MPI_COMM_WORLD, rank < newsize, rank, comm); if (merr) MTestPrintError(merr); if (rank >= newsize) { merr = MPI_Comm_free(comm); if (merr) MTestPrintError(merr); *comm = MPI_COMM_NULL; } else { intraCommName = "Split of WORLD"; } } else { /* Act like default */ *comm = MPI_COMM_NULL; intraCommIdx = -1; } } break; /* Other ideas: dup of self, cart comm, graph comm */ default: *comm = MPI_COMM_NULL; intraCommIdx = -1; break; } if (*comm != MPI_COMM_NULL) { merr = MPI_Comm_size(*comm, &size); if (merr) MTestPrintError(merr); if (size >= min_size) done = 1; } else { intraCommName = "MPI_COMM_NULL"; isBasic = 1; done = 1; } /* we are only done if all processes are done */ MPI_Allreduce(MPI_IN_PLACE, &done, 1, MPI_INT, MPI_LAND, MPI_COMM_WORLD); /* Advance the comm index whether we are done or not, otherwise we could * spin forever trying to allocate a too-small communicator over and * over again. */ intraCommIdx++; if (!done && !isBasic && *comm != MPI_COMM_NULL) { /* avoid leaking communicators */ merr = MPI_Comm_free(comm); if (merr) MTestPrintError(merr); } } return intraCommIdx; }
int main( int argc, char *argv[] ) { MPI_Group g1, g2, g4, g5, g45, selfgroup, g6; int ranks[16], size, rank, myrank, range[1][3]; int errs = 0; int i, rin[16], rout[16], result; MPI_Init(&argc,&argv); MPI_Comm_group( MPI_COMM_WORLD, &g1 ); MPI_Comm_rank( MPI_COMM_WORLD, &myrank ); MPI_Comm_size( MPI_COMM_WORLD, &size ); if (size < 8) { fprintf( stderr, "Test requires 8 processes (16 prefered) only %d provided\n", size ); errs++; } /* 16 members, this process is rank 0, return in group 1 */ ranks[0] = myrank; ranks[1] = 2; ranks[2] = 7; if (myrank == 2) ranks[1] = 3; if (myrank == 7) ranks[2] = 6; MPI_Group_incl( g1, 3, ranks, &g2 ); /* Check the resulting group */ MPI_Group_size( g2, &size ); MPI_Group_rank( g2, &rank ); if (size != 3) { fprintf( stderr, "Size should be %d, is %d\n", 3, size ); errs++; } if (rank != 0) { fprintf( stderr, "Rank should be %d, is %d\n", 0, rank ); errs++; } rin[0] = 0; rin[1] = 1; rin[2] = 2; MPI_Group_translate_ranks( g2, 3, rin, g1, rout ); for (i=0; i<3; i++) { if (rout[i] != ranks[i]) { fprintf( stderr, "translated rank[%d] %d should be %d\n", i, rout[i], ranks[i] ); errs++; } } /* Translate the process of the self group against another group */ MPI_Comm_group( MPI_COMM_SELF, &selfgroup ); rin[0] = 0; MPI_Group_translate_ranks( selfgroup, 1, rin, g1, rout ); if (rout[0] != myrank) { fprintf( stderr, "translated of self is %d should be %d\n", rout[0], myrank ); errs++; } for (i=0; i<size; i++) rin[i] = i; MPI_Group_translate_ranks( g1, size, rin, selfgroup, rout ); for (i=0; i<size; i++) { if (i == myrank && rout[i] != 0) { fprintf( stderr, "translated world to self of %d is %d\n", i, rout[i] ); errs++; } else if (i != myrank && rout[i] != MPI_UNDEFINED) { fprintf( stderr, "translated world to self of %d should be undefined, is %d\n", i, rout[i] ); errs++; } } MPI_Group_free( &selfgroup ); /* Exclude everyone in our group */ { int ii, *lranks, g1size; MPI_Group_size( g1, &g1size ); lranks = (int *)malloc( g1size * sizeof(int) ); for (ii=0; ii<g1size; ii++) lranks[ii] = ii; MPI_Group_excl( g1, g1size, lranks, &g6 ); if (g6 != MPI_GROUP_EMPTY) { fprintf( stderr, "Group formed by excluding all ranks not empty\n" ); errs++; MPI_Group_free( &g6 ); } free( lranks ); } /* Add tests for additional group operations */ /* g2 = incl 1,3,7 g3 = excl 1,3,7 intersect ( w, g2 ) => g2 intersect ( w, g3 ) => g3 intersect ( g2, g3 ) => empty g4 = rincl 1:n-1:2 g5 = rexcl 1:n-1:2 union( g4, g5 ) => world g6 = rincl n-1:1:-1 g7 = rexcl n-1:1:-1 union( g6, g7 ) => concat of entries, similar to world diff( w, g2 ) => g3 */ MPI_Group_free( &g2 ); range[0][0] = 1; range[0][1] = size-1; range[0][2] = 2; MPI_Group_range_excl( g1, 1, range, &g5 ); range[0][0] = 1; range[0][1] = size-1; range[0][2] = 2; MPI_Group_range_incl( g1, 1, range, &g4 ); MPI_Group_union( g4, g5, &g45 ); MPI_Group_compare( MPI_GROUP_EMPTY, g4, &result ); if (result != MPI_UNEQUAL) { errs++; fprintf( stderr, "Comparison with empty group gave %d, not 3\n", result ); } MPI_Group_free( &g4 ); MPI_Group_free( &g5 ); MPI_Group_free( &g45 ); /* Now, duplicate the test, but using negative strides */ range[0][0] = size-1; range[0][1] = 1; range[0][2] = -2; MPI_Group_range_excl( g1, 1, range, &g5 ); range[0][0] = size-1; range[0][1] = 1; range[0][2] = -2; MPI_Group_range_incl( g1, 1, range, &g4 ); MPI_Group_union( g4, g5, &g45 ); MPI_Group_compare( MPI_GROUP_EMPTY, g4, &result ); if (result != MPI_UNEQUAL) { errs++; fprintf( stderr, "Comparison with empty group (formed with negative strides) gave %d, not 3\n", result ); } MPI_Group_free( &g4 ); MPI_Group_free( &g5 ); MPI_Group_free( &g45 ); MPI_Group_free( &g1 ); if (myrank == 0) { if (errs == 0) { printf( " No Errors\n" ); } else { printf( "Found %d errors\n", errs ); } } MPI_Finalize(); return 0; }
int Zoltan_PHG_Initialize_Params( ZZ *zz, /* the Zoltan data structure */ float *part_sizes, PHGPartParams *hgp ) { int err = ZOLTAN_OK; char *yo = "Zoltan_PHG_Initialize_Params"; int nProc; int usePrimeComm; MPI_Comm communicator; char add_obj_weight[MAX_PARAM_STRING_LEN]; char edge_weight_op[MAX_PARAM_STRING_LEN]; char cut_objective[MAX_PARAM_STRING_LEN]; char *package = hgp->hgraph_pkg; char *method = hgp->hgraph_method; char buf[1024]; memset(hgp, 0, sizeof(*hgp)); /* in the future if we forget to initialize another param at least it will be 0 */ Zoltan_Bind_Param(PHG_params, "HYPERGRAPH_PACKAGE", &hgp->hgraph_pkg); Zoltan_Bind_Param(PHG_params, "PHG_MULTILEVEL", &hgp->useMultilevel); Zoltan_Bind_Param(PHG_params, "PHG_FROM_GRAPH_METHOD", hgp->convert_str); Zoltan_Bind_Param(PHG_params, "PHG_OUTPUT_LEVEL", &hgp->output_level); Zoltan_Bind_Param(PHG_params, "FINAL_OUTPUT", &hgp->final_output); Zoltan_Bind_Param(PHG_params, "CHECK_GRAPH", &hgp->check_graph); Zoltan_Bind_Param(PHG_params, "CHECK_HYPERGRAPH", &hgp->check_graph); Zoltan_Bind_Param(PHG_params, "PHG_NPROC_VERTEX", &hgp->nProc_x_req); Zoltan_Bind_Param(PHG_params, "PHG_NPROC_EDGE", &hgp->nProc_y_req); Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_LIMIT", &hgp->redl); Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_NCANDIDATE", &hgp->nCand); Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_METHOD", hgp->redm_str); Zoltan_Bind_Param(PHG_params, "PHG_COARSENING_METHOD_FAST", hgp->redm_fast); Zoltan_Bind_Param(PHG_params, "PHG_VERTEX_VISIT_ORDER", &hgp->visit_order); Zoltan_Bind_Param(PHG_params, "PHG_EDGE_SCALING", &hgp->edge_scaling); Zoltan_Bind_Param(PHG_params, "PHG_VERTEX_SCALING", &hgp->vtx_scaling); Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_METHOD", hgp->refinement_str); Zoltan_Bind_Param(PHG_params, "PHG_DIRECT_KWAY", &hgp->kway); Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_LOOP_LIMIT", &hgp->fm_loop_limit); Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_MAX_NEG_MOVE", &hgp->fm_max_neg_move); Zoltan_Bind_Param(PHG_params, "PHG_REFINEMENT_QUALITY", &hgp->refinement_quality); Zoltan_Bind_Param(PHG_params, "PHG_COARSEPARTITION_METHOD", hgp->coarsepartition_str); Zoltan_Bind_Param(PHG_params, "PHG_USE_TIMERS", (void*) &hgp->use_timers); Zoltan_Bind_Param(PHG_params, "USE_TIMERS", (void*) &hgp->use_timers); Zoltan_Bind_Param(PHG_params, "PHG_EDGE_SIZE_THRESHOLD", (void*) &hgp->EdgeSizeThreshold); Zoltan_Bind_Param(PHG_params, "PHG_MATCH_EDGE_SIZE_THRESHOLD", (void*) &hgp->MatchEdgeSizeThreshold); Zoltan_Bind_Param(PHG_params, "PHG_BAL_TOL_ADJUSTMENT", (void*) &hgp->bal_tol_adjustment); Zoltan_Bind_Param(PHG_params, "PARKWAY_SERPART", (void *) hgp->parkway_serpart); Zoltan_Bind_Param(PHG_params, "PHG_CUT_OBJECTIVE", (void *) &cut_objective); Zoltan_Bind_Param(PHG_params, "ADD_OBJ_WEIGHT", (void *) add_obj_weight); Zoltan_Bind_Param(PHG_params, "PHG_EDGE_WEIGHT_OPERATION", (void *) edge_weight_op); Zoltan_Bind_Param(PHG_params, "PHG_RANDOMIZE_INPUT", (void*) &hgp->RandomizeInitDist); Zoltan_Bind_Param(PHG_params, "PHG_PROCESSOR_REDUCTION_LIMIT", (void*) &hgp->ProRedL); Zoltan_Bind_Param(PHG_params, "PHG_REPART_MULTIPLIER", (void*) &hgp->RepartMultiplier); Zoltan_Bind_Param(PHG_params, "PATOH_ALLOC_POOL0", (void*) &hgp->patoh_alloc_pool0); Zoltan_Bind_Param(PHG_params, "PATOH_ALLOC_POOL1", (void*) &hgp->patoh_alloc_pool1); /* Set default values */ strncpy(hgp->hgraph_pkg, "phg", MAX_PARAM_STRING_LEN); strncpy(hgp->convert_str, "neighbors", MAX_PARAM_STRING_LEN); strncpy(hgp->redm_str, "agg", MAX_PARAM_STRING_LEN); hgp->match_array_type = 0; strncpy(hgp->redm_fast, "l-ipm", MAX_PARAM_STRING_LEN); strncpy(hgp->coarsepartition_str, "auto", MAX_PARAM_STRING_LEN); strncpy(hgp->refinement_str, "fm2", MAX_PARAM_STRING_LEN); strncpy(hgp->parkway_serpart, "patoh", MAX_PARAM_STRING_LEN); strncpy(cut_objective, "connectivity", MAX_PARAM_STRING_LEN); strncpy(add_obj_weight, "none", MAX_PARAM_STRING_LEN); strncpy(edge_weight_op, "max", MAX_PARAM_STRING_LEN); /* LB.Approach is initialized to "REPARTITION", and set in Set_Key_Params */ strncpy(hgp->hgraph_method, zz->LB.Approach, MAX_PARAM_STRING_LEN); if (!strcasecmp(zz->LB.Approach,"REFINE")) hgp->useMultilevel = 0; else hgp->useMultilevel = 1; hgp->use_timers = 0; hgp->LocalCoarsePartition = 0; hgp->edge_scaling = 0; hgp->vtx_scaling = 0; hgp->vtx_scal_size = 0; hgp->vtx_scal = NULL; /* Array for storing vertex degree scale vector. Should perhaps go in hg structure, not the param struct? */ hgp->connectivity_cut = 1; hgp->visit_order = 0; /* Random */ hgp->check_graph = 0; hgp->bal_tol = zz->LB.Imbalance_Tol[0]; /* Make vector for multiconstraint */ hgp->bal_tol_adjustment = 0.7; hgp->nCand = 100; hgp->redl = MAX(2*zz->LB.Num_Global_Parts, 100); hgp->output_level = PHG_DEBUG_NONE; hgp->final_output = 0; hgp->nProc_x_req = -1; hgp->nProc_y_req = -1; hgp->kway = 0; hgp->fm_loop_limit = 10; hgp->fm_max_neg_move = 250; hgp->refinement_quality = 1; hgp->RandomizeInitDist = 0; hgp->EdgeSizeThreshold = 0.25; hgp->MatchEdgeSizeThreshold = 500; hgp->hybrid_keep_factor = 0.; hgp->ProRedL = 0.0; /* UVCUVC: CHECK default set to 0 until we run more experiments */ hgp->RepartMultiplier = 100.; hgp->patoh_alloc_pool0 = 0; hgp->patoh_alloc_pool1 = 0; hgp->UseFixedVtx = 0; hgp->UsePrefPart = 0; /* Get application values of parameters. */ err = Zoltan_Assign_Param_Vals(zz->Params, PHG_params, zz->Debug_Level, zz->Proc, zz->Debug_Proc); nProc = zz->Num_Proc; usePrimeComm = 0; /* Parse add_obj_weight parameter */ if (!strcasecmp(add_obj_weight, "none")) { hgp->add_obj_weight = PHG_ADD_NO_WEIGHT; hgp->part_sizes = part_sizes; } else if (zz->Obj_Weight_Dim > 0) { /* Do not add_obj_weight until multiconstraint PHG is implemented */ ZOLTAN_PRINT_WARN(zz->Proc, yo, "Both application supplied *and* ADD_OBJ_WEIGHT " "calculated vertex weights were provided."); ZOLTAN_PRINT_WARN(zz->Proc, yo, "Only the first application supplied weight per vertex will be used."); hgp->add_obj_weight = PHG_ADD_NO_WEIGHT; hgp->part_sizes = part_sizes; } else { if (!strcasecmp(add_obj_weight, "vertices")){ hgp->add_obj_weight = PHG_ADD_UNIT_WEIGHT; } else if (!strcasecmp(add_obj_weight, "unit")){ hgp->add_obj_weight = PHG_ADD_UNIT_WEIGHT; } else if (!strcasecmp(add_obj_weight, "vertex degree")){ hgp->add_obj_weight = PHG_ADD_PINS_WEIGHT; } else if (!strcasecmp(add_obj_weight, "nonzeros")){ hgp->add_obj_weight = PHG_ADD_PINS_WEIGHT; } else if (!strcasecmp(add_obj_weight, "pins")){ hgp->add_obj_weight = PHG_ADD_PINS_WEIGHT; } else{ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid ADD_OBJ_WEIGHT parameter.\n"); err = ZOLTAN_WARN; } /* Set hgp->part_sizes to new array of part_sizes with added obj weight. */ if (part_sizes) err = Zoltan_LB_Add_Part_Sizes_Weight(zz, (zz->Obj_Weight_Dim ? zz->Obj_Weight_Dim : 1), zz->Obj_Weight_Dim+1, part_sizes, &(hgp->part_sizes)); } if ((zz->Obj_Weight_Dim==0) && /* no application supplied weights */ (hgp->add_obj_weight==PHG_ADD_NO_WEIGHT)){ /* no calculated weight */ hgp->add_obj_weight = PHG_ADD_UNIT_WEIGHT; /* default object weight */ } if (!strcasecmp(cut_objective, "default") || !strcasecmp(cut_objective, "connectivity")) hgp->connectivity_cut = 1; else if (!strcasecmp(cut_objective, "hyperedges")) hgp->connectivity_cut = 0; else { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid PHG_CUT_OBJECTIVE parameter.\n"); goto End; } if (!strcasecmp(edge_weight_op, "max")){ hgp->edge_weight_op = PHG_MAX_EDGE_WEIGHTS; } else if (!strcasecmp(edge_weight_op, "add")){ hgp->edge_weight_op = PHG_ADD_EDGE_WEIGHTS; } else if (!strcasecmp(edge_weight_op, "error")){ hgp->edge_weight_op = PHG_FLAG_ERROR_EDGE_WEIGHTS; } else{ ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Invalid PHG_EDGE_WEIGHT_OPERATION parameter.\n"); err = ZOLTAN_WARN; } if ((strcasecmp(method, "PARTITION")) && (strcasecmp(method, "REPARTITION")) && (strcasecmp(method, "REFINE"))) { sprintf(buf,"%s is not a valid hypergraph method\n",method); ZOLTAN_PRINT_ERROR (zz->Proc, yo, buf); err = ZOLTAN_FATAL; goto End; } /* Adjust refinement parameters using hgp->refinement_quality */ if (hgp->refinement_quality < 0.5/hgp->fm_loop_limit) /* No refinement */ strncpy(hgp->refinement_str, "no", MAX_PARAM_STRING_LEN); else { /* Scale FM parameters */ hgp->fm_loop_limit *= hgp->refinement_quality; hgp->fm_max_neg_move *= hgp->refinement_quality; } if (!strcasecmp(package, "PHG")){ /* Test to determine whether we should change the number of processors used for partitioning to make more efficient 2D decomposition */ if (hgp->nProc_x_req != 1 && hgp->nProc_y_req != 1) /* Want 2D decomp */ if (zz->Num_Proc > SMALL_PRIME && Zoltan_PHG_isPrime(zz->Num_Proc)) /* 2D data decomposition is requested but we have a prime * number of processors. */ usePrimeComm = 1; if ((!strcasecmp(method, "REPARTITION"))){ zz->LB.Remap_Flag = 0; } if ((!strcasecmp(method, "REPARTITION")) || (!strcasecmp(method, "REFINE"))) { hgp->fm_loop_limit = 4; /* experimental evaluation showed that for repartitioning/refinement small number of passes is "good enough". These are all heuristics hence it is possible to create a pathological cases; but in general this seems to be sufficient */ } if (!hgp->useMultilevel) { /* don't do coarsening */ strncpy(hgp->redm_str, "no", MAX_PARAM_STRING_LEN); /* we have modified all coarse partitioners to handle preferred part if user wants to choose one she can choose; otherwise default partitioner (greedy growing) does work better than previous default partitioning for phg_refine ("no"). */ hgp->UsePrefPart = 1; } if (!strcasecmp(method, "REFINE") && hgp->useMultilevel){ /* UVCUVC: as a heuristic we prefer local matching; in our experiments for IPDPS'07 and WileyChapter multilevel_refine didn't prove itself useful; it is too costly even with local matching hence it will not be be released yet (i.e. not in v3). */ strncpy(hgp->redm_str, "l-ipm", MAX_PARAM_STRING_LEN); hgp->UsePrefPart = 1; } } else if (!strcasecmp(package, "PARKWAY")){ if (hgp->nProc_x_req>1) { err = ZOLTAN_FATAL; ZOLTAN_PRINT_ERROR(zz->Proc, yo, "ParKway requires nProc_x=1 or -1."); goto End; } hgp->nProc_x_req = 1; } else if (!strcasecmp(package, "PATOH")){ if (zz->Num_Proc>1) { err = ZOLTAN_FATAL; ZOLTAN_PRINT_ERROR(zz->Proc, yo, "PaToH only works with Num_Proc=1."); goto End; } } if (!usePrimeComm) MPI_Comm_dup(zz->Communicator, &communicator); else { MPI_Group newgrp, zzgrp; nProc--; MPI_Comm_group(zz->Communicator, &zzgrp); MPI_Group_excl(zzgrp, 1, &nProc, &newgrp); MPI_Comm_create(zz->Communicator, newgrp, &communicator); MPI_Group_free(&newgrp); MPI_Group_free(&zzgrp); } err = Zoltan_PHG_Set_2D_Proc_Distrib(zz, communicator, zz->Proc, nProc, hgp->nProc_x_req, hgp->nProc_y_req, &hgp->globalcomm); if (err != ZOLTAN_OK) goto End; /* Convert strings to function pointers. */ err = Zoltan_PHG_Set_Part_Options (zz, hgp); End: return err; }
void SpParHelper::MemoryEfficientPSort(pair<KEY,VAL> * array, IT length, IT * dist, const MPI_Comm & comm) { int nprocs, myrank; MPI_Comm_size(comm, &nprocs); MPI_Comm_rank(comm, &myrank); int nsize = nprocs / 2; // new size if(nprocs < 1000) { bool excluded = false; if(dist[myrank] == 0) excluded = true; int nreals = 0; for(int i=0; i< nprocs; ++i) if(dist[i] != 0) ++nreals; if(nreals == nprocs) // general case { long * dist_in = new long[nprocs]; for(int i=0; i< nprocs; ++i) dist_in[i] = (long) dist[i]; vpsort::parallel_sort (array, array+length, dist_in, comm); delete [] dist_in; } else { long * dist_in = new long[nreals]; int * dist_out = new int[nprocs-nreals]; // ranks to exclude int indin = 0; int indout = 0; for(int i=0; i< nprocs; ++i) { if(dist[i] == 0) dist_out[indout++] = i; else dist_in[indin++] = (long) dist[i]; } #ifdef DEBUG ostringstream outs; outs << "To exclude indices: "; copy(dist_out, dist_out+indout, ostream_iterator<int>(outs, " ")); outs << endl; SpParHelper::Print(outs.str()); #endif MPI_Group sort_group, real_group; MPI_Comm_group(comm, &sort_group); MPI_Group_excl(sort_group, indout, dist_out, &real_group); MPI_Group_free(&sort_group); // The Create() function should be executed by all processes in comm, // even if they do not belong to the new group (in that case MPI_COMM_NULL is returned as real_comm?) // MPI::Intracomm MPI::Intracomm::Create(const MPI::Group& group) const; MPI_Comm real_comm; MPI_Comm_create(comm, real_group, &real_comm); if(!excluded) { vpsort::parallel_sort (array, array+length, dist_in, real_comm); MPI_Comm_free(&real_comm); } MPI_Group_free(&real_group); delete [] dist_in; delete [] dist_out; } } else { IT gl_median = accumulate(dist, dist+nsize, static_cast<IT>(0)); // global rank of the first element of the median processor sort(array, array+length); // re-sort because we might have swapped data in previous iterations int color = (myrank < nsize)? 0: 1; pair<KEY,VAL> * low = array; pair<KEY,VAL> * upp = array; GlobalSelect(gl_median, low, upp, array, length, comm); BipartiteSwap(low, array, length, nsize, color, comm); if(color == 1) dist = dist + nsize; // adjust for the second half of processors // recursive call; two implicit 'spawn's where half of the processors execute different paramaters // MPI::Intracomm MPI::Intracomm::Split(int color, int key) const; MPI_Comm halfcomm; MPI_Comm_split(comm, color, myrank, &halfcomm); // split into two communicators MemoryEfficientPSort(array, length, dist, halfcomm); } }
int main( int argc, char *argv[] ) { int numprocs, myid, server, workerid, ranks[1], request, i, iter, done; long rands[CHUNKSIZE], max, in, out, totalin, totalout; double x, y, Pi, error, epsilon; MPI_Comm world, workers; MPI_Group world_group, worker_group; MPI_Status status; MPI_Init( &argc, &argv ); world = MPI_COMM_WORLD; MPI_Comm_size( world, &numprocs ); MPI_Comm_rank( world, &myid ); server = numprocs-1; // Last process is a random server /*** * Now Master should read epsilon from command line * and distribute it to all processes. */ if (myid == 0) // Read epsilon from command line { sscanf( argv[1], "%lf", &epsilon ); }; // MPE_Start_log(); // MPE_Log_event(START_BCAST,0,"bcast epsilon"); MPI_Bcast( &epsilon, 1, MPI_DOUBLE, 0, MPI_COMM_WORLD ); // MPE_Log_event(END_BCAST,0,"bcast epsilon"); /*** * Create new process group called world_group containing all * processes and its communicator called world * and a group called worker_group containing all processes * except the last one (called here server) * and its communicator called workers. */ MPI_Comm_group( world, &world_group ); ranks[0] = server; MPI_Group_excl( world_group, 1, ranks, &worker_group ); MPI_Comm_create( world, worker_group, &workers ); MPI_Group_free( &worker_group ); /*** * Server part * * Server should loop until request code is 0, in each iteration: * - receiving request code from any slave * - generating a vector of CHUNKSIZE randoms <= INT_MAX * - sending vector back to slave */ if (myid == server) { // I am the random generator server do { MPI_Recv( &request, 1, MPI_INT, MPI_ANY_SOURCE, REQUEST, world, &status ); if (request) { for (i = 0; i < CHUNKSIZE; ) { rands[i] = random(); if ( rands[i] <= INT_MAX ) i++; } MPI_Send( rands, CHUNKSIZE, MPI_LONG, status.MPI_SOURCE, REPLY, world ); } } while( request > 0 ); } /*** * Workers (including Master) part * * Worker should send initial request to server. * Later, in a loop worker should: * - receive vector of randoms * - compute x,y point inside unit square * - check (and count result) if point is inside/outside * unit circle * - sum both counts over all workers * - calculate pi and its error (from "exact" value) * - test if error is within epsilon limit * - test continuation condition (error and max. points limit) * - print pi by master only * - send a request to server (all if more or master only if finish) * Before finishing workers should free their communicator. */ else { // I am a worker process request = 1; done = 0; in = out = 0; max = INT_MAX; // max int, for normalization MPI_Send( &request, 1, MPI_INT, server, REQUEST, world ); MPI_Comm_rank( workers, &workerid ); iter = 0; while (!done) { iter++; request = 1; MPI_Recv( rands, CHUNKSIZE, MPI_LONG, server, REPLY, world, &status ); for (i = 0; i < CHUNKSIZE - 1; ) { x = (((double) rands[i++])/max) * 2 - 1; y = (((double) rands[i++])/max) * 2 - 1; if ( x*x + y*y < 1.0 ) { in++; } else out++; } MPI_Allreduce( &in, &totalin, 1, MPI_LONG, MPI_SUM, workers ); MPI_Allreduce( &out, &totalout, 1, MPI_LONG, MPI_SUM, workers ); Pi = ( 4.0 * totalin ) / ( totalin + totalout ); error = fabs( Pi - PI ); done = ( error < epsilon || (totalin + totalout) > THROW_MAX ); request = (done) ? 0 : 1; if (myid == 0) { printf( "\rpi = %23.20f", Pi ); MPI_Send( &request, 1, MPI_INT, server, REQUEST, world ); } else { if (request) MPI_Send( &request, 1, MPI_INT, server, REQUEST, world ); } } MPI_Comm_free( &workers ); } /*** * Master should print final point counts. */ if (myid == 0) { printf( "\npoints: %ld\nin: %ld, out: %ld, <ret> to exit\n", totalin+totalout, totalin, totalout ); getchar(); } MPI_Finalize(); return 0; }