int main( int argc, char **argv ) { int *sendbuf, *recvcounts; int block_size; int *recvbuf; int size, rank, i; MPI_Comm comm; MPI_Op left_op, right_op, nc_sum_op; MTest_Init( &argc, &argv ); comm = MPI_COMM_WORLD; MPI_Comm_size( comm, &size ); MPI_Comm_rank( comm, &rank ); MPI_Op_create(&left, 0/*non-commutative*/, &left_op); MPI_Op_create(&right, 0/*non-commutative*/, &right_op); MPI_Op_create(&nc_sum, 0/*non-commutative*/, &nc_sum_op); for (block_size = 1; block_size < MAX_BLOCK_SIZE; block_size *= 2) { sendbuf = (int *) malloc( block_size * size * sizeof(int) ); recvbuf = malloc( block_size * sizeof(int) ); for (i=0; i<(size*block_size); i++) sendbuf[i] = rank + i; for (i=0; i<block_size; i++) recvbuf[i] = 0xdeadbeef; recvcounts = (int *)malloc( size * sizeof(int) ); for (i=0; i<size; i++) recvcounts[i] = block_size; MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_INT, left_op, comm ); for (i = 0; i < block_size; ++i) if (recvbuf[i] != (rank * block_size + i)) ++err; MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_INT, right_op, comm ); for (i = 0; i < block_size; ++i) if (recvbuf[i] != ((size - 1) + (rank * block_size) + i)) ++err; MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_INT, nc_sum_op, comm ); for (i = 0; i < block_size; ++i) { int x = rank * block_size + i; if (recvbuf[i] != (size*x + (size-1)*size/2)) ++err; } free(recvbuf); free(sendbuf); } MPI_Op_free(&left_op); MPI_Op_free(&right_op); MPI_Op_free(&nc_sum_op); MTest_Finalize( err ); MPI_Finalize( ); return err; }
int main(int argc, char **argv) { MPI_Init(&argc, &argv); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); int *nsizes = (int*) malloc(size * size); int total = 0; for (int i = 0; i < size; i++) { // ERROR HERE, different sizes for each process nsizes[i] = (i % 3 + 1) + rank; total += nsizes[i]; } int *data = (int*) malloc(sizeof(int) * total); int *out = (int*) malloc(sizeof(int) * nsizes[rank]); for (int i = 0; i < total; i++) { data[i] = (rank + 1) * 100 + i; } MPI_Reduce_scatter(data, out, nsizes, MPI_INT, MPI_SUM, MPI_COMM_WORLD); free(data); for (int i = 0; i < nsizes[rank]; i++) { printf("%i %i\n", rank, out[i]); } free(out); MPI_Finalize(); return 0; }
void SparseMsg::setPattern(UInt num, const UInt *proc) { UInt csize = Par::Size(); // Set dest proc nsend = num; std::vector<int> sendto(nproc, 0); std::vector<int> counts(nproc, 1); for (UInt i = 0; i < num; i++) { ThrowRequire(proc[i] < csize); sendto[proc[i]] = 1; if (proc[i] == (UInt) rank) { sendself = true; self_idx = i; } } !Par::Serial() ? MPI_Reduce_scatter(&sendto[0], &num_incoming, &counts[0], MPI_INT, MPI_SUM, comm) : num_incoming = sendto[0]; //std::cout << "Proc:" << rank << "to receive " << num_incoming << " messages" << std::endl; // Set up send buffers (so we don't have save the proc ids if (nsend > 0) outBuffers.resize(nsend); else outBuffers.clear(); for (UInt i = 0; i < nsend; i++) { outBuffers[i].proc = proc[i]; } }
void mpi_reduce_scatter_f(char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr) { MPI_Comm c_comm; MPI_Datatype c_type; MPI_Op c_op; int size; OMPI_ARRAY_NAME_DECL(recvcounts); c_comm = MPI_Comm_f2c(*comm); c_type = MPI_Type_f2c(*datatype); c_op = MPI_Op_f2c(*op); MPI_Comm_size(c_comm, &size); OMPI_ARRAY_FINT_2_INT(recvcounts, size); sendbuf = (char *) OMPI_F2C_IN_PLACE(sendbuf); sendbuf = (char *) OMPI_F2C_BOTTOM(sendbuf); recvbuf = (char *) OMPI_F2C_BOTTOM(recvbuf); *ierr = OMPI_INT_2_FINT(MPI_Reduce_scatter(sendbuf, recvbuf, OMPI_ARRAY_NAME_CONVERT(recvcounts), c_type, c_op, c_comm)); }
int main(int argc,char *argv[]) { int rank, num_of_processes; int *rdata,*sdata,*rcounts,*expected_data; int i,j; MPI_Comm comm = MPI_COMM_WORLD; MPI_Init(&argc,&argv); MPI_Comm_size( comm, &num_of_processes); MPI_Comm_rank( comm, &rank); sdata = malloc(sizeof(int)*num_of_processes); rcounts = malloc(sizeof(int)*num_of_processes); // malloc more than necessary .. its a test rdata = malloc(sizeof(int)*num_of_processes); expected_data = malloc(sizeof(int)*num_of_processes); /** * Building an send arrays like the following. * rank_0 = 0 1 2 * rank_1 = 1 2 3 * rank_2 = 2 3 4 * * After reduce the receive buffers should each get equal number of items * rank_0 = 3 * rank_1 = 6 * rank_2 = 9 * * **/ for(i=0; i<num_of_processes; i++) { sdata[i] = rank+i; expected_data[i] = 0; rcounts[i] = 1; } if(rank == 0) { printf("Checking mpi_reduce_scatter... (if you see no output then you are good)\n"); } // need to get expected values! for(i=0; i<num_of_processes; i++) { for(j=0; j<rcounts[i]; j++) { expected_data[j] += i+rank; } } MPI_Reduce_scatter( sdata, rdata, rcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); for(j=0; j<rcounts[rank]; j++) { if(rdata[j] != expected_data[j]) { printf("ERROR: Received at rank %d rdata[%d] = %d expected %d\n", rank, j, rdata[j], expected_data[j]); } } MPI_Finalize(); return 0; }
int PLA_MPI_Reduce_scatter( /* aka Distributed Reduce */ void * sendbuf, void * recvbuf, int * rcounts, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) { return (MPI_Reduce_scatter ( sendbuf, recvbuf, rcounts, datatype, op, comm )); }
// MPI_Allreduce with Reduce_scatter and Allgather inline void execute_GL_Allreduce_as_ReducescatterAllgather(collective_params_t* params) { MPI_Reduce_scatter(params->sbuf, params->tmp_buf, params->counts_array, params->datatype, params->op, MPI_COMM_WORLD); MPI_Allgather(params->tmp_buf, params->count, params->datatype, params->rbuf, params->count, params->datatype, MPI_COMM_WORLD); }
//------------------------------------------------------------------------ int mirrorProcs(MPI_Comm comm, std::vector<int>& toProcs, std::vector<int>& fromProcs) { fromProcs.resize(0); #ifdef FEI_SER fromProcs.push_back(0); return(0); #else int num_procs = fei::numProcs(comm); std::vector<int> tmpIntData(num_procs*3, 0); int* buf = &tmpIntData[0]; int* recvbuf = buf+num_procs; for(unsigned i=0; i<toProcs.size(); ++i) { buf[toProcs[i]] = 1; } for(int ii=2*num_procs; ii<3*num_procs; ++ii) { buf[ii] = 1; } CHK_MPI( MPI_Reduce_scatter(buf, &(buf[num_procs]), &(buf[2*num_procs]), MPI_INT, MPI_SUM, comm) ); int numRecvProcs = buf[num_procs]; int tag = 11116; std::vector<MPI_Request> mpiReqs(numRecvProcs); int offset = 0; for(int ii=0; ii<numRecvProcs; ++ii) { CHK_MPI( MPI_Irecv(&(recvbuf[ii]), 1, MPI_INT, MPI_ANY_SOURCE, tag, comm, &(mpiReqs[offset++])) ); } for(unsigned i=0; i<toProcs.size(); ++i) { CHK_MPI( MPI_Send(&(toProcs[i]), 1, MPI_INT, toProcs[i], tag, comm) ); } MPI_Status status; for(int ii=0; ii<numRecvProcs; ++ii) { int index; MPI_Waitany(numRecvProcs, &mpiReqs[0], &index, &status); fromProcs.push_back(status.MPI_SOURCE); } std::sort(fromProcs.begin(), fromProcs.end()); return(0); #endif }
void compute_reduce_scatter(ATYPE *matrix, ATYPE *vector, ATYPE *result, int local_rank, int proc_size , long N, long partition) { ATYPE *temp_result = init_vector(N,0); int recvcounts[proc_size]; for( int i=0; i < proc_size; i++) { recvcounts[i] = N; } for ( int i = 0 ; i < partition ; ++i ){ for ( int j = 0 ; j < N ; ++j ){ temp_result[j] = temp_result[j] + matrix[i*N+j] * vector[i]; } } MPI_Reduce_scatter(temp_result, result, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD); }
int main( int argc, char **argv ) { int err = 0, toterr; int *sendbuf, *recvbuf, *recvcounts; int size, rank, i, sumval; MPI_Comm comm; MPI_Init( &argc, &argv ); comm = MPI_COMM_WORLD; MPI_Comm_size( comm, &size ); MPI_Comm_rank( comm, &rank ); sendbuf = (int *) malloc( size * sizeof(int) ); for (i=0; i<size; i++) sendbuf[i] = rank + i; recvcounts = (int *)malloc( size * sizeof(int) ); recvbuf = (int *)malloc( size * sizeof(int) ); for (i=0; i<size; i++) recvcounts[i] = 1; MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_INT, MPI_SUM, comm ); sumval = size * rank + ((size - 1) * size)/2; /* recvbuf should be size * (rank + i) */ if (recvbuf[0] != sumval) { err++; fprintf( stdout, "Did not get expected value for reduce scatter\n" ); fprintf( stdout, "[%d] Got %d expected %d\n", rank, recvbuf[0], sumval ); } MPI_Allreduce( &err, &toterr, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); if (rank == 0 && toterr == 0) { printf( " No Errors\n" ); } free(sendbuf); free(recvcounts); free(recvbuf); MPI_Finalize( ); return toterr; }
int main(int argc, char **argv) { MPI_Init(&argc, &argv); int rank, num_procs; MPI_Comm_size(MPI_COMM_WORLD, &num_procs); MPI_Comm_rank(MPI_COMM_WORLD, &rank); if(num_procs < 4) { MPI_Finalize(); return 1; } srand(rank); int *buffer = (int *)malloc(num_procs * sizeof(int)); int i; printf("process %d:\n", rank); for (i = 0; i < num_procs; i++) { buffer[i] = rand() % 10; printf("%d ", buffer[i]); } printf("\n"); int local_sum = 0; for (i = 0; i < num_procs; i++) { local_sum += buffer[i]; } printf("local sum of process %d is %d\n", rank, local_sum); int *recv_counts = (int *)malloc(num_procs * sizeof(int)); for (i = 0; i < num_procs; i++) { recv_counts[i] = 1; } int recv_buffer = 0; MPI_Reduce_scatter(buffer, &recv_buffer, recv_counts, MPI_INT, MPI_SUM, MPI_COMM_WORLD); printf("recv buffer of process %d is %d\n", rank, recv_buffer); MPI_Finalize(); }
/* * Class: mpi_Intracomm * Method: Reduce_scatter * Signature: (Ljava/lang/Object;ILjava/lang/Object;I[ILmpi/Datatype;Lmpi/Op;)V */ JNIEXPORT void JNICALL Java_mpi_Intracomm_reduce_1scatter(JNIEnv *env, jobject jthis, jobject sendbuf, jint sendoffset, jobject recvbuf, jint recvoffset, jintArray recvcount, jobject type, jobject op) { jint *rcount; jboolean isCopy ; MPI_Comm mpi_comm = (MPI_Comm)((*env)->GetLongField(env,jthis,ompi_java.CommhandleID)) ; MPI_Datatype mpi_type = (MPI_Datatype)((*env)->GetLongField(env,type,ompi_java.DatatypehandleID)) ; int baseType = (*env)->GetIntField(env, type, ompi_java.DatatypebaseTypeID) ; void *sendptr, *recvptr ; void *sbufbase, *rbufbase ; ompi_java_clearFreeList(env) ; rcount=(*env)->GetIntArrayElements(env,recvcount,&isCopy); recvptr = ompi_java_getBufPtr(&rbufbase, env, recvbuf, baseType, recvoffset) ; sendptr = ompi_java_getBufPtr(&sbufbase, env, sendbuf, baseType, sendoffset) ; MPI_Reduce_scatter(sendptr, recvptr, (int*) rcount, mpi_type, (MPI_Op)((*env)->GetLongField(env,op,ompi_java.OphandleID)), mpi_comm) ; ompi_java_releaseBufPtr(env, sendbuf, sbufbase, baseType) ; ompi_java_releaseBufPtr(env, recvbuf, rbufbase, baseType) ; (*env)->ReleaseIntArrayElements(env,recvcount,rcount,JNI_ABORT); }
int ZMPI_Reduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) /* zmpi_func ZMPI_Reduce_scatter_block */ { #if MPI_VERSION >= 2 && MPI_SUBVERSION >= 2 return MPI_Reduce_scatter_block((void *) sendbuf, recvbuf, recvcount, datatype,op, comm); #else int comm_size, *recvcounts, i, exit_code; MPI_Comm_size(comm, &comm_size); recvcounts = z_alloc(comm_size, sizeof(int)); for (i = 0; i < comm_size; ++i) recvcounts[i] = recvcount; exit_code = MPI_Reduce_scatter((void *) sendbuf, recvbuf, recvcounts, datatype, op, comm); z_free(recvcounts); return exit_code; #endif }
void mpi_reduce_scatter_(void* sendbuf, void* recvbuf, int* recvcounts, int* datatype, int* op, int* comm, int* ierr) { *ierr = MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, get_datatype(*datatype), get_op(*op), get_comm(*comm)); }
int main(int argc, char *argv[]) { int provided; int size, rank; int rc; MPI_Init_thread( &argc, &argv, MPI_THREAD_SINGLE, &provided ); //assert( provided == MPI_THREAD_SINGLE ); MPI_Comm_size( MPI_COMM_WORLD, &size ); MPI_Comm_rank( MPI_COMM_WORLD, &rank ); //printf( "Hello from %d of %d processors\n", rank, size ); //fflush( stdout ); int count = ( argc > 1 ? atoi(argv[1]) : 1 ); MPI_Barrier( MPI_COMM_WORLD ); /* reduce_scatter bandwidth test */ if ( rank == 0 ) printf( "begin reduce_scatter bandwidth test\n" ); { int i; double t0, t1; int * snd_buffer; int * rcv_buffer; int * counts; snd_buffer = malloc( size * count * sizeof(int) ); assert( snd_buffer != NULL ); rcv_buffer = malloc( count * sizeof(int) ); assert( rcv_buffer != NULL ); counts = malloc( size * sizeof(int) ); assert( counts != NULL ); for ( i = 0 ; i < size ; i++ ) snd_buffer[i] = 0; for ( i = 0 ; i < count ; i++ ) snd_buffer[ rank * count + i] = rank; for ( i = 0 ; i < count ; i++ ) rcv_buffer[i] = 0; for ( i = 0 ; i < size ; i++) counts[i] = count; MPI_Barrier( MPI_COMM_WORLD ); t0 = MPI_Wtime(); rc = MPI_Reduce_scatter( snd_buffer, rcv_buffer, counts, MPI_INT, MPI_SUM, MPI_COMM_WORLD ); t1 = MPI_Wtime(); assert ( rc == MPI_SUCCESS ); for ( i = 0 ; i < count ; i++ ) assert( rcv_buffer[i] == rank ); if ( rank == 0 ) printf( "MPI_Reduce_scatter(MPI_SUM): %u bytes transferred in %lf seconds (%lf MB/s)\n", count * (int) sizeof(int), t1 - t0, 1e-6 * count * (int) sizeof(int) / (t1-t0) ); free(snd_buffer); free(rcv_buffer); free(counts); } if ( rank == 0 ) printf( "done with all tests\n" ); fflush( stdout ); MPI_Barrier( MPI_COMM_WORLD ); MPI_Finalize(); return 0; }
void IMB_reduce_scatter(struct comm_info* c_info, int size, struct iter_schedule* ITERATIONS, MODES RUN_MODE, double* time) /* MPI-1 benchmark kernel Benchmarks MPI_Reduce_scatter Input variables: -c_info (type struct comm_info*) Collection of all base data for MPI; see [1] for more information -size (type int) Basic message size in bytes -ITERATIONS (type struct iter_schedule *) Repetition scheduling -RUN_MODE (type MODES) (only MPI-2 case: see [1]) Output variables: -time (type double*) Timing result per sample */ { double t1, t2; int i; size_t pos1,pos2; #ifdef CHECK size_t pos; int Locsize; #endif Type_Size s_size; #ifdef CHECK defect=0.; #endif ierr = 0; /* GET SIZE OF DATA TYPE */ MPI_Type_size(c_info->red_data_type,&s_size); for (i=0;i<c_info->num_procs ;i++) { if( size > 0) { IMB_get_rank_portion(i, c_info->num_procs, size, s_size, &pos1, &pos2); c_info->reccnt[i] = (pos2-pos1+1)/s_size; #ifdef CHECK if( i==c_info->rank ) {pos=pos1; Locsize= s_size*c_info->reccnt[i];} #endif } else { c_info->reccnt[i] = 0; #ifdef CHECK if( i==c_info->rank ) {pos=0; Locsize= 0;} #endif } } if(c_info->rank!=-1) { for(i=0; i<N_BARR; i++) MPI_Barrier(c_info->communicator); t1 = MPI_Wtime(); for(i=0;i< ITERATIONS->n_sample;i++) { ierr = MPI_Reduce_scatter ((char*)c_info->s_buffer+i%ITERATIONS->s_cache_iter*ITERATIONS->s_offs, (char*)c_info->r_buffer+i%ITERATIONS->r_cache_iter*ITERATIONS->r_offs, c_info->reccnt, c_info->red_data_type,c_info->op_type, c_info->communicator); MPI_ERRHAND(ierr); CHK_DIFF("Reduce_scatter",c_info, (char*)c_info->r_buffer+i%ITERATIONS->r_cache_iter*ITERATIONS->r_offs, pos, Locsize, size, asize, put, 0, ITERATIONS->n_sample, i, -1, &defect); } t2 = MPI_Wtime(); *time=(t2 - t1)/ITERATIONS->n_sample; } else /*if(c_info->rank==-1)*/ { *time = 0.; } }
void MpiComm<Ordinal>::reduceAllAndScatter( const ValueTypeReductionOp<Ordinal,char> &reductOp ,const Ordinal sendBytes, const char sendBuffer[] ,const Ordinal recvCounts[], char myGlobalReducts[] ) const { (void)sendBytes; // Ignore if not in debug mode TEUCHOS_COMM_TIME_MONITOR( "Teuchos::MpiComm<"<<OrdinalTraits<Ordinal>::name()<<">::reduceAllAndScatter(...)" ); #ifdef TEUCHOS_DEBUG Ordinal sumRecvBytes = 0; for( Ordinal i = 0; i < size_; ++i ) { sumRecvBytes += recvCounts[i]; } TEST_FOR_EXCEPT(!(sumRecvBytes==sendBytes)); #endif // TEUCHOS_DEBUG #ifdef TEUCHOS_MPI_COMM_DUMP if(show_dump) { dumpBuffer<Ordinal,char>( "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)", "sendBuffer", sendBytes, sendBuffer ); dumpBuffer<Ordinal,Ordinal>( "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)", "recvCounts", as<Ordinal>(size_), recvCounts ); dumpBuffer<Ordinal,char>( "Teuchos::MpiComm<Ordinal>::reduceAllAndScatter(...)", "myGlobalReducts", as<char>(recvCounts[rank_]), myGlobalReducts ); } #endif // TEUCHOS_MPI_COMM_DUMP // Create a new recvCount[] if Ordinal!=int WorkspaceStore* wss = get_default_workspace_store().get(); const bool Ordinal_is_int = typeid(int)==typeid(Ordinal); Workspace<int> ws_int_recvCounts(wss,Ordinal_is_int?0:size_); const int *int_recvCounts = 0; if(Ordinal_is_int) { int_recvCounts = reinterpret_cast<const int*>(recvCounts); // Note: We must do an reinterpet cast since this must // compile even if it is not executed. I could implement // code that would not need to do this using template // conditionals but I don't want to bother. } else { std::copy(recvCounts, recvCounts+size_, &ws_int_recvCounts[0]); int_recvCounts = &ws_int_recvCounts[0]; } // Perform the operation MpiReductionOpSetter op(mpiReductionOp(rcp(&reductOp, false))); MPI_Reduce_scatter( const_cast<char*>(sendBuffer), myGlobalReducts, const_cast<int*>(int_recvCounts), MPI_CHAR, op.mpi_op(), *rawMpiComm_ ); }
int main(int argc, char* argv[]) { MPI_Init(&argc, &argv); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if (size!=4) { if (rank==0) printf("Use 4 processes\n"); MPI_Finalize(); return size; } { if (rank==0) printf("MPI_Reduce_scatter(sendbuf, recvbuf...\n"); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); int junk = rank+1; int sendbuf[4] = {junk, junk*2, junk*3, junk*4}; int recvbuf[1] = {0}; int recvcounts[4] = {1,1,1,1}; MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD); printf("%d: sendbuf = {%d,%d,%d,%d}, recvbuf = {%d} \n", rank, sendbuf[0], sendbuf[1], sendbuf[2], sendbuf[3], recvbuf[0]); } fflush(stdout); usleep(1000); MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("===================\n"); { if (rank==0) printf("MPI_Reduce_scatter(MPI_IN_PLACE, recvbuf...\n"); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); int junk = rank+1; int recvbuf[4] = {junk, junk*2, junk*3, junk*4}; int recvcounts[4] = {1,1,1,1}; MPI_Reduce_scatter(MPI_IN_PLACE, recvbuf, recvcounts, MPI_INT, MPI_SUM, MPI_COMM_WORLD); printf("%d: recvbuf = {%d,%d,%d,%d} \n", rank, recvbuf[0], recvbuf[1], recvbuf[2], recvbuf[3]); } fflush(stdout); usleep(1000); MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("===================\n"); { if (rank==0) printf("MPI_Reduce_scatter_block(sendbuf, recvbuf...\n"); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); int junk = rank+1; int sendbuf[4] = {junk, junk*2, junk*3, junk*4}; int recvbuf[1] = {0}; int recvcount = 1; MPI_Reduce_scatter_block(sendbuf, recvbuf, recvcount, MPI_INT, MPI_SUM, MPI_COMM_WORLD); printf("%d: sendbuf = {%d,%d,%d,%d}, recvbuf = {%d} \n", rank, sendbuf[0], sendbuf[1], sendbuf[2], sendbuf[3], recvbuf[0]); } fflush(stdout); usleep(1000); MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("===================\n"); { if (rank==0) printf("MPI_Reduce_scatter_block(MPI_IN_PLACE, recvbuf...\n"); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); int junk = rank+1; int recvbuf[4] = {junk, junk*2, junk*3, junk*4}; int recvcount = 1; MPI_Reduce_scatter_block(MPI_IN_PLACE, recvbuf, recvcount, MPI_INT, MPI_SUM, MPI_COMM_WORLD); printf("%d: recvbuf = {%d,%d,%d,%d} \n", rank, recvbuf[0], recvbuf[1], recvbuf[2], recvbuf[3]); } fflush(stdout); usleep(1000); MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("===================\n"); { if (rank==0) printf("MPI_Reduce(sendbuf, tempbuf... + MPI_Scatter(tempbuf, recvcount...\n"); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); int junk = rank+1; int sendbuf[4] = {junk, junk*2, junk*3, junk*4}; int tempbuf[4] = {0,0,0,0}; int recvbuf[1] = {0}; int recvcount = 1; MPI_Reduce(sendbuf, tempbuf, 4*recvcount, MPI_INT, MPI_SUM, 0 /* root */, MPI_COMM_WORLD); MPI_Scatter(tempbuf, recvcount, MPI_INT, recvbuf, recvcount, MPI_INT, 0 /* root */, MPI_COMM_WORLD); printf("%d: sendbuf = {%d,%d,%d,%d}, recvbuf = {%d} \n", rank, sendbuf[0], sendbuf[1], sendbuf[2], sendbuf[3], recvbuf[0]); } fflush(stdout); usleep(1000); MPI_Barrier(MPI_COMM_WORLD); if (rank==0) printf("===================\n"); { if (rank==0) printf("MPI_Reduce(MPI_IN_PLACE, recvbuf... + MPI_Scatter(MPI_IN_PLACE, recvcount...\n"); fflush(stdout); MPI_Barrier(MPI_COMM_WORLD); int junk = rank+1; int recvbuf[4] = {junk, junk*2, junk*3, junk*4}; int recvcount = 1; MPI_Reduce(rank==0 ? MPI_IN_PLACE : recvbuf, rank==0 ? recvbuf : NULL, 4*recvcount, MPI_INT, MPI_SUM, 0 /* root */, MPI_COMM_WORLD); MPI_Scatter(recvbuf, recvcount, MPI_INT, rank==0 ? MPI_IN_PLACE : recvbuf, recvcount, MPI_INT, 0 /* root */, MPI_COMM_WORLD); printf("%d: recvbuf = {%d,%d,%d,%d} \n", rank, recvbuf[0], recvbuf[1], recvbuf[2], recvbuf[3]); } MPI_Finalize(); return 0; }
FC_FUNC(mpi_reduce_scatter, MPI_REDUCE_SCATTER) (void * sendbuf, void * recvbuf, int *recvcounts, int *datatype, int *op, int *comm, int *ierr) { *ierr = MPI_Reduce_scatter(sendbuf, recvbuf, recvcounts, *datatype, *op, *comm); }
int main(int argc, char **argv) { int errs = 0; int i; int rank, size; int *sbuf = NULL; int *rbuf = NULL; int *scounts = NULL; int *rcounts = NULL; int *sdispls = NULL; int *rdispls = NULL; MPI_Datatype *types = NULL; MPI_Comm comm; /* intentionally not using MTest_Init/MTest_Finalize in order to make it * easy to take this test and use it as an NBC sanity test outside of the * MPICH test suite */ MPI_Init(&argc, &argv); comm = MPI_COMM_WORLD; MPI_Comm_size(comm, &size); MPI_Comm_rank(comm, &rank); MPI_Comm_set_errhandler(MPI_COMM_WORLD, MPI_ERRORS_RETURN); /* enough space for every process to contribute at least NUM_INTS ints to any * collective operation */ sbuf = malloc(NUM_INTS * size * sizeof(int)); my_assert(sbuf); rbuf = malloc(NUM_INTS * size * sizeof(int)); my_assert(rbuf); scounts = malloc(size * sizeof(int)); my_assert(scounts); rcounts = malloc(size * sizeof(int)); my_assert(rcounts); sdispls = malloc(size * sizeof(int)); my_assert(sdispls); rdispls = malloc(size * sizeof(int)); my_assert(rdispls); types = malloc(size * sizeof(MPI_Datatype)); my_assert(types); for (i = 0; i < size; ++i) { sbuf[2 * i] = i; sbuf[2 * i + 1] = i; rbuf[2 * i] = i; rbuf[2 * i + 1] = i; scounts[i] = NUM_INTS; rcounts[i] = NUM_INTS; sdispls[i] = i * NUM_INTS; rdispls[i] = i * NUM_INTS; types[i] = MPI_INT; } if (rank == 0 && MPI_SUCCESS == MPI_Gather(sbuf, NUM_INTS, MPI_INT, sbuf, NUM_INTS, MPI_INT, 0, comm)) errs++; if (rank == 0 && MPI_SUCCESS == MPI_Gatherv(sbuf, NUM_INTS, MPI_INT, sbuf, rcounts, rdispls, MPI_INT, 0, comm)) errs++; if (rank == 0 && MPI_SUCCESS == MPI_Scatter(sbuf, NUM_INTS, MPI_INT, sbuf, NUM_INTS, MPI_INT, 0, comm)) errs++; if (rank == 0 && MPI_SUCCESS == MPI_Scatterv(sbuf, scounts, sdispls, MPI_INT, sbuf, NUM_INTS, MPI_INT, 0, comm)) errs++; if (MPI_SUCCESS == MPI_Allgather(&sbuf[rank], 1, MPI_INT, sbuf, 1, MPI_INT, comm)) errs++; if (MPI_SUCCESS == MPI_Allgatherv(&sbuf[rank * rcounts[rank]], rcounts[rank], MPI_INT, sbuf, rcounts, rdispls, MPI_INT, comm)) errs++; if (MPI_SUCCESS == MPI_Alltoall(sbuf, NUM_INTS, MPI_INT, sbuf, NUM_INTS, MPI_INT, comm)) errs++; if (MPI_SUCCESS == MPI_Alltoallv(sbuf, scounts, sdispls, MPI_INT, sbuf, scounts, sdispls, MPI_INT, comm)) errs++; if (MPI_SUCCESS == MPI_Alltoallw(sbuf, scounts, sdispls, types, sbuf, scounts, sdispls, types, comm)) errs++; if (rank == 0 && MPI_SUCCESS == MPI_Reduce(sbuf, sbuf, NUM_INTS, MPI_INT, MPI_SUM, 0, comm)) errs++; if (MPI_SUCCESS == MPI_Allreduce(sbuf, sbuf, NUM_INTS, MPI_INT, MPI_SUM, comm)) errs++; if (MPI_SUCCESS == MPI_Reduce_scatter(sbuf, sbuf, rcounts, MPI_INT, MPI_SUM, comm)) errs++; if (MPI_SUCCESS == MPI_Reduce_scatter_block(sbuf, sbuf, NUM_INTS, MPI_INT, MPI_SUM, comm)) errs++; if (MPI_SUCCESS == MPI_Scan(sbuf, sbuf, NUM_INTS, MPI_INT, MPI_SUM, comm)) errs++; if (MPI_SUCCESS == MPI_Exscan(sbuf, sbuf, NUM_INTS, MPI_INT, MPI_SUM, comm)) errs++; if (sbuf) free(sbuf); if (rbuf) free(rbuf); if (scounts) free(scounts); if (rcounts) free(rcounts); if (sdispls) free(sdispls); if (rdispls) free(rdispls); if (types) free(types); if (rank == 0) { if (errs) fprintf(stderr, "Found %d errors\n", errs); else printf(" No errors\n"); } MPI_Finalize(); return 0; }
void declareBindings (void) { /* === Point-to-point === */ void* buf; int count; MPI_Datatype datatype; int dest; int tag; MPI_Comm comm; MPI_Send (buf, count, datatype, dest, tag, comm); // L12 int source; MPI_Status status; MPI_Recv (buf, count, datatype, source, tag, comm, &status); // L15 MPI_Get_count (&status, datatype, &count); MPI_Bsend (buf, count, datatype, dest, tag, comm); MPI_Ssend (buf, count, datatype, dest, tag, comm); MPI_Rsend (buf, count, datatype, dest, tag, comm); void* buffer; int size; MPI_Buffer_attach (buffer, size); // L22 MPI_Buffer_detach (buffer, &size); MPI_Request request; MPI_Isend (buf, count, datatype, dest, tag, comm, &request); // L25 MPI_Ibsend (buf, count, datatype, dest, tag, comm, &request); MPI_Issend (buf, count, datatype, dest, tag, comm, &request); MPI_Irsend (buf, count, datatype, dest, tag, comm, &request); MPI_Irecv (buf, count, datatype, source, tag, comm, &request); MPI_Wait (&request, &status); int flag; MPI_Test (&request, &flag, &status); // L32 MPI_Request_free (&request); MPI_Request* array_of_requests; int index; MPI_Waitany (count, array_of_requests, &index, &status); // L36 MPI_Testany (count, array_of_requests, &index, &flag, &status); MPI_Status* array_of_statuses; MPI_Waitall (count, array_of_requests, array_of_statuses); // L39 MPI_Testall (count, array_of_requests, &flag, array_of_statuses); int incount; int outcount; int* array_of_indices; MPI_Waitsome (incount, array_of_requests, &outcount, array_of_indices, array_of_statuses); // L44--45 MPI_Testsome (incount, array_of_requests, &outcount, array_of_indices, array_of_statuses); // L46--47 MPI_Iprobe (source, tag, comm, &flag, &status); // L48 MPI_Probe (source, tag, comm, &status); MPI_Cancel (&request); MPI_Test_cancelled (&status, &flag); MPI_Send_init (buf, count, datatype, dest, tag, comm, &request); MPI_Bsend_init (buf, count, datatype, dest, tag, comm, &request); MPI_Ssend_init (buf, count, datatype, dest, tag, comm, &request); MPI_Rsend_init (buf, count, datatype, dest, tag, comm, &request); MPI_Recv_init (buf, count, datatype, source, tag, comm, &request); MPI_Start (&request); MPI_Startall (count, array_of_requests); void* sendbuf; int sendcount; MPI_Datatype sendtype; int sendtag; void* recvbuf; int recvcount; MPI_Datatype recvtype; MPI_Datatype recvtag; MPI_Sendrecv (sendbuf, sendcount, sendtype, dest, sendtag, recvbuf, recvcount, recvtype, source, recvtag, comm, &status); // L67--69 MPI_Sendrecv_replace (buf, count, datatype, dest, sendtag, source, recvtag, comm, &status); // L70--71 MPI_Datatype oldtype; MPI_Datatype newtype; MPI_Type_contiguous (count, oldtype, &newtype); // L74 int blocklength; { int stride; MPI_Type_vector (count, blocklength, stride, oldtype, &newtype); // L78 } { MPI_Aint stride; MPI_Type_hvector (count, blocklength, stride, oldtype, &newtype); // L82 } int* array_of_blocklengths; { int* array_of_displacements; MPI_Type_indexed (count, array_of_blocklengths, array_of_displacements, oldtype, &newtype); // L87--88 } { MPI_Aint* array_of_displacements; MPI_Type_hindexed (count, array_of_blocklengths, array_of_displacements, oldtype, &newtype); // L92--93 MPI_Datatype* array_of_types; MPI_Type_struct (count, array_of_blocklengths, array_of_displacements, array_of_types, &newtype); // L95--96 } void* location; MPI_Aint address; MPI_Address (location, &address); // L100 MPI_Aint extent; MPI_Type_extent (datatype, &extent); // L102 MPI_Type_size (datatype, &size); MPI_Aint displacement; MPI_Type_lb (datatype, &displacement); // L105 MPI_Type_ub (datatype, &displacement); MPI_Type_commit (&datatype); MPI_Type_free (&datatype); MPI_Get_elements (&status, datatype, &count); void* inbuf; void* outbuf; int outsize; int position; MPI_Pack (inbuf, incount, datatype, outbuf, outsize, &position, comm); // L114 int insize; MPI_Unpack (inbuf, insize, &position, outbuf, outcount, datatype, comm); // L116--117 MPI_Pack_size (incount, datatype, comm, &size); /* === Collectives === */ MPI_Barrier (comm); // L121 int root; MPI_Bcast (buffer, count, datatype, root, comm); // L123 MPI_Gather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); // L124--125 int* recvcounts; int* displs; MPI_Gatherv (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm); // L128--130 MPI_Scatter (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm); // L131--132 int* sendcounts; MPI_Scatterv (sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm); // L134--135 MPI_Allgather (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); // L136--137 MPI_Allgatherv (sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, comm); // L138--140 MPI_Alltoall (sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm); // L141--142 int* sdispls; int* rdispls; MPI_Alltoallv (sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm); // L145--147 MPI_Op op; MPI_Reduce (sendbuf, recvbuf, count, datatype, op, root, comm); // L149 #if 0 MPI_User_function function; int commute; MPI_Op_create (function, commute, &op); // L153 #endif MPI_Op_free (&op); // L155 MPI_Allreduce (sendbuf, recvbuf, count, datatype, op, comm); MPI_Reduce_scatter (sendbuf, recvbuf, recvcounts, datatype, op, comm); MPI_Scan (sendbuf, recvbuf, count, datatype, op, comm); /* === Groups, contexts, and communicators === */ MPI_Group group; MPI_Group_size (group, &size); // L162 int rank; MPI_Group_rank (group, &rank); // L164 MPI_Group group1; int n; int* ranks1; MPI_Group group2; int* ranks2; MPI_Group_translate_ranks (group1, n, ranks1, group2, ranks2); // L170 int result; MPI_Group_compare (group1, group2, &result); // L172 MPI_Group newgroup; MPI_Group_union (group1, group2, &newgroup); // L174 MPI_Group_intersection (group1, group2, &newgroup); MPI_Group_difference (group1, group2, &newgroup); int* ranks; MPI_Group_incl (group, n, ranks, &newgroup); // L178 MPI_Group_excl (group, n, ranks, &newgroup); extern int ranges[][3]; MPI_Group_range_incl (group, n, ranges, &newgroup); // L181 MPI_Group_range_excl (group, n, ranges, &newgroup); MPI_Group_free (&group); MPI_Comm_size (comm, &size); MPI_Comm_rank (comm, &rank); MPI_Comm comm1; MPI_Comm comm2; MPI_Comm_compare (comm1, comm2, &result); MPI_Comm newcomm; MPI_Comm_dup (comm, &newcomm); MPI_Comm_create (comm, group, &newcomm); int color; int key; MPI_Comm_split (comm, color, key, &newcomm); // L194 MPI_Comm_free (&comm); MPI_Comm_test_inter (comm, &flag); MPI_Comm_remote_size (comm, &size); MPI_Comm_remote_group (comm, &group); MPI_Comm local_comm; int local_leader; MPI_Comm peer_comm; int remote_leader; MPI_Comm newintercomm; MPI_Intercomm_create (local_comm, local_leader, peer_comm, remote_leader, tag, &newintercomm); // L204--205 MPI_Comm intercomm; MPI_Comm newintracomm; int high; MPI_Intercomm_merge (intercomm, high, &newintracomm); // L209 int keyval; #if 0 MPI_Copy_function copy_fn; MPI_Delete_function delete_fn; void* extra_state; MPI_Keyval_create (copy_fn, delete_fn, &keyval, extra_state); // L215 #endif MPI_Keyval_free (&keyval); // L217 void* attribute_val; MPI_Attr_put (comm, keyval, attribute_val); // L219 MPI_Attr_get (comm, keyval, attribute_val, &flag); MPI_Attr_delete (comm, keyval); /* === Environmental inquiry === */ char* name; int resultlen; MPI_Get_processor_name (name, &resultlen); // L226 MPI_Errhandler errhandler; #if 0 MPI_Handler_function function; MPI_Errhandler_create (function, &errhandler); // L230 #endif MPI_Errhandler_set (comm, errhandler); // L232 MPI_Errhandler_get (comm, &errhandler); MPI_Errhandler_free (&errhandler); int errorcode; char* string; MPI_Error_string (errorcode, string, &resultlen); // L237 int errorclass; MPI_Error_class (errorcode, &errorclass); // L239 MPI_Wtime (); MPI_Wtick (); int argc; char** argv; MPI_Init (&argc, &argv); // L244 MPI_Finalize (); MPI_Initialized (&flag); MPI_Abort (comm, errorcode); }
void comm_recv_procs_and_msg_sizes(ParallelMachine comm , const std::vector<CommBuffer>& send_bufs , std::vector<CommBuffer>& recv_bufs, std::vector<int>& send_procs, std::vector<int>& recv_procs) { static const char method[] = "stk::comm_procs_and_msg_recv_sizes" ; const int p_size = parallel_machine_size( comm ); int result = MPI_SUCCESS ; MPI_Datatype uint_type = MPI_LONG_LONG; if (sizeof(int) == sizeof(unsigned)) uint_type = MPI_INT; else if (sizeof(long) == sizeof(unsigned)) uint_type = MPI_LONG; else if (sizeof(long long) == sizeof(unsigned)) uint_type = MPI_LONG_LONG; else { std::ostringstream msg ; msg << method << " ERROR: No matching MPI type found for size_t argument"; throw std::runtime_error(msg.str()); } std::vector<unsigned> buf; buf.reserve(p_size*2); int* recvcounts = reinterpret_cast<int*>(&buf[0]); unsigned * tmp = &buf[p_size]; send_procs.clear(); send_procs.reserve(16); for ( int i = 0 ; i < p_size ; ++i ) { recvcounts[i] = 1; tmp[i] = 0; if ( send_bufs[i].size() > 0 ) { tmp[i] = 1 ; send_procs.push_back(i); } } unsigned num_recv = 0; result = MPI_Reduce_scatter(tmp,&num_recv,recvcounts,uint_type,MPI_SUM,comm); if ( result != MPI_SUCCESS ) { // PARALLEL ERROR std::ostringstream msg ; msg << method << " ERROR: " << result << " == MPI_Reduce_scatter" ; throw std::runtime_error( msg.str() ); } // do point-to-point send/recvs const int mpi_tag = STK_COMMSPARSE_MPI_TAG_PROC_SIZING; MPI_Request request_null = MPI_REQUEST_NULL ; MPI_Status init_status; std::vector<MPI_Request> request( num_recv , request_null ); std::vector<MPI_Status> status( num_recv , init_status ); // Post receives for point-to-point message sizes for ( unsigned i = 0 ; i < num_recv ; ++i ) { unsigned * const p_buf = & buf[i] ; MPI_Request * const p_request = & request[i] ; result = MPI_Irecv( p_buf , 1 , uint_type, MPI_ANY_SOURCE , mpi_tag , comm , p_request ); if ( MPI_SUCCESS != result ) { // LOCAL ERROR std::ostringstream msg ; msg << method << " ERROR: " << result << " == MPI_Irecv" ; throw std::runtime_error( msg.str() ); } } // Send the point-to-point message sizes, for ( size_t i = 0 ; i < send_procs.size() ; ++i ) { int dst = send_procs[i]; unsigned value = send_bufs[dst].size(); result = MPI_Send( & value , 1 , uint_type, dst , mpi_tag , comm ); if ( MPI_SUCCESS != result ) { // LOCAL ERROR std::ostringstream msg ; msg << method << " ERROR: " << result << " == MPI_Send" ; throw std::runtime_error( msg.str() ); } } // Wait for all receives { MPI_Request * const p_request = (request.empty() ? NULL : & request[0]) ; MPI_Status * const p_status = (status.empty() ? NULL : & status[0]) ; result = MPI_Waitall( num_recv , p_request , p_status ); } if ( MPI_SUCCESS != result ) { // LOCAL ERROR ? std::ostringstream msg ; msg << method << " ERROR: " << result << " == MPI_Waitall" ; throw std::runtime_error( msg.str() ); } recv_procs.resize(num_recv); // Set the receive message sizes for ( unsigned i = 0 ; i < num_recv ; ++i ) { MPI_Status * const recv_status = & status[i] ; const int recv_proc = recv_status->MPI_SOURCE ; #ifndef NDEBUG //debug-mode-only error check const int recv_tag = recv_status->MPI_TAG ; int recv_count = 0 ; MPI_Get_count( recv_status , uint_type , & recv_count ); if ( recv_tag != mpi_tag || recv_count != 1 ) { std::ostringstream msg ; const int p_rank = parallel_machine_rank( comm ); msg << method << " ERROR: Received buffer mismatch " ; msg << "P" << p_rank << " <- P" << recv_proc ; msg << " " << 1 << " != " << recv_count ; throw std::runtime_error( msg.str() ); } #endif recv_bufs[ recv_proc ].set_size(buf[i]); recv_procs[i] = recv_proc; } }
void Irregular::pattern(int n, int *proclist) { // free any previous irregular info deallocate(); init(); patternflag = SET; sizestyle = NONE; ndatumsend = n; // list = 1 for procs I send to, including self // nrecv = # of messages I receive, not including self // self = 0 if no data for self, 1 if there is int *list = new int[nprocs]; int *counts = new int[nprocs]; for (int i = 0; i < nprocs; i++) { list[i] = 0; counts[i] = 1; } for (int i = 0; i < n; i++) list[proclist[i]] = 1; MPI_Reduce_scatter(list,&nrecv,counts,MPI_INT,MPI_SUM,comm); self = 0; if (list[me]) self = 1; if (self) nrecv--; // storage for recv info, not including self recvproc = new int[nrecv]; recvcount = new int[nrecv]; recvsize = new int[nrecv]; request = new MPI_Request[nrecv]; status = new MPI_Status[nrecv]; // list = # of datums to send to each proc, including self // nsend = # of messages I send, not including self for (int i = 0; i < nprocs; i++) list[i] = 0; for (int i = 0; i < n; i++) list[proclist[i]]++; nsend = 0; for (int i = 0; i < nprocs; i++) if (list[i] > 0) nsend++; if (self) nsend--; // storage for send info, including self sendproc = new int[nsend+self]; sendcount = new int[nsend+self]; sendsize = new int[nsend+self]; sendindices = (int *) memory->smalloc(n*sizeof(int),"sendindices"); // setup sendprocs and sendcounts, including self // each proc begins with iproc > me, and continues until iproc = me // list ends up with pointer to which send that proc is associated with int iproc = me; int isend = 0; for (int i = 0; i < nprocs; i++) { iproc++; if (iproc == nprocs) iproc = 0; if (list[iproc] > 0) { sendproc[isend] = iproc; sendcount[isend] = list[iproc]; list[iproc] = isend; isend++; } } // post all receives for datum counts for (int i = 0; i < nrecv; i++) MPI_Irecv(&recvcount[i],1,MPI_INT,MPI_ANY_SOURCE,0,comm,&request[i]); // barrier to insure receives are posted MPI_Barrier(comm); // send each datum count, packing buf with needed datums for (int i = 0; i < nsend; i++) MPI_Send(&sendcount[i],1,MPI_INT,sendproc[i],0,comm); // insure all MPI_ANY_SOURCE messages are received // set recvproc if (nrecv) MPI_Waitall(nrecv,request,status); for (int i = 0; i < nrecv; i++) recvproc[i] = status[i].MPI_SOURCE; // ndatumrecv = total datums received, including self ndatumrecv = 0; for (int i = 0; i < nrecv; i++) ndatumrecv += recvcount[i]; if (self) ndatumrecv += sendcount[nsend]; // setup sendindices, including self // counts = offset into sendindices for each proc I send to // let sc0 = sendcount[0], sc1 = sendcount[1], etc // sendindices[0:sc0-1] = indices of datums in 1st message // sendindices[sc0:sc0+sc1-1] = indices of datums in 2nd message, etc counts[0] = 0; for (int i = 1; i < nsend+self; i++) counts[i] = counts[i-1] + sendcount[i-1]; for (int i = 0; i < n; i++) { isend = list[proclist[i]]; sendindices[counts[isend]++] = i; } // clean up delete [] counts; delete [] list; }
//============================================================================== //--------------------------------------------------------------------------- //ComputeRecvs Method //--------------------------------------------------------------------------- int Epetra_MpiDistributor::ComputeRecvs_( int my_proc, int nprocs ) { int * msg_count = new int[ nprocs ]; int * counts = new int[ nprocs ]; int i; MPI_Status status; for( i = 0; i < nprocs; i++ ) { msg_count[i] = 0; counts[i] = 1; } for( i = 0; i < nsends_+self_msg_; i++ ) msg_count[ procs_to_[i] ] = 1; #if defined(REDUCE_SCATTER_BUG) // the bug is found in mpich on linux platforms MPI_Reduce(msg_count, counts, nprocs, MPI_INT, MPI_SUM, 0, comm_); MPI_Scatter(counts, 1, MPI_INT, &nrecvs_, 1, MPI_INT, 0, comm_); #else MPI_Reduce_scatter( msg_count, &nrecvs_, counts, MPI_INT, MPI_SUM, comm_ ); #endif delete [] msg_count; delete [] counts; if (nrecvs_>0) { lengths_from_ = new int[nrecvs_]; procs_from_ = new int[nrecvs_]; for(i=0; i<nrecvs_; ++i) { lengths_from_[i] = 0; procs_from_[i] = 0; } } #ifndef NEW_COMM_PATTERN for( i = 0; i < (nsends_+self_msg_); i++ ) if( procs_to_[i] != my_proc ) { MPI_Send( &(lengths_to_[i]), 1, MPI_INT, procs_to_[i], tag_, comm_ ); } else { //set self_msg_ to end block of recv arrays lengths_from_[nrecvs_-1] = lengths_to_[i]; procs_from_[nrecvs_-1] = my_proc; } for( i = 0; i < (nrecvs_-self_msg_); i++ ) { MPI_Recv( &(lengths_from_[i]), 1, MPI_INT, MPI_ANY_SOURCE, tag_, comm_, &status ); procs_from_[i] = status.MPI_SOURCE; } MPI_Barrier( comm_ ); #else if (nrecvs_>0) { if( !request_ ) { request_ = new MPI_Request[nrecvs_-self_msg_]; status_ = new MPI_Status[nrecvs_-self_msg_]; } } for( i = 0; i < (nrecvs_-self_msg_); i++ ) MPI_Irecv( &(lengths_from_[i]), 1, MPI_INT, MPI_ANY_SOURCE, tag_, comm_, &(request_[i]) ); MPI_Barrier( comm_ ); for( i = 0; i < (nsends_+self_msg_); i++ ) if( procs_to_[i] != my_proc ) { MPI_Rsend( &(lengths_to_[i]), 1, MPI_INT, procs_to_[i], tag_, comm_ ); } else { //set self_msg_ to end block of recv arrays lengths_from_[nrecvs_-1] = lengths_to_[i]; procs_from_[nrecvs_-1] = my_proc; } if( (nrecvs_-self_msg_) > 0 ) MPI_Waitall( (nrecvs_-self_msg_), request_, status_ ); for( i = 0; i < (nrecvs_-self_msg_); i++ ) procs_from_[i] = status_[i].MPI_SOURCE; #endif Sort_ints_( procs_from_, lengths_from_, nrecvs_ ); // Compute indices_from_ // Seems to break some rvs communication /* Not necessary since rvs communication is always blocked size_indices_from_ = 0; if( nrecvs_ > 0 ) { for( i = 0; i < nrecvs_; i++ ) size_indices_from_ += lengths_from_[i]; indices_from_ = new int[ size_indices_from_ ]; for (i=0; i<size_indices_from_; i++) indices_from_[i] = i; } */ if (nrecvs_>0) starts_from_ = new int[nrecvs_]; int j = 0; for( i=0; i<nrecvs_; ++i ) { starts_from_[i] = j; j += lengths_from_[i]; } total_recv_length_ = 0; for( i = 0; i < nrecvs_; i++ ) total_recv_length_ += lengths_from_[i]; nrecvs_ -= self_msg_; MPI_Barrier( comm_ ); return false; }
void BIL_Pio_exchange_blocks(BIL_Sched_IO* io_sched, BIL_Sched_IO* inv_io_sched) { BIL_Timing_comm_start(); int i; // Convenience variables int num_recv_blocks = inv_io_sched->num_io_blocks; BIL_Block* recv_blocks = inv_io_sched->io_blocks; int num_io_blocks = io_sched->num_io_blocks; BIL_Block* io_blocks = io_sched->io_blocks; // Sort your recv blocks by their read rank. qsort(recv_blocks, num_recv_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_read_rank); // Sort the I/O blocks by their group name to perform binary searching. qsort(io_blocks, num_io_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_group_name); qsort(BIL->blocks, BIL->num_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_group_name); // In netcdf mode, you will not know the variable size of the blocks you // are receiving until now. The variable sizes are stored in the // BIL global variable after reading. Fill in these sizes into the inverse // I/O schedule. for (i = 0; i < num_recv_blocks; i++) { BIL_Block* io_block = bsearch(&(recv_blocks[i]), BIL->blocks, BIL->num_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_group_name); recv_blocks[i].var_size = io_block->var_size; } int* all_num_send_blocks = BIL_Misc_malloc(sizeof(int) * BIL->world_size); memset(all_num_send_blocks, 0, sizeof(int) * BIL->world_size); for (i = 0; i < num_recv_blocks; i++) { all_num_send_blocks[recv_blocks[i].read_rank]++; } int* recv_counts = BIL_Misc_malloc(sizeof(int) * BIL->world_size); for (i = 0; i < BIL->world_size; i++) { recv_counts[i] = 1; } int num_send_blocks = 0; MPI_Reduce_scatter(all_num_send_blocks, &num_send_blocks, recv_counts, MPI_INT, MPI_SUM, BIL->world_comm); BIL_Misc_free(all_num_send_blocks); BIL_Misc_free(recv_counts); // Receive the blocks that you will send. BIL_Block* send_blocks = BIL_Misc_malloc(sizeof(BIL_Block) * num_send_blocks); MPI_Request* recv_requests = BIL_Misc_malloc(sizeof(MPI_Request) * num_send_blocks); for (i = 0; i < num_send_blocks; i++) { MPI_Irecv(send_blocks + i, sizeof(BIL_Block), MPI_BYTE, MPI_ANY_SOURCE, 0, BIL->world_comm, recv_requests + i); } // Send the blocks that you will receive. MPI_Request* send_requests = BIL_Misc_malloc(sizeof(MPI_Request) * inv_io_sched->num_io_blocks); for (i = 0; i < num_recv_blocks; i++) { MPI_Isend(recv_blocks + i, sizeof(BIL_Block), MPI_BYTE, recv_blocks[i].read_rank, 0, BIL->world_comm, send_requests + i); } assert(MPI_Waitall(num_send_blocks, recv_requests, MPI_STATUSES_IGNORE) == MPI_SUCCESS); assert(MPI_Waitall(num_recv_blocks, send_requests, MPI_STATUSES_IGNORE) == MPI_SUCCESS); // This barrier is need on my mac to get this to work. Not sure why, but // it seems that Waitall doesn't block until the isends/irecvs are complete. MPI_Barrier(BIL->world_comm); BIL_Misc_free(recv_requests); BIL_Misc_free(send_requests); // Sort your send blocks by their request rank. qsort(send_blocks, num_send_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_request_rank); // Find the total amount of send data. int64_t total_send_data = 0; for (i = 0; i < num_send_blocks; i++) { total_send_data += send_blocks[i].var_size * send_blocks[i].total_size; } // Allocate a big array for the sending data. Once this array is allocated, // the I/O data will be copied to it in such a manner that MPI_Alltoallv // can be performed. void* send_data = BIL_Misc_malloc(total_send_data); // For convenience, set the blocks send data to the appropriate offset // in the send_data array. if (num_send_blocks > 0) { send_blocks[0].data = send_data; for (i = 1; i < num_send_blocks; i++) { send_blocks[i].data = send_blocks[i - 1].data + (send_blocks[i - 1].var_size * send_blocks[i - 1].total_size); } } // Pack each block into send_data. for (i = 0; i < num_send_blocks; i++) { // Find the corresponding I/O block. BIL_Block* io_block = bsearch(&(send_blocks[i]), io_blocks, num_io_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_group_name); assert(io_block != NULL); BIL_Pio_extract_block(&(send_blocks[i]), io_block); } // Free the I/O data now. for (i = 0; i < num_io_blocks; i++) { BIL_Misc_free(io_blocks[i].data); } // Allocate a big array for receiving data. This array will be populated // during the MPI_Alltoallv call. int64_t total_recv_data = 0; for (i = 0; i < num_recv_blocks; i++) { total_recv_data += recv_blocks[i].var_size * recv_blocks[i].total_size; } void* recv_data = NULL; // Special case - single block I/O where the user has already allocated a // buffer. if (num_recv_blocks == 1 && BIL->blocks[0].data != NULL) { recv_data = BIL->blocks[0].data; } else if (num_recv_blocks > 0) { recv_data = BIL_Misc_malloc(total_recv_data); recv_blocks[0].data = recv_data; for (i = 1; i < num_recv_blocks; i++) { recv_blocks[i].data = recv_blocks[i - 1].data + (recv_blocks[i - 1].var_size * recv_blocks[i - 1].total_size); } } // Find out the recv counts and offsets. Keep in mind that you may receive // multiple blocks from the same process. recv_counts = BIL_Misc_malloc(sizeof(int) * BIL->world_size); memset(recv_counts, 0, sizeof(int) * BIL->world_size); int* recv_offsets = BIL_Misc_malloc(sizeof(int) * BIL->world_size); memset(recv_offsets, 0, sizeof(int) * BIL->world_size); int cur_recv_offset = 0; for (i = 0; i < num_recv_blocks; i++) { if (recv_counts[recv_blocks[i].read_rank] == 0) { recv_offsets[recv_blocks[i].read_rank] = cur_recv_offset; } recv_counts[recv_blocks[i].read_rank] += recv_blocks[i].total_size * recv_blocks[i].var_size; cur_recv_offset += recv_blocks[i].total_size * recv_blocks[i].var_size; } // Find out the send counts and offsets. Keep in mind you might send // multiple blocks to the same process. int* send_counts = BIL_Misc_malloc(sizeof(int) * BIL->world_size); memset(send_counts, 0, sizeof(int) * BIL->world_size); int* send_offsets = BIL_Misc_malloc(sizeof(int) * BIL->world_size); memset(send_offsets, 0, sizeof(int) * BIL->world_size); int cur_send_offset = 0; for (i = 0; i < num_send_blocks; i++) { if (send_counts[send_blocks[i].request_rank] == 0) { send_offsets[send_blocks[i].request_rank] = cur_send_offset; } send_counts[send_blocks[i].request_rank] += send_blocks[i].total_size * send_blocks[i].var_size; cur_send_offset += send_blocks[i].total_size * send_blocks[i].var_size; } // Exchange blocks. MPI_Alltoallv(send_data, send_counts, send_offsets, MPI_BYTE, recv_data, recv_counts, recv_offsets, MPI_BYTE, BIL->world_comm); // Free your send data. BIL_Misc_free(send_data); // Copy your recv blocks into BIL->blocks. Remember that multiple recv blocks // might account for only one block in BIL->blocks. qsort(BIL->blocks, BIL->num_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_id); if (num_recv_blocks > 1) { for (i = 0; i < num_recv_blocks; i++) { // Find the corresponding I/O block. BIL_Block* io_block = bsearch(&(recv_blocks[i]), BIL->blocks, BIL->num_blocks, sizeof(BIL_Block), BIL_Pio_compare_block_id); assert(io_block != NULL); if (io_block->data == NULL) { io_block->data = BIL_Misc_malloc(io_block->var_size * io_block->total_size); } BIL_Pio_insert_block(&(recv_blocks[i]), io_block); } // Free the recv_data. BIL_Misc_free(recv_data); } else if (num_recv_blocks == 1 && BIL->blocks[0].data == NULL) { assert(BIL->num_blocks == 1); // The reason we are deferencing this void** is because the user passed // a NULL buffer that was allocated for them. BIL->blocks[0].data = recv_blocks[0].data; } BIL_Misc_free(send_counts); BIL_Misc_free(send_offsets); BIL_Misc_free(recv_counts); BIL_Misc_free(recv_offsets); BIL_Misc_free(send_blocks); BIL_Timing_comm_stop(); }
int main( int argc, char **argv ) { int err = 0; int *recvcounts; int size, rsize, rank, i; int recvcount, /* Each process receives this much data */ sendcount, /* Each process contributes this much data */ basecount; /* Unit of elements - basecount *rsize is recvcount, etc. */ int isLeftGroup; long long *sendbuf, *recvbuf; long long sumval; MPI_Comm comm; MTest_Init( &argc, &argv ); comm = MPI_COMM_WORLD; basecount = 1024; while (MTestGetIntercomm( &comm, &isLeftGroup, 2 )) { if (comm == MPI_COMM_NULL) continue; MPI_Comm_remote_size( comm, &rsize ); MPI_Comm_size( comm, &size ); MPI_Comm_rank( comm, &rank ); if (0) { printf( "[%d] %s (%d,%d) remote %d\n", rank, isLeftGroup ? "L" : "R", rank, size, rsize ); } recvcount = basecount * rsize; sendcount = basecount * rsize * size; recvcounts = (int *)malloc( size * sizeof(int) ); if (!recvcounts) { fprintf( stderr, "Could not allocate %d int for recvcounts\n", size ); MPI_Abort( MPI_COMM_WORLD, 1 ); } for (i=0; i<size; i++) recvcounts[i] = recvcount; sendbuf = (long long *) malloc( sendcount * sizeof(long long) ); if (!sendbuf) { fprintf( stderr, "Could not allocate %d ints for sendbuf\n", sendcount ); MPI_Abort( MPI_COMM_WORLD, 1 ); } for (i=0; i<sendcount; i++) { sendbuf[i] = (long long)(rank*sendcount + i); } recvbuf = (long long *)malloc( recvcount * sizeof(long long) ); if (!recvbuf) { fprintf( stderr, "Could not allocate %d ints for recvbuf\n", recvcount ); MPI_Abort( MPI_COMM_WORLD, 1 ); } for (i=0; i<recvcount; i++) { recvbuf[i] = (long long)(-i); } MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_LONG_LONG, MPI_SUM, comm ); /* Check received data */ for (i=0; i<recvcount; i++) { sumval = (long long)(sendcount) * (long long)((rsize * (rsize-1))/2) + (long long)(i + rank * rsize * basecount) * (long long)rsize; if (recvbuf[i] != sumval) { err++; if (err < 4) { fprintf( stdout, "Did not get expected value for reduce scatter\n" ); fprintf( stdout, "[%d] %s recvbuf[%d] = %lld, expected %lld\n", rank, isLeftGroup ? "L" : "R", i, recvbuf[i], sumval ); } } } free(sendbuf); free(recvbuf); free(recvcounts); MTestFreeComm( &comm ); } MTest_Finalize( err ); MPI_Finalize( ); return 0; }
int main( int argc, char **argv ) { int err = 0; int *sendbuf, *recvbuf, *recvcounts; int size, rank, i, j, idx, mycount, sumval; MPI_Comm comm; MTest_Init( &argc, &argv ); comm = MPI_COMM_WORLD; MPI_Comm_size( comm, &size ); MPI_Comm_rank( comm, &rank ); recvcounts = (int *)malloc( size * sizeof(int) ); if (!recvcounts) { fprintf( stderr, "Could not allocate %d ints for recvcounts\n", size ); MPI_Abort( MPI_COMM_WORLD, 1 ); } mycount = (1024 * 1024) / size; for (i=0; i<size; i++) recvcounts[i] = mycount; sendbuf = (int *) malloc( mycount * size * sizeof(int) ); if (!sendbuf) { fprintf( stderr, "Could not allocate %d ints for sendbuf\n", mycount * size ); MPI_Abort( MPI_COMM_WORLD, 1 ); } idx = 0; for (i=0; i<size; i++) { for (j=0; j<mycount; j++) { sendbuf[idx++] = rank + i; } } recvbuf = (int *)malloc( mycount * sizeof(int) ); if (!recvbuf) { fprintf( stderr, "Could not allocate %d ints for recvbuf\n", mycount ); MPI_Abort( MPI_COMM_WORLD, 1 ); } for (i=0; i<mycount; i++) { recvbuf[i] = -1; } MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_INT, MPI_SUM, comm ); sumval = size * rank + ((size - 1) * size)/2; /* recvbuf should be size * (rank + i) */ for (i=0; i<mycount; i++) { if (recvbuf[i] != sumval) { err++; if (err < MAX_ERRORS) { fprintf( stdout, "Did not get expected value for reduce scatter\n" ); fprintf( stdout, "[%d] Got recvbuf[%d] = %d expected %d\n", rank, i, recvbuf[i], sumval ); } } } #if MTEST_HAVE_MIN_MPI_VERSION(2,2) MPI_Reduce_scatter( MPI_IN_PLACE, sendbuf, recvcounts, MPI_INT, MPI_SUM, comm ); sumval = size * rank + ((size - 1) * size)/2; /* recv'ed values for my process should be size * (rank + i) */ for (i=0; i<mycount; i++) { if (sendbuf[i] != sumval) { err++; if (err < MAX_ERRORS) { fprintf( stdout, "Did not get expected value for reduce scatter (in place)\n" ); fprintf( stdout, "[%d] Got buf[%d] = %d expected %d\n", rank, i, sendbuf[i], sumval ); } } } #endif free(sendbuf); free(recvbuf); free(recvcounts); MTest_Finalize( err ); MPI_Finalize( ); return 0; }
static int Zoltan_Reftree_Sum_Weights(ZZ *zz) { /* * Function to sum the weights in the refinement tree. On input the * refinement tree should be valid and have weight set. On output the * values in summed_weight at each node is the sum of the weights in the * subtree with that node as root. * This function also sets assigned_to_me for interior nodes to be * 1 if the entire subtree is assigned to this processor * 0 if none of the subtree is assigned to this processor * -1 if some of the subtree is assigned to this processor */ char *yo = "Zoltan_Reftree_Sum_Weights"; ZOLTAN_REFTREE *root; /* Root of the refinement tree */ int wdim; /* Dimension of the weight array */ int i,j; /* loop counters */ int count; /* counter */ ZOLTAN_ID_PTR leaf_list = NULL; /* leaves for which some proc requests weight */ ZOLTAN_ID_PTR all_leaflist = NULL; /* leaf_list from all processors */ int reqsize; /* length of leaf_list */ int *reqsize_all; /* reqsize from all processors */ int sum_reqsize; /* sum of all reqsize */ int *displs; /* running sum of all reqsize */ int my_start; /* position in leaf_list of this proc's list */ int nproc; /* number of processors */ ZOLTAN_REFTREE *node; /* a node in the refinement tree */ struct Zoltan_Reftree_hash_node **hashtab; /* hash table */ int hashsize; /* dimension of hash table */ float *send_float; /* sending message of floats */ float *req_weights; /* the requested weights */ int num_gid_entries = zz->Num_GID; /* Number of array entries in a global ID */ ZOLTAN_TRACE_ENTER(zz, yo); /* * set the root and hash table */ root = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->reftree_root; if (root == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Refinement tree not defined."); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_FATAL); } hashtab = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table; hashsize = ((struct Zoltan_Reftree_data_struct *)zz->LB.Data_Structure)->hash_table_size; /* * Determine the dimension of the weight array */ if (zz->Obj_Weight_Dim == 0) { wdim = 1; } else { wdim = zz->Obj_Weight_Dim; } /* * In the first pass, sum the weights of the nodes that are assigned to * this processor, and count the leaves that are not. */ count = 0; for (i=0; i<root->num_child; i++) { Zoltan_Reftree_Sum_My_Weights(zz,&(root->children[i]),&count,wdim); } root->assigned_to_me = -1; /* * Make a list of the leaves that are not assigned to this processor */ if (count == 0) leaf_list = ZOLTAN_MALLOC_GID(zz); else leaf_list = ZOLTAN_MALLOC_GID_ARRAY(zz, count); if (leaf_list == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } count = 0; Zoltan_Reftree_List_Other_Leaves(zz, root,leaf_list,&count); /* * Get the unknown leaf weights from other processors. */ nproc = zz->Num_Proc; reqsize = count; /* * Build a list of all processor's request list by concatinating them in * the order of the processor ranks */ /* * Determine the request size of all processors */ reqsize_all = (int *)ZOLTAN_MALLOC(nproc*sizeof(int)); displs = (int *)ZOLTAN_MALLOC(nproc*sizeof(int)); if (reqsize_all == NULL || displs == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 3, &displs, &reqsize_all, &leaf_list); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } MPI_Allgather((void *)&reqsize,1,MPI_INT,(void *)reqsize_all,1,MPI_INT, zz->Communicator); displs[0] = 0; for (i=1; i<nproc; i++) displs[i] = displs[i-1]+reqsize_all[i-1]; sum_reqsize = displs[nproc-1] + reqsize_all[nproc-1]; my_start = displs[zz->Proc]; /* * If sum_reqsize is 0, nothing needs to be communciated */ if (sum_reqsize == 0) { Zoltan_Multifree(__FILE__, __LINE__, 3, &displs, &reqsize_all, &leaf_list); } else { /* * Gather the request list from all processors */ all_leaflist = ZOLTAN_MALLOC_GID_ARRAY(zz, sum_reqsize); if (all_leaflist == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 4, &all_leaflist, &displs, &reqsize_all, &leaf_list); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } /* KDDKDD Changed MPI_BYTE to ZOLTAN_ID_MPI_TYPE */ /* Account for number of array entries in an ID. */ for (i=0; i<nproc; i++) { reqsize_all[i] = reqsize_all[i]*num_gid_entries; displs[i] = displs[i]*num_gid_entries; } MPI_Allgatherv((void *)leaf_list,reqsize*num_gid_entries,ZOLTAN_ID_MPI_TYPE, (void *)all_leaflist,reqsize_all,displs,ZOLTAN_ID_MPI_TYPE, zz->Communicator); ZOLTAN_FREE(&displs); ZOLTAN_FREE(&leaf_list); for (i=0; i<nproc; i++) reqsize_all[i] = reqsize_all[i]/num_gid_entries; /* * Create a list with the partial sums this processor has */ send_float = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*sum_reqsize); if (send_float == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 3, &send_float, &all_leaflist, &reqsize_all); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } for (i=0; i<sum_reqsize; i++) { node = Zoltan_Reftree_hash_lookup(zz, hashtab, &(all_leaflist[i*num_gid_entries]), hashsize); if (node == NULL) for (j=0; j<wdim; j++) send_float[i*wdim+j] = 0.0; else for (j=0; j<wdim; j++) send_float[i*wdim+j] = node->my_sum_weight[j]; } /* * Sum the weights over all the processors */ if (reqsize == 0) req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim); else req_weights = (float *) ZOLTAN_MALLOC(sizeof(float)*wdim*reqsize); if (req_weights == NULL) { ZOLTAN_PRINT_ERROR(zz->Proc, yo, "Insufficient memory."); Zoltan_Multifree(__FILE__, __LINE__, 4, &req_weights, &send_float, &all_leaflist, &reqsize_all); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_MEMERR); } MPI_Reduce_scatter((void *)send_float, (void *)req_weights, reqsize_all, MPI_FLOAT, MPI_SUM, zz->Communicator); ZOLTAN_FREE(&send_float); ZOLTAN_FREE(&reqsize_all); /* * Set the weights this processor requested */ for (i=0; i<count; i++) { node = Zoltan_Reftree_hash_lookup(zz, hashtab, &(all_leaflist[(i+my_start)*num_gid_entries]), hashsize); for (j=0; j<wdim; j++) node->summed_weight[j] = req_weights[i*wdim+j]; } ZOLTAN_FREE(&req_weights); ZOLTAN_FREE(&all_leaflist); } /* * All the leaves now have summed_weight set. * Sum the weights throughout the tree. */ Zoltan_Reftree_Sum_All_Weights(zz,root,wdim); ZOLTAN_TRACE_EXIT(zz, yo); return(ZOLTAN_OK); }
int main(int argc, char *argv[]) { int i, numprocs, rank, size, align_size; int skip; double latency = 0.0, t_start = 0.0, t_stop = 0.0; double timer=0.0; double avg_time = 0.0, max_time = 0.0, min_time = 0.0; float *sendbuf, *recvbuf; int *recvcounts; int po_ret; size_t bufsize; set_header(HEADER); set_benchmark_name("osu_scatter"); enable_accel_support(); po_ret = process_options(argc, argv); if (po_okay == po_ret && none != options.accel) { if (init_accel()) { fprintf(stderr, "Error initializing device\n"); exit(EXIT_FAILURE); } } MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &numprocs); switch (po_ret) { case po_bad_usage: print_bad_usage_message(rank); MPI_Finalize(); exit(EXIT_FAILURE); case po_help_message: print_help_message(rank); MPI_Finalize(); exit(EXIT_SUCCESS); case po_version_message: print_version_message(rank); MPI_Finalize(); exit(EXIT_SUCCESS); case po_okay: break; } if(numprocs < 2) { if (rank == 0) { fprintf(stderr, "This test requires at least two processes\n"); } MPI_Finalize(); exit(EXIT_FAILURE); } if (options.max_message_size > options.max_mem_limit) { options.max_message_size = options.max_mem_limit; } if (allocate_buffer((void**)&recvcounts, numprocs*sizeof(int), none)) { fprintf(stderr, "Could Not Allocate Memory [rank %d]\n", rank); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } bufsize = sizeof(float)*(options.max_message_size/sizeof(float)); if (allocate_buffer((void**)&sendbuf, bufsize, options.accel)) { fprintf(stderr, "Could Not Allocate Memory [rank %d]\n", rank); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } set_buffer(sendbuf, options.accel, 1, bufsize); bufsize = sizeof(float)*((options.max_message_size/numprocs + 1)/sizeof(float)); if (allocate_buffer((void**)&recvbuf, bufsize, options.accel)) { fprintf(stderr, "Could Not Allocate Memory [rank %d]\n", rank); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); } set_buffer(recvbuf, options.accel, 0, bufsize); print_preamble(rank); for(size=1; size*sizeof(float)<= options.max_message_size; size *= 2) { if(size > LARGE_MESSAGE_SIZE) { skip = SKIP_LARGE; options.iterations = options.iterations_large; } else { skip = SKIP; } int portion=0, remainder=0; portion=size/numprocs; remainder=size%numprocs; for (i=0; i<numprocs; i++){ recvcounts[i]=0; if(size<numprocs){ if(i<size) recvcounts[i]=1; } else{ if((remainder!=0) && (i<remainder)){ recvcounts[i]+=1; } recvcounts[i]+=portion; } } MPI_Barrier(MPI_COMM_WORLD); timer=0.0; for(i=0; i < options.iterations + skip ; i++) { t_start = MPI_Wtime(); MPI_Reduce_scatter( sendbuf, recvbuf, recvcounts, MPI_FLOAT, MPI_SUM, MPI_COMM_WORLD ); t_stop=MPI_Wtime(); if(i>=skip){ timer+=t_stop-t_start; } MPI_Barrier(MPI_COMM_WORLD); } latency = (double)(timer * 1e6) / options.iterations; MPI_Reduce(&latency, &min_time, 1, MPI_DOUBLE, MPI_MIN, 0, MPI_COMM_WORLD); MPI_Reduce(&latency, &max_time, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD); MPI_Reduce(&latency, &avg_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); avg_time = avg_time/numprocs; print_stats(rank, size, avg_time, min_time, max_time); MPI_Barrier(MPI_COMM_WORLD); } free_buffer(recvcounts, none); free_buffer(sendbuf, options.accel); free_buffer(recvbuf, options.accel); MPI_Finalize(); if (none != options.accel) { if (cleanup_accel()) { fprintf(stderr, "Error cleaning up device\n"); exit(EXIT_FAILURE); } } return EXIT_SUCCESS; }