int MPIX_Get_accumulate_x(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win) { int rc = MPI_SUCCESS; if (likely (origin_count <= bigmpi_int_max && result_count <= bigmpi_int_max && target_count <= bigmpi_int_max)) { rc = MPI_Get_accumulate(origin_addr, origin_count, origin_datatype, result_addr, result_count, result_datatype, target_rank, target_disp, target_count, target_datatype, op, win); } else { MPI_Datatype neworigin_datatype, newresult_datatype, newtarget_datatype; MPIX_Type_contiguous_x(origin_count, origin_datatype, &neworigin_datatype); MPIX_Type_contiguous_x(result_count, result_datatype, &newresult_datatype); MPIX_Type_contiguous_x(target_count, target_datatype, &newtarget_datatype); MPI_Type_commit(&neworigin_datatype); MPI_Type_commit(&newresult_datatype); MPI_Type_commit(&newtarget_datatype); rc = MPI_Get_accumulate(origin_addr, 1, neworigin_datatype, result_addr, 1, newresult_datatype, target_rank, target_disp, 1, newtarget_datatype, op, win); MPI_Type_free(&neworigin_datatype); MPI_Type_free(&newresult_datatype); MPI_Type_free(&newtarget_datatype); } return rc; }
int do_test(int origin_count, MPI_Datatype origin_type, int result_count, MPI_Datatype result_type, int target_count, MPI_Datatype target_type) { int errs = 0, ret, origin_type_size, result_type_size; ret = MPI_Put(origin_buf, origin_count, origin_type, 1, 0, target_count, target_type, win); if (ret) errs++; ret = MPI_Get(origin_buf, origin_count, origin_type, 1, 0, target_count, target_type, win); if (ret) errs++; ret = MPI_Accumulate(origin_buf, origin_count, origin_type, 1, 0, target_count, target_type, MPI_SUM, win); if (ret) errs++; ret = MPI_Get_accumulate(origin_buf, origin_count, origin_type, result_buf, result_count, result_type, 1, 0, target_count, target_type, MPI_SUM, win); if (ret) errs++; MPI_Type_size(origin_type, &origin_type_size); MPI_Type_size(result_type, &result_type_size); if (origin_count == 0 || origin_type_size == 0) { ret = MPI_Put(NULL, origin_count, origin_type, 1, 0, target_count, target_type, win); if (ret) errs++; ret = MPI_Get(NULL, origin_count, origin_type, 1, 0, target_count, target_type, win); if (ret) errs++; ret = MPI_Accumulate(NULL, origin_count, origin_type, 1, 0, target_count, target_type, MPI_SUM, win); if (ret) errs++; ret = MPI_Get_accumulate(NULL, origin_count, origin_type, result_buf, result_count, result_type, 1, 0, target_count, target_type, MPI_SUM, win); if (ret) errs++; if (result_count == 0 || result_type_size == 0) { ret = MPI_Get_accumulate(NULL, origin_count, origin_type, NULL, result_count, result_type, 1, 0, target_count, target_type, MPI_SUM, win); if (ret) errs++; } } return errs; }
/*Run Get_accumulate with Fence */ void run_get_acc_with_fence(int rank, WINDOW type) { int size, i; MPI_Aint disp = 0; MPI_Win win; for (size = 0; size <= MAX_SIZE; size = (size ? size * 2 : size + 1)) { allocate_memory(rank, rbuf, size, type, &win); if (type == WIN_DYNAMIC) { disp = sdisp_remote; } if(size > LARGE_MESSAGE_SIZE) { loop = LOOP_LARGE; skip = SKIP_LARGE; } MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); if(rank == 0) { for (i = 0; i < skip + loop; i++) { if (i == skip) { t_start = MPI_Wtime (); } MPI_CHECK(MPI_Win_fence(0, win)); MPI_CHECK(MPI_Get_accumulate(sbuf, size, MPI_CHAR, cbuf, size, MPI_CHAR, 1, disp, size, MPI_CHAR, MPI_SUM, win)); MPI_CHECK(MPI_Win_fence(0, win)); MPI_CHECK(MPI_Win_fence(0, win)); } t_end = MPI_Wtime (); } else { for (i = 0; i < skip + loop; i++) { MPI_CHECK(MPI_Win_fence(0, win)); MPI_CHECK(MPI_Win_fence(0, win)); MPI_CHECK(MPI_Get_accumulate(sbuf, size, MPI_CHAR, cbuf, size, MPI_CHAR, 0, disp, size, MPI_CHAR, MPI_SUM, win)); MPI_CHECK(MPI_Win_fence(0, win)); } } MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); if (rank == 0) { fprintf(stdout, "%-*d%*.*f\n", 10, size, FIELD_WIDTH, FLOAT_PRECISION, (t_end - t_start) * 1.0e6 / loop / 2); fflush(stdout); } MPI_Win_free(&win); } }
int main(int argc, char **argv) { int rank, nproc; int out_val, i, counter = 0; MPI_Win win; MTest_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); MPI_Win_create(&counter, sizeof(int), sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &win); for (i = 0; i < NITER; i++) { MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); MPI_Get_accumulate(&acc_val, 1, MPI_INT, &out_val, 1, MPI_INT, rank, 0, 1, MPI_INT, MPI_SUM, win); MPI_Win_unlock(rank, win); if (out_val != acc_val * i) { errs++; printf("Error: got %d, expected %d at iter %d\n", out_val, acc_val * i, i); break; } } MPI_Win_free(&win); MTest_Finalize(errs); return MTestReturnValue(errs); }
int main(int argc, char **argv) { int i, rank, nproc; int errors = 0, all_errors = 0; TYPE_C *win_ptr, *res_ptr, *val_ptr; MPI_Win win; #if defined (GACC_TYPE_DERIVED) MPI_Datatype derived_type; #endif MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &nproc); win_ptr = malloc(sizeof(TYPE_C)*nproc*COUNT); res_ptr = malloc(sizeof(TYPE_C)*nproc*COUNT); val_ptr = malloc(sizeof(TYPE_C)*COUNT); #if defined (GACC_TYPE_DERIVED) MPI_Type_contiguous(1, TYPE_MPI_BASE, &derived_type); MPI_Type_commit(&derived_type); #endif MPI_Win_create(win_ptr, sizeof(TYPE_C)*nproc*COUNT, sizeof(TYPE_C), MPI_INFO_NULL, MPI_COMM_WORLD, &win); /* Test self communication */ reset_bufs(win_ptr, res_ptr, val_ptr, 1, win); for (i = 0; i < ITER; i++) { MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win); MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI, rank, 0, COUNT, TYPE_MPI, MPI_SUM, win); MPI_Win_unlock(rank, win); } MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win); for (i = 0; i < COUNT; i++) { if (win_ptr[i] != ITER) { SQUELCH( printf("%d->%d -- SELF[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n", rank, rank, i, (TYPE_C) ITER, win_ptr[i]); ); errors++; }
/*Run Get_accumulate with flush local*/ void run_get_acc_with_flush_local(int rank, WINDOW type) { int size, i; MPI_Aint disp = 0; MPI_Win win; for (size = 0; size <= MAX_SIZE; size = (size ? size * 2 : size + 1)) { allocate_memory(rank, rbuf, size, type, &win); if (type == WIN_DYNAMIC) { disp = sdisp_remote; } if(size > LARGE_MESSAGE_SIZE) { loop = LOOP_LARGE; skip = SKIP_LARGE; } if(rank == 0) { MPI_CHECK(MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 1, 0, win)); for (i = 0; i < skip + loop; i++) { if (i == skip) { t_start = MPI_Wtime (); } MPI_CHECK(MPI_Get_accumulate(sbuf, size, MPI_CHAR, cbuf, size, MPI_CHAR, 1, disp, size, MPI_CHAR, MPI_SUM, win)); MPI_CHECK(MPI_Win_flush_local(1, win)); } t_end = MPI_Wtime (); MPI_CHECK(MPI_Win_unlock(1, win)); } MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); print_latency(rank, size); MPI_Win_free(&win); } }
void test_dim(int ndim) { int dim,elems; int i,j, proc; /* double a[DIM4][DIM3][DIM2][DIM1], b[EDIM4][EDIM3][EDIM2][EDIM1];*/ double *b; double *a, *a1, *a2, *c; int ridx; MPI_Datatype typeA, typeB; int rstrideB[MAXDIMS]; int rcount[MAXDIMS]; int pidx1 = -1, pidx2 = -1, pidx3 = -1; elems = 1; strideA[0]=sizeof(double); strideB[0]=sizeof(double); for(i=0;i<ndim;i++){ strideA[i] *= dimsA[i]; strideB[i] *= dimsB[i]; if(i<ndim-1){ strideA[i+1] = strideA[i]; strideB[i+1] = strideB[i]; } elems *= dimsA[i]; } /* create shared and local arrays */ create_safe_array((void**)&b, sizeof(double),ndim,dimsB); a1 = (double *)malloc(sizeof(double)*elems); assert(a1); a2 = (double *)malloc(sizeof(double)*elems); assert(a2); c = (double *)malloc(sizeof(double)*elems); assert(c); init(a1, ndim, elems, dimsA, me!=0, 0); init(a2, ndim, elems, dimsA, me!=0, 1); if(me==0){ printf("--------array[%d",dimsA[0]); for(dim=1;dim<ndim;dim++)printf(",%d",dimsA[dim]); printf("]--------\n"); } sleep(1); MP_BARRIER(); for(i=0;i<LOOP;i++){ int idx1, idx2, idx3, ridx; MPI_Request request; if (i%2) { a = a2; } else { a = a1; } get_range(ndim, dimsA, loA, hiA); new_range(ndim, dimsB, loA, hiA, loB, hiB); new_range(ndim, dimsA, loA, hiA, loC, hiC); proc=nproc-1-me; if(me==0){ print_range("local",ndim,loA, hiA,"-> "); print_range("remote",ndim,loB, hiB,"-> "); print_range("local",ndim,loC, hiC,"\n"); } idx1 = Index(ndim, loA, dimsA); idx2 = Index(ndim, loB, dimsB); idx3 = Index(ndim, loC, dimsA); MPI_Sendrecv(&idx2, 1, MPI_INT, proc, 666, &ridx, 1, MPI_INT, proc, 666, MPI_COMM_WORLD, MPI_STATUS_IGNORE); for(j=0;j<ndim;j++)count[j]=hiA[j]-loA[j]+1; count[0] *= sizeof(double); /* convert range to bytes at stride level zero */ Strided_to_dtype(strideA, count, ndim-1, MPI_BYTE, &typeA); MPI_Type_commit(&typeA); Strided_to_dtype(strideB, count, ndim-1, MPI_BYTE, &typeB); MPI_Type_commit(&typeB); MPI_Accumulate(a + idx1, 1, typeA, proc, (MPI_Aint)(idx2*sizeof(double)), 1, typeB, MPI_REPLACE, win); MP_FLUSH(proc); /* note that we do not need Fence here since * consectutive operations targeting the same process are ordered */ MPI_Get_accumulate(NULL, 0, MPI_BYTE, c + idx3, 1, typeA, proc, (MPI_Aint)(idx2*sizeof(double)), 1, typeB, MPI_NO_OP, win); MP_FLUSH(proc); compare_patches(0., ndim, a+idx1, loA, hiA, dimsA, c+idx3, loC, hiC, dimsA); pidx1 = idx1; pidx2 = idx2; pidx3 = idx3; MPI_Type_free(&typeA); MPI_Type_free(&typeB); } free(c); destroy_safe_array(); free(a); }
int main(int argc, char *argv[]) { int err, errs = 0; int rank, size, orig, target; int minsize = 2, count; int i, j; MPI_Aint origcount, targetcount; MPI_Comm comm; MPI_Win win; MPI_Aint lb, extent; MPI_Datatype origtype, targettype; DTP_t orig_dtp, target_dtp; void *origbuf, *targetbuf; MTest_Init(&argc, &argv); #ifndef USE_DTP_POOL_TYPE__STRUCT /* set in 'test/mpi/structtypetest.txt' to split tests */ MPI_Datatype basic_type; int len; char type_name[MPI_MAX_OBJECT_NAME] = { 0 }; err = MTestInitBasicSignature(argc, argv, &count, &basic_type); if (err) return MTestReturnValue(1); err = DTP_pool_create(basic_type, count, &orig_dtp); if (err != DTP_SUCCESS) { MPI_Type_get_name(basic_type, type_name, &len); fprintf(stdout, "Error while creating orig pool (%s,%d)\n", type_name, count); fflush(stdout); } err = DTP_pool_create(basic_type, count, &target_dtp); if (err != DTP_SUCCESS) { MPI_Type_get_name(basic_type, type_name, &len); fprintf(stdout, "Error while creating target pool (%s,%d)\n", type_name, count); fflush(stdout); } #else MPI_Datatype *basic_types = NULL; int *basic_type_counts = NULL; int basic_type_num; err = MTestInitStructSignature(argc, argv, &basic_type_num, &basic_type_counts, &basic_types); if (err) return MTestReturnValue(1); err = DTP_pool_create_struct(basic_type_num, basic_types, basic_type_counts, &orig_dtp); if (err != DTP_SUCCESS) { fprintf(stdout, "Error while creating struct pool\n"); fflush(stdout); } err = DTP_pool_create_struct(basic_type_num, basic_types, basic_type_counts, &target_dtp); if (err != DTP_SUCCESS) { fprintf(stdout, "Error while creating struct pool\n"); fflush(stdout); } /* this is ignored */ count = 0; #endif while (MTestGetIntracommGeneral(&comm, minsize, 1)) { if (comm == MPI_COMM_NULL) continue; MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); orig = 0; target = size - 1; for (i = 0; i < target_dtp->DTP_num_objs; i++) { err = DTP_obj_create(target_dtp, i, 0, 0, 0); if (err != DTP_SUCCESS) { errs++; break; } targetcount = target_dtp->DTP_obj_array[i].DTP_obj_count; targettype = target_dtp->DTP_obj_array[i].DTP_obj_type; targetbuf = target_dtp->DTP_obj_array[i].DTP_obj_buf; MPI_Type_get_extent(targettype, &lb, &extent); MPI_Win_create(targetbuf, lb + targetcount * extent, (int) extent, MPI_INFO_NULL, comm, &win); for (j = 0; j < orig_dtp->DTP_num_objs; j++) { err = DTP_obj_create(orig_dtp, j, 0, 1, count); if (err != DTP_SUCCESS) { errs++; break; } origcount = orig_dtp->DTP_obj_array[j].DTP_obj_count; origtype = orig_dtp->DTP_obj_array[j].DTP_obj_type; origbuf = orig_dtp->DTP_obj_array[j].DTP_obj_buf; if (rank == orig) { MPI_Win_lock(MPI_LOCK_SHARED, target, 0, win); MPI_Accumulate(origbuf, origcount, origtype, target, 0, targetcount, targettype, MPI_REPLACE, win); MPI_Win_unlock(target, win); MPI_Barrier(comm); char *resbuf = (char *) calloc(lb + extent * targetcount, sizeof(char)); /*wait for the destination to finish checking and reinitializing the buffer */ MPI_Barrier(comm); MPI_Win_lock(MPI_LOCK_SHARED, target, 0, win); MPI_Get_accumulate(origbuf, origcount, origtype, resbuf, targetcount, targettype, target, 0, targetcount, targettype, MPI_REPLACE, win); MPI_Win_unlock(target, win); MPI_Barrier(comm); free(resbuf); } else if (rank == target) { /* TODO: add a DTP_buf_set() function to replace this */ char *tmp = (char *) calloc(lb + extent * targetcount, sizeof(char)); memcpy(tmp, targetbuf, lb + extent * targetcount); MPI_Barrier(comm); MPI_Win_lock(MPI_LOCK_SHARED, target, 0, win); err = DTP_obj_buf_check(target_dtp, i, 0, 1, count); if (err != DTP_SUCCESS) { errs++; } /* restore target buffer */ memcpy(targetbuf, tmp, lb + extent * targetcount); free(tmp); MPI_Win_unlock(target, win); /*signal the source that checking and reinitialization is done */ MPI_Barrier(comm); MPI_Barrier(comm); MPI_Win_lock(MPI_LOCK_SHARED, target, 0, win); err = DTP_obj_buf_check(target_dtp, i, 0, 1, count); if (err != DTP_SUCCESS) { errs++; } MPI_Win_unlock(target, win); } DTP_obj_free(orig_dtp, j); } MPI_Win_free(&win); DTP_obj_free(target_dtp, i); } MTestFreeComm(&comm); } DTP_pool_free(orig_dtp); DTP_pool_free(target_dtp); #ifdef USE_DTP_POOL_TYPE__STRUCT /* cleanup array if any */ if (basic_types) { free(basic_types); } if (basic_type_counts) { free(basic_type_counts); } #endif MTest_Finalize(errs); return MTestReturnValue(errs); }
int main(int argc, char **argv) { int rank, nranks, rank_world, nranks_world; int i, j, peer, bufsize, errors; double *win_buf, *src_buf, *dst_buf; MPI_Win buf_win; MPI_Comm shr_comm; MTest_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank_world); MPI_Comm_size(MPI_COMM_WORLD, &nranks_world); MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, rank, MPI_INFO_NULL, &shr_comm); MPI_Comm_rank(shr_comm, &rank); MPI_Comm_size(shr_comm, &nranks); bufsize = XDIM * YDIM * sizeof(double); MPI_Alloc_mem(bufsize, MPI_INFO_NULL, &src_buf); MPI_Alloc_mem(bufsize, MPI_INFO_NULL, &dst_buf); MPI_Win_allocate_shared(bufsize, 1, MPI_INFO_NULL, shr_comm, &win_buf, &buf_win); MPI_Win_fence(0, buf_win); for (i = 0; i < XDIM*YDIM; i++) { *(win_buf + i) = -1.0; *(src_buf + i) = 1.0 + rank; } MPI_Win_fence(0, buf_win); peer = (rank+1) % nranks; /* Perform ITERATIONS strided accumulate operations */ for (i = 0; i < ITERATIONS; i++) { int idx_rem[SUB_YDIM]; int blk_len[SUB_YDIM]; MPI_Datatype src_type, dst_type; for (j = 0; j < SUB_YDIM; j++) { idx_rem[j] = j*XDIM; blk_len[j] = SUB_XDIM; } MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &src_type); MPI_Type_indexed(SUB_YDIM, blk_len, idx_rem, MPI_DOUBLE, &dst_type); MPI_Type_commit(&src_type); MPI_Type_commit(&dst_type); /* PUT */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, 1, dst_type, MPI_REPLACE, buf_win); MPI_Win_unlock(peer, buf_win); /* GET */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win); MPI_Get_accumulate(src_buf, 1, src_type, dst_buf, 1, src_type, peer, 0, 1, dst_type, MPI_NO_OP, buf_win); MPI_Win_unlock(peer, buf_win); MPI_Type_free(&src_type); MPI_Type_free(&dst_type); } MPI_Barrier(MPI_COMM_WORLD); /* Verify that the results are correct */ MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win); errors = 0; for (i = 0; i < SUB_XDIM; i++) { for (j = 0; j < SUB_YDIM; j++) { const double actual = *(win_buf + i + j*XDIM); const double expected = (1.0 + ((rank+nranks-1)%nranks)); if (fabs(actual - expected) > 1.0e-10) { SQUELCH( printf("%d: Data validation failed at [%d, %d] expected=%f actual=%f\n", rank, j, i, expected, actual); ); errors++; fflush(stdout); } }
void IMB_rma_get_accumulate (struct comm_info* c_info, int size, struct iter_schedule* iterations, MODES run_mode, double* time) { double res_time = -1.; Type_Size s_size,r_size; int s_num, r_num; int r_off; int i; int root = c_info->pair1; ierr = 0; if (c_info->rank < 0) { *time = res_time; return; } MPI_Type_size(c_info->red_data_type,&s_size); s_num=size/s_size; r_size=s_size; r_num=s_num; r_off=iterations->r_offs/r_size; for(i=0; i<N_BARR; i++) MPI_Barrier(c_info->communicator); if (c_info->rank == c_info->pair0) { MPI_Win_lock(MPI_LOCK_SHARED, root, 0, c_info->WIN); if (run_mode->AGGREGATE) { res_time = MPI_Wtime(); for (i = 0; i < iterations->n_sample; i++) { ierr = MPI_Get_accumulate( (char*)c_info->s_buffer+i%iterations->s_cache_iter*iterations->s_offs, s_num, c_info->red_data_type, (char*)c_info->r_buffer+i%iterations->r_cache_iter*iterations->r_offs, r_num, c_info->red_data_type, root, i%iterations->r_cache_iter*r_off, r_num, c_info->red_data_type, c_info->op_type, c_info->WIN ); MPI_ERRHAND(ierr); } ierr = MPI_Win_flush(root, c_info->WIN); res_time = (MPI_Wtime() - res_time)/iterations->n_sample; } else if ( !run_mode->AGGREGATE ) { res_time = MPI_Wtime(); for (i = 0; i < iterations->n_sample; i++) { ierr = MPI_Get_accumulate( (char*)c_info->s_buffer+i%iterations->s_cache_iter*iterations->s_offs, s_num, c_info->red_data_type, (char*)c_info->r_buffer+i%iterations->r_cache_iter*iterations->r_offs, r_num, c_info->red_data_type, root, i%iterations->r_cache_iter*r_off, r_num, c_info->red_data_type, c_info->op_type, c_info->WIN ); MPI_ERRHAND(ierr); ierr = MPI_Win_flush(root, c_info->WIN); MPI_ERRHAND(ierr); } res_time = (MPI_Wtime() - res_time)/iterations->n_sample; } MPI_Win_unlock(root, c_info->WIN); } MPI_Barrier(c_info->communicator); *time = res_time; return; }
int main(int argc, char *argv[]) { int errs = 0; int rank, size; int minsize = 2, count; MPI_Comm comm; MPI_Win win; MPI_Aint lb, extent; MTestDatatype sendtype, recvtype; MTest_Init(&argc, &argv); while (MTestGetIntracommGeneral(&comm, minsize, 1)) { if (comm == MPI_COMM_NULL) continue; MPI_Comm_rank(comm, &rank); MPI_Comm_size(comm, &size); int source = 0; MTEST_DATATYPE_FOR_EACH_COUNT(count) { while (MTestGetDatatypes(&sendtype, &recvtype, count)) { recvtype.printErrors = 1; recvtype.InitBuf(&recvtype); MPI_Type_get_extent(recvtype.datatype, &lb, &extent); MPI_Win_create(recvtype.buf, lb + recvtype.count * extent, (int) extent, MPI_INFO_NULL, comm, &win); if (rank == source) { int dest; sendtype.InitBuf(&sendtype); MPI_Win_lock_all(0, win); for (dest = 0; dest < size; dest++) if (dest != source) { MPI_Accumulate(sendtype.buf, sendtype.count, sendtype.datatype, dest, 0, recvtype.count, recvtype.datatype, MPI_REPLACE, win); } MPI_Win_unlock_all(win); MPI_Barrier(comm); char *resbuf = (char *) calloc(lb + extent * recvtype.count, sizeof(char)); /*wait for the destinations to finish checking and reinitializing the buffers */ MPI_Barrier(comm); MPI_Win_lock_all(0, win); for (dest = 0; dest < size; dest++) if (dest != source) { MPI_Get_accumulate(sendtype.buf, sendtype.count, sendtype.datatype, resbuf, recvtype.count, recvtype.datatype, dest, 0, recvtype.count, recvtype.datatype, MPI_REPLACE, win); } MPI_Win_unlock_all(win); MPI_Barrier(comm); free(resbuf); } else { int err; MPI_Barrier(comm); MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); err = MTestCheckRecv(0, &recvtype); if (err) errs++; recvtype.InitBuf(&recvtype); MPI_Win_unlock(rank, win); /*signal the source that checking and reinitialization is done */ MPI_Barrier(comm); MPI_Barrier(comm); MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win); err = MTestCheckRecv(0, &recvtype); if (err) errs++; MPI_Win_unlock(rank, win); } MPI_Win_free(&win); MTestFreeDatatype(&sendtype); MTestFreeDatatype(&recvtype); } } MTestFreeComm(&comm); } MTest_Finalize(errs); MPI_Finalize(); return 0; }
/*Run GET with Post/Start/Complete/Wait */ void run_get_acc_with_pscw(int rank, WINDOW type) { int destrank, size, i; MPI_Aint disp = 0; MPI_Win win; MPI_Group comm_group, group; MPI_CHECK(MPI_Comm_group(MPI_COMM_WORLD, &comm_group)); for (size = 0; size <= MAX_SIZE; size = (size ? size * 2 : 1)) { allocate_memory(rank, rbuf, size, type, &win); if (type == WIN_DYNAMIC) { disp = sdisp_remote; } if (size > LARGE_MESSAGE_SIZE) { loop = LOOP_LARGE; skip = SKIP_LARGE; } MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); if (rank == 0) { destrank = 1; MPI_CHECK(MPI_Group_incl(comm_group, 1, &destrank, &group)); MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); for (i = 0; i < skip + loop; i++) { MPI_CHECK(MPI_Win_start (group, 0, win)); if (i == skip) { t_start = MPI_Wtime (); } MPI_CHECK(MPI_Get_accumulate(sbuf, size, MPI_CHAR, cbuf, size, MPI_CHAR, 1, disp, size, MPI_CHAR, MPI_SUM, win)); MPI_CHECK(MPI_Win_complete(win)); MPI_CHECK(MPI_Win_post(group, 0, win)); MPI_CHECK(MPI_Win_wait(win)); } t_end = MPI_Wtime (); } else { /* rank=1 */ destrank = 0; MPI_CHECK(MPI_Group_incl(comm_group, 1, &destrank, &group)); MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); for (i = 0; i < skip + loop; i++) { MPI_CHECK(MPI_Win_post(group, 0, win)); MPI_CHECK(MPI_Win_wait(win)); MPI_CHECK(MPI_Win_start(group, 0, win)); MPI_CHECK(MPI_Get_accumulate(sbuf, size, MPI_CHAR, cbuf, size, MPI_CHAR, 0, disp, size, MPI_CHAR, MPI_SUM, win)); MPI_CHECK(MPI_Win_complete(win)); } } MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD)); if (rank == 0) { fprintf(stdout, "%-*d%*.*f\n", 10, size, FIELD_WIDTH, FLOAT_PRECISION, (t_end - t_start) * 1.0e6 / loop / 2); fflush(stdout); } MPI_CHECK(MPI_Group_free(&group)); MPI_Win_free(&win); } MPI_CHECK(MPI_Group_free(&comm_group)); }
int main(int argc, char **argv) { int procid, nproc, i, j, my_nelem; int pollint = 0; double time; MPI_Win llist_win; llist_ptr_t head_ptr, tail_ptr; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &procid); MPI_Comm_size(MPI_COMM_WORLD, &nproc); MPI_Win_create_dynamic(MPI_INFO_NULL, MPI_COMM_WORLD, &llist_win); /* Process 0 creates the head node */ if (procid == 0) head_ptr.disp = alloc_elem(procid, llist_win); /* Broadcast the head pointer to everyone */ head_ptr.rank = 0; MPI_Bcast(&head_ptr.disp, 1, MPI_AINT, 0, MPI_COMM_WORLD); tail_ptr = head_ptr; /* All processes append NUM_ELEMS elements to the list; rank 0 has already * appended an element. */ if (procid == 0) i = 1; else i = 0; my_nelem = NUM_ELEMS/nproc; if (procid < NUM_ELEMS % nproc) my_nelem++; MPI_Barrier(MPI_COMM_WORLD); time = MPI_Wtime(); for ( ; i < my_nelem; i++) { llist_ptr_t new_elem_ptr; int success = 0; /* Create a new list element and register it with the window */ new_elem_ptr.rank = procid; new_elem_ptr.disp = alloc_elem(procid, llist_win); /* Append the new node to the list. This might take multiple attempts if others have already appended and our tail pointer is stale. */ do { int flag; /* The tail is at my left neighbor, append my element. */ if (tail_ptr.rank == (procid + nproc-1) % nproc) { if (verbose) printf("%d: Appending to <%d, %p>\n", procid, tail_ptr.rank, (void*) tail_ptr.disp); MPI_Win_lock(MPI_LOCK_EXCLUSIVE, tail_ptr.rank, 0, llist_win); #if USE_ACC MPI_Accumulate(&new_elem_ptr, sizeof(llist_ptr_t), MPI_BYTE, tail_ptr.rank, (MPI_Aint) &(((llist_elem_t*)tail_ptr.disp)->next), sizeof(llist_ptr_t), MPI_BYTE, MPI_REPLACE, llist_win); #else MPI_Put(&new_elem_ptr, sizeof(llist_ptr_t), MPI_BYTE, tail_ptr.rank, (MPI_Aint) &(((llist_elem_t*)tail_ptr.disp)->next), sizeof(llist_ptr_t), MPI_BYTE, llist_win); #endif MPI_Win_unlock(tail_ptr.rank, llist_win); success = 1; tail_ptr = new_elem_ptr; } /* Otherwise, chase the tail. */ else { llist_ptr_t next_tail_ptr; MPI_Win_lock(MPI_LOCK_EXCLUSIVE, tail_ptr.rank, 0, llist_win); #if USE_ACC MPI_Get_accumulate( NULL, 0, MPI_DATATYPE_NULL, &next_tail_ptr, sizeof(llist_ptr_t), MPI_BYTE, tail_ptr.rank, (MPI_Aint) &(((llist_elem_t*)tail_ptr.disp)->next), sizeof(llist_ptr_t), MPI_BYTE, MPI_NO_OP, llist_win); #else MPI_Get(&next_tail_ptr, sizeof(llist_ptr_t), MPI_BYTE, tail_ptr.rank, (MPI_Aint) &(((llist_elem_t*)tail_ptr.disp)->next), sizeof(llist_ptr_t), MPI_BYTE, llist_win); #endif MPI_Win_unlock(tail_ptr.rank, llist_win); if (next_tail_ptr.rank != nil.rank) { if (verbose) printf("%d: Chasing to <%d, %p>\n", procid, next_tail_ptr.rank, (void*) next_tail_ptr.disp); tail_ptr = next_tail_ptr; pollint = MAX(MIN_NPROBE, pollint/2); } else { for (j = 0; j < pollint; j++) MPI_Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &flag, MPI_STATUS_IGNORE); pollint = MIN(MAX_NPROBE, pollint*2); } } } while (!success); } MPI_Barrier(MPI_COMM_WORLD); time = MPI_Wtime() - time; /* Traverse the list and verify that all processes inserted exactly the correct number of elements. */ if (procid == 0) { int errors = 0; int *counts, count = 0; counts = (int*) malloc(sizeof(int) * nproc); assert(counts != NULL); for (i = 0; i < nproc; i++) counts[i] = 0; tail_ptr = head_ptr; MPI_Win_lock_all(0, llist_win); /* Walk the list and tally up the number of elements inserted by each rank */ while (tail_ptr.disp != nil.disp) { llist_elem_t elem; MPI_Get(&elem, sizeof(llist_elem_t), MPI_BYTE, tail_ptr.rank, tail_ptr.disp, sizeof(llist_elem_t), MPI_BYTE, llist_win); MPI_Win_flush(tail_ptr.rank, llist_win); tail_ptr = elem.next; assert(elem.value >= 0 && elem.value < nproc); counts[elem.value]++; count++; if (verbose) { int last_elem = tail_ptr.disp == nil.disp; printf("%2d%s", elem.value, last_elem ? "" : " -> "); if (count % ELEM_PER_ROW == 0 && !last_elem) printf("\n"); } } MPI_Win_unlock_all(llist_win); if (verbose) printf("\n\n"); /* Verify the counts we collected */ for (i = 0; i < nproc; i++) { int expected; expected = NUM_ELEMS/nproc; if (i < NUM_ELEMS % nproc) expected++; if (counts[i] != expected) { printf("Error: Rank %d inserted %d elements, expected %d\n", i, counts[i], expected); errors++; } } printf("%s\n", errors == 0 ? " No Errors" : "FAIL"); free(counts); } if (print_perf) { double max_time; MPI_Reduce(&time, &max_time, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); if (procid == 0) { printf("Total time = %0.2f sec, elem/sec = %0.2f, sec/elem = %0.2f usec\n", max_time, NUM_ELEMS/max_time, max_time/NUM_ELEMS*1.0e6); } } MPI_Win_free(&llist_win); /* Free all the elements in the list */ for ( ; my_elems_count > 0; my_elems_count--) MPI_Free_mem(my_elems[my_elems_count-1]); MPI_Finalize(); return 0; }