int main(int argc, char **argv) { int ret, rank, size; starpu_data_handle_t handle; int var; MPI_Init(&argc, &argv); starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank); starpu_mpi_comm_size(MPI_COMM_WORLD, &size); ret = starpu_init(NULL); STARPU_CHECK_RETURN_VALUE(ret, "starpu_init"); ret = starpu_mpi_init(NULL, NULL, 0); STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init"); if (size<3) { FPRINTF(stderr, "We need more than 2 processes.\n"); starpu_mpi_shutdown(); starpu_shutdown(); MPI_Finalize(); return STARPU_TEST_SKIPPED; } if (rank == 0) { int n; for(n=1 ; n<size ; n++) { MPI_Status status; FPRINTF_MPI(stderr, "receiving from node %d\n", n); starpu_variable_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)&var, sizeof(var)); starpu_mpi_recv(handle, n, 42, MPI_COMM_WORLD, &status); starpu_data_acquire(handle, STARPU_R); STARPU_ASSERT_MSG(var == n, "Received incorrect value <%d> from node <%d>\n", var, n); FPRINTF_MPI(stderr, "received <%d> from node %d\n", var, n); starpu_data_release(handle); starpu_data_unregister(handle); } } else { FPRINTF_MPI(stderr, "sending to node %d\n", 0); var = rank; starpu_variable_data_register(&handle, STARPU_MAIN_RAM, (uintptr_t)&var, sizeof(var)); starpu_mpi_send(handle, 0, 42, MPI_COMM_WORLD); starpu_data_unregister(handle); } starpu_mpi_shutdown(); starpu_shutdown(); MPI_Finalize(); return 0; }
int main(int argc, char **argv) { MPI_Init(NULL, NULL); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if (size != 2) { if (rank == 0) fprintf(stderr, "We need exactly 2 processes.\n"); MPI_Finalize(); return 0; } starpu_init(NULL); starpu_mpi_initialize(); tab = malloc(SIZE*sizeof(float)); starpu_vector_data_register(&tab_handle, 0, (uintptr_t)tab, SIZE, sizeof(float)); unsigned nloops = NITER; unsigned loop; int other_rank = (rank + 1)%2; for (loop = 0; loop < nloops; loop++) { if ((loop % 2) == rank) { starpu_mpi_send(tab_handle, other_rank, loop, MPI_COMM_WORLD); } else { MPI_Status status; starpu_mpi_recv(tab_handle, other_rank, loop, MPI_COMM_WORLD, &status); } } starpu_mpi_shutdown(); starpu_shutdown(); MPI_Finalize(); return 0; }
int main(int argc, char **argv) { int ret, rank, size; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if (size<3) { FPRINTF(stderr, "We need more than 2 processes.\n"); MPI_Finalize(); return STARPU_TEST_SKIPPED; } ret = starpu_init(NULL); STARPU_CHECK_RETURN_VALUE(ret, "starpu_init"); ret = starpu_mpi_init(NULL, NULL, 0); STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init"); if (rank == 0) { int n; for(n=1 ; n<size ; n++) { int i, var[2]; MPI_Status status[3]; starpu_data_handle_t handle[2]; FPRINTF_MPI(stderr, "receiving from node %d\n", n); for(i=0 ; i<2 ; i++) starpu_variable_data_register(&handle[i], STARPU_MAIN_RAM, (uintptr_t)&var[i], sizeof(var[i])); starpu_mpi_recv(handle[0], n, 42, MPI_COMM_WORLD, &status[0]); starpu_data_acquire(handle[0], STARPU_R); STARPU_ASSERT_MSG(var[0] == n, "Received incorrect value <%d> from node <%d>\n", var[0], n); FPRINTF_MPI(stderr, "received <%d> from node %d\n", var[0], n); starpu_data_release(handle[0]); starpu_mpi_recv(handle[0], n, 42, MPI_COMM_WORLD, &status[1]); starpu_mpi_recv(handle[1], n, 44, MPI_COMM_WORLD, &status[2]); for(i=0 ; i<2 ; i++) starpu_data_acquire(handle[i], STARPU_R); STARPU_ASSERT_MSG(var[0] == n*2, "Received incorrect value <%d> from node <%d>\n", var[0], n); STARPU_ASSERT_MSG(var[1] == n*4, "Received incorrect value <%d> from node <%d>\n", var[0], n); FPRINTF_MPI(stderr, "received <%d> and <%d> from node %d\n", var[0], var[1], n); for(i=0 ; i<2 ; i++) starpu_data_release(handle[i]); for(i=0 ; i<2 ; i++) starpu_data_unregister(handle[i]); } } else { int i, var[3]; starpu_data_handle_t handle[3]; FPRINTF_MPI(stderr, "sending to node %d\n", 0); var[0] = rank; var[1] = var[0] * 2; var[2] = var[0] * 4; for(i=0 ; i<3 ; i++) starpu_variable_data_register(&handle[i], STARPU_MAIN_RAM, (uintptr_t)&var[i], sizeof(var[i])); starpu_mpi_send(handle[0], 0, 42, MPI_COMM_WORLD); starpu_mpi_send(handle[1], 0, 42, MPI_COMM_WORLD); starpu_mpi_send(handle[2], 0, 44, MPI_COMM_WORLD); for(i=0 ; i<3 ; i++) starpu_data_unregister(handle[i]); } starpu_mpi_shutdown(); starpu_shutdown(); MPI_Finalize(); return 0; }
int main(int argc, char **argv) { int size, n, x=789; int rank, other_rank; int ret; starpu_data_handle_t data[2]; MPI_Init(&argc, &argv); starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank); starpu_mpi_comm_size(MPI_COMM_WORLD, &size); if (size % 2) { FPRINTF(stderr, "We need a even number of processes.\n"); MPI_Finalize(); return STARPU_TEST_SKIPPED; } other_rank = rank%2 == 0 ? rank+1 : rank-1; FPRINTF_MPI(stderr, "rank %d exchanging with rank %d\n", rank, other_rank); if (rank % 2) { MPI_Send(&rank, 1, MPI_INT, other_rank, 10, MPI_COMM_WORLD); FPRINTF(stderr, "[%d] sending %d\n", rank, rank); } else { MPI_Recv(&x, 1, MPI_INT, other_rank, 10, MPI_COMM_WORLD, MPI_STATUS_IGNORE); FPRINTF(stderr, "[%d] received %d\n", rank, x); } ret = starpu_init(NULL); STARPU_CHECK_RETURN_VALUE(ret, "starpu_init"); ret = starpu_mpi_init(NULL, NULL, 0); STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init"); if (rank % 2) { starpu_variable_data_register(&data[0], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(unsigned)); starpu_variable_data_register(&data[1], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(unsigned)); starpu_mpi_data_register(data[1], 22, 0); } else starpu_variable_data_register(&data[0], -1, (uintptr_t)NULL, sizeof(unsigned)); starpu_mpi_data_register(data[0], 12, 0); if (rank % 2) { starpu_mpi_req req; starpu_mpi_issend(data[1], &req, other_rank, 22, MPI_COMM_WORLD); starpu_mpi_send(data[0], other_rank, 12, MPI_COMM_WORLD); starpu_mpi_wait(&req, MPI_STATUS_IGNORE); } else { int *xx; starpu_mpi_recv(data[0], other_rank, 12, MPI_COMM_WORLD, MPI_STATUS_IGNORE); xx = (int *)starpu_variable_get_local_ptr(data[0]); FPRINTF_MPI(stderr, "received %d\n", *xx); STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x); starpu_variable_data_register(&data[1], -1, (uintptr_t)NULL, sizeof(unsigned)); starpu_mpi_data_register(data[1], 22, 0); starpu_mpi_recv(data[0], other_rank, 22, MPI_COMM_WORLD, MPI_STATUS_IGNORE); xx = (int *)starpu_variable_get_local_ptr(data[0]); STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x); } starpu_data_unregister(data[0]); starpu_data_unregister(data[1]); starpu_mpi_shutdown(); starpu_shutdown(); MPI_Finalize(); return 0; }
int exchange(int rank, starpu_data_handle_t *handles, check_func func, int detached) { int other_rank = rank%2 == 0 ? rank+1 : rank-1; int i; if (rank%2) { starpu_mpi_send(handles[0], other_rank, 0, MPI_COMM_WORLD); starpu_mpi_send(handles[NB-1], other_rank, NB-1, MPI_COMM_WORLD); for(i=1 ; i<NB-1 ; i++) { starpu_mpi_send(handles[i], other_rank, i, MPI_COMM_WORLD); } return 0; } else { int ret=0; starpu_mpi_req req[NB]; int received = 0; if (detached) { starpu_mpi_irecv_detached(handles[0], other_rank, 0, MPI_COMM_WORLD, callback, &received); } else { memset(req, 0, NB*sizeof(starpu_mpi_req)); starpu_mpi_irecv(handles[0], &req[0], other_rank, 0, MPI_COMM_WORLD); STARPU_ASSERT(req[0] != NULL); } // We sleep to make sure that the data for the tag 9 will be received before the recv is posted usleep(2000000); for(i=1 ; i<NB ; i++) { if (detached) { starpu_mpi_irecv_detached(handles[i], other_rank, i, MPI_COMM_WORLD, callback, &received); } else { starpu_mpi_irecv(handles[i], &req[i], other_rank, i, MPI_COMM_WORLD); STARPU_ASSERT(req[i] != NULL); } } if (detached) { STARPU_PTHREAD_MUTEX_LOCK(&mutex); while (received != NB) { FPRINTF_MPI(stderr, "Received %d messages\n", received); STARPU_PTHREAD_COND_WAIT(&cond, &mutex); } STARPU_PTHREAD_MUTEX_UNLOCK(&mutex); } else { for(i=0 ; i<NB ; i++) { starpu_mpi_wait(&req[i], MPI_STATUS_IGNORE); func(handles[i], i, rank, &ret); } } return ret; } }
int do_test(int rank, int sdetached, int rdetached) { int ret, i; int val[2]; starpu_data_handle_t data[2]; ret = starpu_init(NULL); STARPU_CHECK_RETURN_VALUE(ret, "starpu_init"); ret = starpu_mpi_init(NULL, NULL, 0); STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init"); if (rank == 1) { val[0] = VAL0; val[1] = VAL1; } else { val[0] = -1; val[1] = -1; } starpu_variable_data_register(&data[0], STARPU_MAIN_RAM, (uintptr_t)&val[0], sizeof(val[0])); starpu_variable_data_register(&data[1], STARPU_MAIN_RAM, (uintptr_t)&val[1], sizeof(val[1])); starpu_mpi_data_register(data[0], 77, 1); starpu_mpi_data_register(data[1], 88, 1); if (rank == 1) { for(i=1 ; i>=0 ; i--) { if (sdetached) starpu_mpi_isend_detached(data[i], 0, starpu_data_get_tag(data[i]), MPI_COMM_WORLD, NULL, NULL); else starpu_mpi_send(data[i], 0, starpu_data_get_tag(data[i]), MPI_COMM_WORLD); } } else if (rank == 0) { int received = 0; for(i=0 ; i<2 ; i++) FPRINTF_MPI(stderr, "Value[%d] = %d\n", i, val[i]); for(i=0 ; i<2 ; i++) { if (rdetached) starpu_mpi_irecv_detached(data[i], 1, starpu_data_get_tag(data[i]), MPI_COMM_WORLD, callback, &received); else starpu_mpi_recv(data[i], 1, starpu_data_get_tag(data[i]), MPI_COMM_WORLD, MPI_STATUS_IGNORE); } if (rdetached) { STARPU_PTHREAD_MUTEX_LOCK(&mutex); while (received != 2) { FPRINTF_MPI(stderr, "Received %d messages\n", received); STARPU_PTHREAD_COND_WAIT(&cond, &mutex); } STARPU_PTHREAD_MUTEX_UNLOCK(&mutex); } for(i=0 ; i<2 ; i++) starpu_data_acquire(data[i], STARPU_R); for(i=0 ; i<2 ; i++) FPRINTF_MPI(stderr, "Value[%d] = %d\n", i, val[i]); for(i=0 ; i<2 ; i++) starpu_data_release(data[i]); } FPRINTF_MPI(stderr, "Waiting ...\n"); starpu_task_wait_for_all(); starpu_data_unregister(data[0]); starpu_data_unregister(data[1]); if (rank == 0) { ret = (val[0] == VAL0 && val[1] == VAL1) ? 0 : 1; } starpu_mpi_shutdown(); starpu_shutdown(); return ret; }
int main(int argc, char **argv) { int size, x; int color; MPI_Comm newcomm; int rank, newrank; int ret; starpu_data_handle_t data[3]; int value = 90; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if (size < 4) { FPRINTF(stderr, "We need at least 4 processes.\n"); MPI_Finalize(); return STARPU_TEST_SKIPPED; } color = rank%2; MPI_Comm_split(MPI_COMM_WORLD, color, rank, &newcomm); MPI_Comm_rank(newcomm, &newrank); FPRINTF(stderr, "[%d][%d] color %d\n", rank, newrank, color); if (newrank == 0) { FPRINTF(stderr, "[%d][%d] sending %d\n", rank, newrank, rank); MPI_Send(&rank, 1, MPI_INT, 1, 10, newcomm); } else if (newrank == 1) { MPI_Recv(&x, 1, MPI_INT, 0, 10, newcomm, MPI_STATUS_IGNORE); FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, x); } ret = starpu_init(NULL); STARPU_CHECK_RETURN_VALUE(ret, "starpu_init"); ret = starpu_mpi_init(NULL, NULL, 0); STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init"); if (rank == 0) { starpu_variable_data_register(&data[2], STARPU_MAIN_RAM, (uintptr_t)&value, sizeof(int)); } else starpu_variable_data_register(&data[2], -1, (uintptr_t)NULL, sizeof(int)); starpu_mpi_data_register_comm(data[2], 44, 0, MPI_COMM_WORLD); if (newrank == 0) { starpu_variable_data_register(&data[0], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(int)); starpu_variable_data_register(&data[1], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(int)); starpu_mpi_data_register_comm(data[1], 22, 0, newcomm); } else starpu_variable_data_register(&data[0], -1, (uintptr_t)NULL, sizeof(int)); starpu_mpi_data_register_comm(data[0], 12, 0, newcomm); if (newrank == 0) { starpu_mpi_req req[2]; starpu_mpi_issend(data[1], &req[0], 1, 22, newcomm); starpu_mpi_isend(data[0], &req[1], 1, 12, newcomm); starpu_mpi_wait(&req[0], MPI_STATUS_IGNORE); starpu_mpi_wait(&req[1], MPI_STATUS_IGNORE); } else if (newrank == 1) { int *xx; starpu_mpi_recv(data[0], 0, 12, newcomm, MPI_STATUS_IGNORE); starpu_data_acquire(data[0], STARPU_RW); xx = (int *)starpu_variable_get_local_ptr(data[0]); starpu_data_release(data[0]); FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, *xx); STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x); starpu_variable_data_register(&data[1], -1, (uintptr_t)NULL, sizeof(int)); starpu_mpi_data_register_comm(data[1], 22, 0, newcomm); starpu_mpi_recv(data[0], 0, 22, newcomm, MPI_STATUS_IGNORE); starpu_data_acquire(data[0], STARPU_RW); xx = (int *)starpu_variable_get_local_ptr(data[0]); starpu_data_release(data[0]); FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, *xx); STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x); } if (rank == 0) { starpu_data_acquire(data[2], STARPU_RW); int rvalue = *((int *)starpu_variable_get_local_ptr(data[2])); starpu_data_release(data[2]); FPRINTF_MPI(stderr, "sending value %d to %d and receiving from %d\n", rvalue, 1, size-1); starpu_mpi_send(data[2], 1, 44, MPI_COMM_WORLD); starpu_mpi_recv(data[2], size-1, 44, MPI_COMM_WORLD, MPI_STATUS_IGNORE); starpu_data_acquire(data[2], STARPU_RW); int *xx = (int *)starpu_variable_get_local_ptr(data[2]); starpu_data_release(data[2]); FPRINTF_MPI(stderr, "Value back is %d\n", *xx); STARPU_ASSERT_MSG(*xx == rvalue + (2*(size-1)), "Received value %d is incorrect (should be %d)\n", *xx, rvalue + (2*(size-1))); } else { int next = (rank == size-1) ? 0 : rank+1; starpu_mpi_recv(data[2], rank-1, 44, MPI_COMM_WORLD, MPI_STATUS_IGNORE); starpu_data_acquire(data[2], STARPU_RW); int *xx = (int *)starpu_variable_get_local_ptr(data[2]); FPRINTF_MPI(stderr, "receiving %d from %d and sending %d to %d\n", *xx, rank-1, *xx+2, next); *xx = *xx + 2; starpu_data_release(data[2]); starpu_mpi_send(data[2], next, 44, MPI_COMM_WORLD); } if (newrank == 0 || newrank == 1) { starpu_mpi_insert_task(newcomm, &mycodelet, STARPU_RW, data[0], STARPU_VALUE, &x, sizeof(x), STARPU_EXECUTE_ON_NODE, 1, 0); starpu_task_wait_for_all(); starpu_data_unregister(data[0]); starpu_data_unregister(data[1]); } starpu_data_unregister(data[2]); starpu_mpi_shutdown(); starpu_shutdown(); MPI_Comm_free(&newcomm); MPI_Finalize(); return 0; }
int main(int argc, char **argv) { int ret, rank, size; MPI_Init(&argc, &argv); MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if (size < 2) { if (rank == 0) FPRINTF(stderr, "We need at least 2 processes.\n"); MPI_Finalize(); return STARPU_TEST_SKIPPED; } ret = starpu_init(NULL); STARPU_CHECK_RETURN_VALUE(ret, "starpu_init"); ret = starpu_mpi_init(NULL, NULL, 0); STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init"); starpu_vector_data_register(&token_handle, STARPU_MAIN_RAM, (uintptr_t)&token, 1, sizeof(token)); int nloops = NITER; int loop; int last_loop = nloops - 1; int last_rank = size - 1; for (loop = 0; loop < nloops; loop++) { int tag = loop*size + rank; if (loop == 0 && rank == 0) { token = 0; FPRINTF(stdout, "Start with token value %u\n", token); } else { MPI_Status status; starpu_mpi_recv(token_handle, (rank+size-1)%size, tag, MPI_COMM_WORLD, &status); } increment_token(); if (loop == last_loop && rank == last_rank) { starpu_data_acquire(token_handle, STARPU_R); FPRINTF(stdout, "Finished : token value %u\n", token); starpu_data_release(token_handle); } else { starpu_mpi_send(token_handle, (rank+1)%size, tag+1, MPI_COMM_WORLD); } } starpu_data_unregister(token_handle); starpu_mpi_shutdown(); starpu_shutdown(); MPI_Finalize(); if (rank == last_rank) { STARPU_ASSERT(token == nloops*size); } return 0; }
int main(int argc, char **argv) { MPI_Init(NULL, NULL); int rank, size; MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &size); if (size < 2) { if (rank == 0) fprintf(stderr, "We need at least processes.\n"); MPI_Finalize(); return 0; } /* We only use 2 nodes for that test */ if (rank >= 2) { MPI_Finalize(); return 0; } starpu_init(NULL); starpu_mpi_initialize(); /* Node 0 will allocate a big block and only register an inner part of * it as the block data, Node 1 will allocate a block of small size and * register it directly. Node 0 and 1 will then exchange the content of * their blocks. */ float *block; starpu_data_handle block_handle; if (rank == 0) { block = calloc(BIGSIZE*BIGSIZE*BIGSIZE, sizeof(float)); assert(block); /* fill the inner block */ unsigned i, j, k; for (k = 0; k < SIZE; k++) for (j = 0; j < SIZE; j++) for (i = 0; i < SIZE; i++) { block[i + j*BIGSIZE + k*BIGSIZE*BIGSIZE] = 1.0f; } starpu_block_data_register(&block_handle, 0, (uintptr_t)block, BIGSIZE, BIGSIZE*BIGSIZE, SIZE, SIZE, SIZE, sizeof(float)); } else /* rank == 1 */ { block = calloc(SIZE*SIZE*SIZE, sizeof(float)); assert(block); starpu_block_data_register(&block_handle, 0, (uintptr_t)block, SIZE, SIZE*SIZE, SIZE, SIZE, SIZE, sizeof(float)); } if (rank == 0) { starpu_mpi_send(block_handle, 1, 0x42, MPI_COMM_WORLD); MPI_Status status; starpu_mpi_recv(block_handle, 1, 0x1337, MPI_COMM_WORLD, &status); /* check the content of the block */ starpu_data_acquire(block_handle, STARPU_R); unsigned i, j, k; for (k = 0; k < SIZE; k++) for (j = 0; j < SIZE; j++) for (i = 0; i < SIZE; i++) { assert(block[i + j*BIGSIZE + k*BIGSIZE*BIGSIZE] == 33.0f); } starpu_data_release(block_handle); } else /* rank == 1 */ { MPI_Status status; starpu_mpi_recv(block_handle, 0, 0x42, MPI_COMM_WORLD, &status); /* check the content of the block and modify it */ starpu_data_acquire(block_handle, STARPU_RW); unsigned i, j, k; for (k = 0; k < SIZE; k++) for (j = 0; j < SIZE; j++) for (i = 0; i < SIZE; i++) { assert(block[i + j*SIZE + k*SIZE*SIZE] == 1.0f); block[i + j*SIZE + k*SIZE*SIZE] = 33.0f; } starpu_data_release(block_handle); starpu_mpi_send(block_handle, 0, 0x1337, MPI_COMM_WORLD); } fprintf(stdout, "Rank %d is done\n", rank); fflush(stdout); starpu_mpi_shutdown(); starpu_shutdown(); MPI_Finalize(); return 0; }