int exchange(int rank, starpu_data_handle_t *handles, check_func func, int detached)
{
int other_rank = rank%2 == 0 ? rank+1 : rank-1;
int i;
if (rank%2)
{
starpu_mpi_send(handles[0], other_rank, 0, MPI_COMM_WORLD);
starpu_mpi_send(handles[NB-1], other_rank, NB-1, MPI_COMM_WORLD);
for(i=1 ; i<NB-1 ; i++)
{
starpu_mpi_send(handles[i], other_rank, i, MPI_COMM_WORLD);
}
return 0;
}
else
{
int ret=0;
starpu_mpi_req req[NB];
int received = 0;
if (detached)
{
starpu_mpi_irecv_detached(handles[0], other_rank, 0, MPI_COMM_WORLD, callback, &received);
}
else
{
memset(req, 0, NB*sizeof(starpu_mpi_req));
starpu_mpi_irecv(handles[0], &req[0], other_rank, 0, MPI_COMM_WORLD);
STARPU_ASSERT(req[0] != NULL);
}
// We sleep to make sure that the data for the tag 9 will be received before the recv is posted
usleep(2000000);
for(i=1 ; i<NB ; i++)
{
if (detached)
{
starpu_mpi_irecv_detached(handles[i], other_rank, i, MPI_COMM_WORLD, callback, &received);
}
else
{
starpu_mpi_irecv(handles[i], &req[i], other_rank, i, MPI_COMM_WORLD);
STARPU_ASSERT(req[i] != NULL);
}
}
if (detached)
{
STARPU_PTHREAD_MUTEX_LOCK(&mutex);
while (received != NB)
{
FPRINTF_MPI(stderr, "Received %d messages\n", received);
STARPU_PTHREAD_COND_WAIT(&cond, &mutex);
}
STARPU_PTHREAD_MUTEX_UNLOCK(&mutex);
}
else
{
for(i=0 ; i<NB ; i++)
{
starpu_mpi_wait(&req[i], MPI_STATUS_IGNORE);
func(handles[i], i, rank, &ret);
}
}
return ret;
}
}
int main(int argc, char **argv)
{
int size, n, x=789;
int rank, other_rank;
int ret;
starpu_data_handle_t data[2];
MPI_Init(&argc, &argv);
starpu_mpi_comm_rank(MPI_COMM_WORLD, &rank);
starpu_mpi_comm_size(MPI_COMM_WORLD, &size);
if (size % 2)
{
FPRINTF(stderr, "We need a even number of processes.\n");
MPI_Finalize();
return STARPU_TEST_SKIPPED;
}
other_rank = rank%2 == 0 ? rank+1 : rank-1;
FPRINTF_MPI(stderr, "rank %d exchanging with rank %d\n", rank, other_rank);
if (rank % 2)
{
MPI_Send(&rank, 1, MPI_INT, other_rank, 10, MPI_COMM_WORLD);
FPRINTF(stderr, "[%d] sending %d\n", rank, rank);
}
else
{
MPI_Recv(&x, 1, MPI_INT, other_rank, 10, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
FPRINTF(stderr, "[%d] received %d\n", rank, x);
}
ret = starpu_init(NULL);
STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
ret = starpu_mpi_init(NULL, NULL, 0);
STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
if (rank % 2)
{
starpu_variable_data_register(&data[0], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(unsigned));
starpu_variable_data_register(&data[1], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(unsigned));
starpu_mpi_data_register(data[1], 22, 0);
}
else
starpu_variable_data_register(&data[0], -1, (uintptr_t)NULL, sizeof(unsigned));
starpu_mpi_data_register(data[0], 12, 0);
if (rank % 2)
{
starpu_mpi_req req;
starpu_mpi_issend(data[1], &req, other_rank, 22, MPI_COMM_WORLD);
starpu_mpi_send(data[0], other_rank, 12, MPI_COMM_WORLD);
starpu_mpi_wait(&req, MPI_STATUS_IGNORE);
}
else
{
int *xx;
starpu_mpi_recv(data[0], other_rank, 12, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
xx = (int *)starpu_variable_get_local_ptr(data[0]);
FPRINTF_MPI(stderr, "received %d\n", *xx);
STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x);
starpu_variable_data_register(&data[1], -1, (uintptr_t)NULL, sizeof(unsigned));
starpu_mpi_data_register(data[1], 22, 0);
starpu_mpi_recv(data[0], other_rank, 22, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
xx = (int *)starpu_variable_get_local_ptr(data[0]);
STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x);
}
starpu_data_unregister(data[0]);
starpu_data_unregister(data[1]);
starpu_mpi_shutdown();
starpu_shutdown();
MPI_Finalize();
return 0;
}
int main(int argc, char **argv)
{
int size, x;
int color;
MPI_Comm newcomm;
int rank, newrank;
int ret;
starpu_data_handle_t data[3];
int value = 90;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
if (size < 4)
{
FPRINTF(stderr, "We need at least 4 processes.\n");
MPI_Finalize();
return STARPU_TEST_SKIPPED;
}
color = rank%2;
MPI_Comm_split(MPI_COMM_WORLD, color, rank, &newcomm);
MPI_Comm_rank(newcomm, &newrank);
FPRINTF(stderr, "[%d][%d] color %d\n", rank, newrank, color);
if (newrank == 0)
{
FPRINTF(stderr, "[%d][%d] sending %d\n", rank, newrank, rank);
MPI_Send(&rank, 1, MPI_INT, 1, 10, newcomm);
}
else if (newrank == 1)
{
MPI_Recv(&x, 1, MPI_INT, 0, 10, newcomm, MPI_STATUS_IGNORE);
FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, x);
}
ret = starpu_init(NULL);
STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
ret = starpu_mpi_init(NULL, NULL, 0);
STARPU_CHECK_RETURN_VALUE(ret, "starpu_mpi_init");
if (rank == 0)
{
starpu_variable_data_register(&data[2], STARPU_MAIN_RAM, (uintptr_t)&value, sizeof(int));
}
else
starpu_variable_data_register(&data[2], -1, (uintptr_t)NULL, sizeof(int));
starpu_mpi_data_register_comm(data[2], 44, 0, MPI_COMM_WORLD);
if (newrank == 0)
{
starpu_variable_data_register(&data[0], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(int));
starpu_variable_data_register(&data[1], STARPU_MAIN_RAM, (uintptr_t)&rank, sizeof(int));
starpu_mpi_data_register_comm(data[1], 22, 0, newcomm);
}
else
starpu_variable_data_register(&data[0], -1, (uintptr_t)NULL, sizeof(int));
starpu_mpi_data_register_comm(data[0], 12, 0, newcomm);
if (newrank == 0)
{
starpu_mpi_req req[2];
starpu_mpi_issend(data[1], &req[0], 1, 22, newcomm);
starpu_mpi_isend(data[0], &req[1], 1, 12, newcomm);
starpu_mpi_wait(&req[0], MPI_STATUS_IGNORE);
starpu_mpi_wait(&req[1], MPI_STATUS_IGNORE);
}
else if (newrank == 1)
{
int *xx;
starpu_mpi_recv(data[0], 0, 12, newcomm, MPI_STATUS_IGNORE);
starpu_data_acquire(data[0], STARPU_RW);
xx = (int *)starpu_variable_get_local_ptr(data[0]);
starpu_data_release(data[0]);
FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, *xx);
STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x);
starpu_variable_data_register(&data[1], -1, (uintptr_t)NULL, sizeof(int));
starpu_mpi_data_register_comm(data[1], 22, 0, newcomm);
starpu_mpi_recv(data[0], 0, 22, newcomm, MPI_STATUS_IGNORE);
starpu_data_acquire(data[0], STARPU_RW);
xx = (int *)starpu_variable_get_local_ptr(data[0]);
starpu_data_release(data[0]);
FPRINTF(stderr, "[%d][%d] received %d\n", rank, newrank, *xx);
STARPU_ASSERT_MSG(x==*xx, "Received value %d is incorrect (should be %d)\n", *xx, x);
}
if (rank == 0)
{
starpu_data_acquire(data[2], STARPU_RW);
int rvalue = *((int *)starpu_variable_get_local_ptr(data[2]));
starpu_data_release(data[2]);
FPRINTF_MPI(stderr, "sending value %d to %d and receiving from %d\n", rvalue, 1, size-1);
starpu_mpi_send(data[2], 1, 44, MPI_COMM_WORLD);
starpu_mpi_recv(data[2], size-1, 44, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
starpu_data_acquire(data[2], STARPU_RW);
int *xx = (int *)starpu_variable_get_local_ptr(data[2]);
starpu_data_release(data[2]);
FPRINTF_MPI(stderr, "Value back is %d\n", *xx);
STARPU_ASSERT_MSG(*xx == rvalue + (2*(size-1)), "Received value %d is incorrect (should be %d)\n", *xx, rvalue + (2*(size-1)));
}
else
{
int next = (rank == size-1) ? 0 : rank+1;
starpu_mpi_recv(data[2], rank-1, 44, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
starpu_data_acquire(data[2], STARPU_RW);
int *xx = (int *)starpu_variable_get_local_ptr(data[2]);
FPRINTF_MPI(stderr, "receiving %d from %d and sending %d to %d\n", *xx, rank-1, *xx+2, next);
*xx = *xx + 2;
starpu_data_release(data[2]);
starpu_mpi_send(data[2], next, 44, MPI_COMM_WORLD);
}
if (newrank == 0 || newrank == 1)
{
starpu_mpi_insert_task(newcomm, &mycodelet,
STARPU_RW, data[0],
STARPU_VALUE, &x, sizeof(x),
STARPU_EXECUTE_ON_NODE, 1,
0);
starpu_task_wait_for_all();
starpu_data_unregister(data[0]);
starpu_data_unregister(data[1]);
}
starpu_data_unregister(data[2]);
starpu_mpi_shutdown();
starpu_shutdown();
MPI_Comm_free(&newcomm);
MPI_Finalize();
return 0;
}
