int main(int argc, char *argv[])
{
MPI_Win win;
int flag, tmp, rank;
int base[1024], errs = 0;
MPI_Request req;
MTest_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Win_create(base, 1024 * sizeof(int), sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &win);
if (rank == 0) {
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, win);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Win_unlock(0, win);
} else {
MPI_Barrier(MPI_COMM_WORLD);
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 0, 0, win);
MPI_Rput(&tmp, 1, MPI_INT, 0, 0, 1, MPI_INT, win, &req);
MPI_Test(&req, &flag, MPI_STATUS_IGNORE);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Win_unlock(0, win);
}
MPI_Win_free(&win);
MTest_Finalize(errs);
return MTestReturnValue(errs);
}
int main(int argc, char **argv) {
int i, rank, nproc, mpi_type_size;
int errors = 0, all_errors = 0;
TYPE_C *val_ptr, *res_ptr;
MPI_Win win;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Type_size(TYPE_MPI, &mpi_type_size);
assert(mpi_type_size == sizeof(TYPE_C));
val_ptr = malloc(sizeof(TYPE_C)*nproc);
res_ptr = malloc(sizeof(TYPE_C)*nproc);
MPI_Win_create(val_ptr, sizeof(TYPE_C)*nproc, sizeof(TYPE_C), MPI_INFO_NULL, MPI_COMM_WORLD, &win);
/* Test self communication */
reset_vars(val_ptr, res_ptr, win);
for (i = 0; i < ITER; i++) {
TYPE_C one = 1, result = -1;
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
MPI_Fetch_and_op(&one, &result, TYPE_MPI, rank, 0, MPI_SUM, win);
MPI_Win_unlock(rank, win);
}
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
if ( CMP(val_ptr[0], ITER) ) {
SQUELCH( printf("%d->%d -- SELF: expected "TYPE_FMT", got "TYPE_FMT"\n", rank, rank, (TYPE_C) ITER, val_ptr[0]); );
errors++;
}
int main(int argc, char *argv[])
{
int rank, nprocs, A[SIZE2], B[SIZE2], i;
MPI_Win win;
int errs = 0;
MTest_Init(&argc,&argv);
MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
if (nprocs != 2) {
printf("Run this program with 2 processes\n");
MPI_Abort(MPI_COMM_WORLD,1);
}
if (rank == 0) {
for (i=0; i<SIZE2; i++) A[i] = B[i] = i;
MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win);
for (i=0; i<SIZE1; i++) {
MPI_Win_lock(MPI_LOCK_SHARED, 1, 0, win);
MPI_Put(A+i, 1, MPI_INT, 1, i, 1, MPI_INT, win);
MPI_Win_unlock(1, win);
}
for (i=0; i<SIZE1; i++) {
MPI_Win_lock(MPI_LOCK_SHARED, 1, 0, win);
MPI_Get(B+i, 1, MPI_INT, 1, SIZE1+i, 1, MPI_INT, win);
MPI_Win_unlock(1, win);
}
MPI_Win_free(&win);
for (i=0; i<SIZE1; i++)
if (B[i] != (-4)*(i+SIZE1)) {
printf("Get Error: B[%d] is %d, should be %d\n", i, B[i], (-4)*(i+SIZE1));
errs++;
}
}
else { /* rank=1 */
for (i=0; i<SIZE2; i++) B[i] = (-4)*i;
MPI_Win_create(B, SIZE2*sizeof(int), sizeof(int), MPI_INFO_NULL,
MPI_COMM_WORLD, &win);
MPI_Win_free(&win);
for (i=0; i<SIZE1; i++) {
if (B[i] != i) {
printf("Put Error: B[%d] is %d, should be %d\n", i, B[i], i);
errs++;
}
}
}
/* if (rank==0) printf("Done\n");*/
MTest_Finalize(errs);
MPI_Finalize();
return 0;
}
int read_last_task_own( task_type_unit * task0, int target_rank) // queue_del
{
// return codes:
// 0 - element read
// 1 - q is empty
int ret = 0;
int iamfree = 0;
int my_offset;
int i;
while(iamfree == 0 ) //try to lock offs window putting -2 value
{
MPI_Win_lock( MPI_LOCK_EXCLUSIVE, target_rank, 0, win_offs );
my_offset = OFFSET[0];
OFFSET[0] = lock;
MPI_Win_unlock( target_rank, win_offs );
if(my_offset >= -1) //if the window was not locked before
{
iamfree = 1;
}
}
// offs window is now locked by me! work!
if(my_offset == -1) //q is empty
{
ret = 1;
}
else
{
// get params
//if(ts_logging==1)
{
sched_log_file = fopen(sched_log,"a");
fprintf(sched_log_file, "[%f] Take task N %d\n", MPI_Wtime(), my_offset);
fclose(sched_log_file);
}
MPI_Win_lock( MPI_LOCK_EXCLUSIVE, target_rank, 0, win_q ); //lock the q
for(i=0; i<task_type_length; i++)
{
task0[i] = QUEUE[task_type_length * my_offset + i]; // or use memcpy
}
MPI_Win_unlock( target_rank, win_q );
my_offset--;
}
MPI_Win_lock( MPI_LOCK_EXCLUSIVE, target_rank, 0, win_offs );
OFFSET[0] = my_offset; //UNBLOCK OFFSET win (put proper offs val - either changed or not)
MPI_Win_unlock( target_rank, win_offs );
return(ret);
}
void test_put(void)
{
int me, nproc;
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Comm_rank(MPI_COMM_WORLD, &me);
MPI_Win dst_win;
double *dst_buf;
double src_buf[MAXELEMS];
int i, j;
MPI_Alloc_mem(sizeof(double) * nproc * MAXELEMS, MPI_INFO_NULL, &dst_buf);
MPI_Win_create(dst_buf, sizeof(double) * nproc * MAXELEMS, 1, MPI_INFO_NULL, MPI_COMM_WORLD,
&dst_win);
for (i = 0; i < MAXELEMS; i++)
src_buf[i] = me + 1.0;
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, me, 0, dst_win);
for (i = 0; i < nproc * MAXELEMS; i++)
dst_buf[i] = 0.0;
MPI_Win_unlock(me, dst_win);
MPI_Barrier(MPI_COMM_WORLD);
for (i = 0; i < nproc; i++) {
int target = i;
for (j = 0; j < COUNT; j++) {
if (verbose)
printf("%2d -> %2d [%2d]\n", me, target, j);
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, dst_win);
MPI_Put(&src_buf[j], sizeof(double), MPI_BYTE, target,
(me * MAXELEMS + j) * sizeof(double), sizeof(double), MPI_BYTE, dst_win);
MPI_Win_unlock(target, dst_win);
}
for (j = 0; j < COUNT; j++) {
if (verbose)
printf("%2d <- %2d [%2d]\n", me, target, j);
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, target, 0, dst_win);
MPI_Get(&src_buf[j], sizeof(double), MPI_BYTE, target,
(me * MAXELEMS + j) * sizeof(double), sizeof(double), MPI_BYTE, dst_win);
MPI_Win_unlock(target, dst_win);
}
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Win_free(&dst_win);
MPI_Free_mem(dst_buf);
}
int main(int argc, char **argv)
{
int i, j, rank, nranks, peer, bufsize, errors;
double *buffer, *src_buf;
MPI_Win buf_win;
MTest_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nranks);
bufsize = XDIM * YDIM * sizeof(double);
MPI_Alloc_mem(bufsize, MPI_INFO_NULL, &buffer);
MPI_Alloc_mem(bufsize, MPI_INFO_NULL, &src_buf);
for (i = 0; i < XDIM * YDIM; i++) {
*(buffer + i) = 1.0 + rank;
*(src_buf + i) = 1.0 + rank;
}
MPI_Win_create(buffer, bufsize, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &buf_win);
peer = (rank + 1) % nranks;
for (i = 0; i < ITERATIONS; i++) {
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, peer, 0, buf_win);
for (j = 0; j < YDIM; j++) {
MPI_Accumulate(src_buf + j * XDIM, XDIM, MPI_DOUBLE, peer,
j * XDIM * sizeof(double), XDIM, MPI_DOUBLE, MPI_SUM, buf_win);
}
MPI_Win_unlock(peer, buf_win);
}
MPI_Barrier(MPI_COMM_WORLD);
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, buf_win);
for (i = errors = 0; i < XDIM; i++) {
for (j = 0; j < YDIM; j++) {
const double actual = *(buffer + i + j * XDIM);
const double expected =
(1.0 + rank) + (1.0 + ((rank + nranks - 1) % nranks)) * (ITERATIONS);
if (fabs(actual - expected) > 1.0e-10) {
SQUELCH(printf("%d: Data validation failed at [%d, %d] expected=%f actual=%f\n",
rank, j, i, expected, actual););
errors++;
fflush(stdout);
}
}
/** Lock a mutex.
*
* @param[in] hdl Mutex group that the mutex belongs to.
* @param[in] mutex Desired mutex number [0..count-1]
* @param[in] world_proc Absolute ID of process where the mutex lives
*/
void ARMCIX_Lock_hdl(armcix_mutex_hdl_t hdl, int mutex, int world_proc) {
int rank, nproc, proc;
long lock_val, unlock_val, lock_out;
int timeout = 1;
MPI_Comm_rank(hdl->comm, &rank);
MPI_Comm_size(hdl->comm, &nproc);
/* User gives us the absolute ID. Translate to the rank in the mutex's group. */
proc = ARMCII_Translate_absolute_to_group(hdl->comm, world_proc);
ARMCII_Assert(proc >= 0);
lock_val = rank+1; // Map into range 1..nproc
unlock_val = -1 * (rank+1);
/* mutex <- mutex + rank */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->window);
MPI_Accumulate(&lock_val, 1, MPI_LONG, proc, mutex, 1, MPI_LONG, MPI_SUM, hdl->window);
MPI_Win_unlock(proc, hdl->window);
for (;;) {
/* read mutex value */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->window);
MPI_Get(&lock_out, 1, MPI_LONG, proc, mutex, 1, MPI_LONG, hdl->window);
MPI_Win_unlock(proc, hdl->window);
ARMCII_Assert(lock_out > 0);
ARMCII_Assert(lock_out <= nproc*(nproc+1)/2); // Must be < sum of all ranks
/* We are holding the mutex */
if (lock_out == rank+1)
break;
/* mutex <- mutex - rank */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->window);
MPI_Accumulate(&unlock_val, 1, MPI_LONG, proc, mutex, 1, MPI_LONG, MPI_SUM, hdl->window);
MPI_Win_unlock(proc, hdl->window);
/* Exponential backoff */
usleep(timeout + rand()%timeout);
timeout = MIN(timeout*TIMEOUT_MUL, MAX_TIMEOUT);
if (rand() % nproc == 0) // Chance to reset timeout
timeout = 1;
/* mutex <- mutex + rank */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->window);
MPI_Accumulate(&lock_val, 1, MPI_LONG, proc, mutex, 1, MPI_LONG, MPI_SUM, hdl->window);
MPI_Win_unlock(proc, hdl->window);
}
}
int main( int argc, char *argv[] )
{
int errs = 0;
int rank, size, i;
MPI_Comm comm;
MPI_Win win;
int *winbuf, count;
MTest_Init( &argc, &argv );
comm = MPI_COMM_WORLD;
MPI_Comm_rank( comm, &rank );
MPI_Comm_size( comm, &size );
/* Allocate and initialize buf */
count = 1000;
MPI_Alloc_mem( count*sizeof(int), MPI_INFO_NULL, &winbuf );
MPI_Win_create( winbuf, count * sizeof(int), sizeof(int), MPI_INFO_NULL,
comm, &win );
/* Clear winbuf */
memset( winbuf, 0, count*sizeof(int) );
/* Note that for i == rank, this is a useful operation - it allows
the programmer to use direct loads and stores, rather than
put/get/accumulate, to access the local memory window. */
for (i=0; i<size; i++) {
MPI_Win_lock( MPI_LOCK_EXCLUSIVE, i, 0, win );
MPI_Win_unlock( i, win );
}
for (i=0; i<size; i++) {
MPI_Win_lock( MPI_LOCK_SHARED, i, 0, win );
MPI_Win_unlock( i, win );
}
MPI_Win_free( &win );
MPI_Free_mem( winbuf );
/* If this test completes, no error has been found */
/* A more complete test may ensure that local locks in fact block
remote, exclusive locks */
MTest_Finalize( errs );
MPI_Finalize();
return 0;
}
int main(int argc, char *argv[])
{
int rank, nprocs, A[SIZE2], B[SIZE2], i, j;
MPI_Comm CommDeuce;
MPI_Win win;
int errs = 0;
MTest_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (nprocs < 2) {
printf("Run this program with 2 or more processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Comm_split(MPI_COMM_WORLD, (rank < 2), rank, &CommDeuce);
if (rank < 2) {
if (rank == 0) {
for (i = 0; i < SIZE2; i++)
A[i] = B[i] = i;
MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, CommDeuce, &win);
for (j = 0; j < 2; j++) {
for (i = 0; i < SIZE1; i++) {
MPI_Win_lock(MPI_LOCK_SHARED, 1, j == 0 ? 0 : MPI_MODE_NOCHECK, win);
MPI_Put(A + i, 1, MPI_INT, 1, i, 1, MPI_INT, win);
MPI_Win_unlock(1, win);
}
for (i = 0; i < SIZE1; i++) {
MPI_Win_lock(MPI_LOCK_SHARED, 1, j == 0 ? 0 : MPI_MODE_NOCHECK, win);
MPI_Get(B + i, 1, MPI_INT, 1, SIZE1 + i, 1, MPI_INT, win);
MPI_Win_unlock(1, win);
}
}
MPI_Win_free(&win);
for (i = 0; i < SIZE1; i++)
if (B[i] != (-4) * (i + SIZE1)) {
SQUELCH(printf
("Get Error: B[%d] is %d, should be %d\n", i, B[i],
(-4) * (i + SIZE1)););
errs++;
}
}
void MPIMutex::lock(int proc) {
int rank, nproc, already_locked;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &nproc);
//std::cout << "trying to get lock" << std::endl;
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, win);
byte *buff = (byte*)malloc(sizeof(byte)*nproc);
buff[rank] = 1;
MPI_Put(&(buff[rank]), 1, MPI_BYTE, proc, rank, 1, MPI_BYTE, win);
/* Get data to the left of rank */
if (rank > 0) {
MPI_Get(buff, rank, MPI_BYTE, proc, 0, rank, MPI_BYTE, win);
}
/* Get data to the right of rank */
if (rank < nproc - 1) {
MPI_Get(&(buff[rank+1]), nproc-1-rank, MPI_BYTE, proc, rank+1, nproc-1-rank,
MPI_BYTE, win);
}
MPI_Win_unlock(proc, win);
/* check if anyone has the lock*/
for (int i = already_locked = 0; i < nproc; i++)
if (buff[i] && i != rank)
already_locked = 1;
/* Wait for notification */
if (already_locked) {
MPI_Status status;
//std::cout << "waiting for notification [proc = "<<proc<<"]" << std::endl;
MPI_Recv(NULL, 0, MPI_BYTE, MPI_ANY_SOURCE, MPI_MUTEX_TAG+id, comm, &status);
}
//std::cout << "lock acquired [proc = "<<proc<<"]" << std::endl;
free(buff);
};
bool MPIMutex::try_lock(int proc) {
int rank, nproc, already_locked;
MPI_Comm_rank(comm, &rank);
MPI_Comm_size(comm, &nproc);
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, win);
byte *buff = (byte*)malloc(sizeof(byte)*nproc);
buff[rank] = 1;
MPI_Put(&(buff[rank]), 1, MPI_BYTE, proc, rank, 1, MPI_BYTE, win);
/* Get data to the left of rank */
if (rank > 0) {
MPI_Get(buff, rank, MPI_BYTE, proc, 0, rank, MPI_BYTE, win);
}
/* Get data to the right of rank */
if (rank < nproc - 1) {
MPI_Get(&(buff[rank+1]), nproc-1-rank, MPI_BYTE, proc, rank+1, nproc-1-rank,
MPI_BYTE, win);
}
MPI_Win_unlock(proc, win);
/* check if anyone has the lock*/
already_locked = 1; //1 means succesfully got the lock
for (int i = 0; i < nproc; i++)
if (buff[i] && i != rank)
already_locked = 0;
free(buff);
return already_locked;
};
int main(int argc, char **argv)
{
int rank, nproc;
int out_val, i, counter = 0;
MPI_Win win;
MTest_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Win_create(&counter, sizeof(int), sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &win);
for (i = 0; i < NITER; i++) {
MPI_Win_lock(MPI_LOCK_SHARED, rank, 0, win);
MPI_Get_accumulate(&acc_val, 1, MPI_INT, &out_val, 1, MPI_INT,
rank, 0, 1, MPI_INT, MPI_SUM, win);
MPI_Win_unlock(rank, win);
if (out_val != acc_val * i) {
errs++;
printf("Error: got %d, expected %d at iter %d\n", out_val, acc_val * i, i);
break;
}
}
MPI_Win_free(&win);
MTest_Finalize(errs);
return MTestReturnValue(errs);
}
/*Run FOP with Lock/unlock */
void run_fop_with_lock(int rank, WINDOW type)
{
int i;
MPI_Aint disp = 0;
MPI_Win win;
allocate_atomic_memory(rank, sbuf_original, rbuf_original,
tbuf_original, NULL, (char **)&sbuf, (char **)&rbuf,
(char **)&tbuf, NULL, (char **)&rbuf, MAX_MSG_SIZE, type, &win);
if(rank == 0) {
if (type == WIN_DYNAMIC) {
disp = disp_remote;
}
for (i = 0; i < skip + loop; i++) {
if (i == skip) {
t_start = MPI_Wtime ();
}
MPI_CHECK(MPI_Win_lock(MPI_LOCK_EXCLUSIVE, 1, 0, win));
MPI_CHECK(MPI_Fetch_and_op(sbuf, tbuf, MPI_LONG_LONG, 1, disp, MPI_SUM, win));
MPI_CHECK(MPI_Win_unlock(1, win));
}
t_end = MPI_Wtime ();
}
MPI_CHECK(MPI_Barrier(MPI_COMM_WORLD));
print_latency(rank, 8);
free_atomic_memory (sbuf, rbuf, tbuf, NULL, win, rank);
}
inline void SpParHelper::LockWindows(int ownind, vector<MPI_Win> & arrwin)
{
for(vector<MPI_Win>::iterator itr = arrwin.begin(); itr != arrwin.end(); ++itr)
{
MPI_Win_lock(MPI_LOCK_SHARED, ownind, 0, *itr);
}
}
/*
* Class: mpi_Win
* Method: lock
* Signature: (JIII)V
*/
JNIEXPORT void JNICALL Java_mpi_Win_lock(
JNIEnv *env, jobject jthis, jlong win,
jint lockType, jint rank, jint assertion)
{
int rc = MPI_Win_lock(lockType, rank, assertion, (MPI_Win)win);
ompi_java_exceptionCheck(env, rc);
}
int main(int argc, char *argv[])
{
int rank, nproc;
int errors = 0, all_errors = 0;
int buf, my_buf;
MPI_Win win;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Win_create(&buf, sizeof(int), sizeof(int),
MPI_INFO_NULL, MPI_COMM_WORLD, &win);
MPI_Win_set_errhandler(win, MPI_ERRORS_RETURN);
MPI_Win_fence(0, win);
MPI_Win_lock(MPI_LOCK_SHARED, 0, MPI_MODE_NOCHECK, win);
MPI_Get(&my_buf, 1, MPI_INT, 0, 0, 1, MPI_INT, win);
MPI_Win_unlock(0, win);
/* This should fail because the window is no longer in a fence epoch */
CHECK_ERR(MPI_Get(&my_buf, 1, MPI_INT, 0, 0, 1, MPI_INT, win));
MPI_Win_fence(0, win);
MPI_Win_free(&win);
MPI_Reduce(&errors, &all_errors, 1, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
if (rank == 0 && all_errors == 0) printf(" No Errors\n");
MPI_Finalize();
return 0;
}
void add_element_unsorted( task_type task0, int rank )
{
int iamfree = 0;
int my_offset;
int i;
while(iamfree == 0) //try to lock offs window putting -2 value
{
MPI_Win_lock( MPI_LOCK_EXCLUSIVE, rank, 0, win_offs );
my_offset = OFFSET[0];
OFFSET[0] = lock;
MPI_Win_unlock( rank, win_offs );
if(my_offset >= -1) //if the window was not locked before
{
iamfree = 1;
}
}
//if(ts_logging==1)
{
sched_log_file = fopen(sched_log,"a");
fprintf(sched_log_file, "[%f] Adding task [%3.1f][%6.4f] to my queue\n", MPI_Wtime(), task0[0], task0[1]);
fclose(sched_log_file);
}
my_offset++;
MPI_Win_lock( MPI_LOCK_EXCLUSIVE, rank, 0, win_q ); //lock the q
for(i=0; i<task_type_length; i++)
{
QUEUE[task_type_length * my_offset + i] = task0[i]; // or use memcpy
}
MPI_Win_unlock( rank, win_q );
//if(ts_logging==1)
{
sched_log_file = fopen(sched_log,"a");
fprintf(sched_log_file, "[%f] New number of tasks is N %d\n", MPI_Wtime(), my_offset);
fclose(sched_log_file);
}
MPI_Win_lock( MPI_LOCK_EXCLUSIVE, rank, 0, win_offs );
OFFSET[0] = my_offset;
MPI_Win_unlock( rank, win_offs );
}
int main(int argc, char **argv)
{
int rank, size;
MPI_Win win = MPI_WIN_NULL;
int *baseptr = NULL;
int errs = 0, mpi_errno = MPI_SUCCESS;
int val1 = 0, val2 = 0, flag = 0;
MPI_Request reqs[2];
MPI_Status stats[2];
MTest_Init(&argc, &argv);
MPI_Comm_size(MPI_COMM_WORLD, &size);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Errhandler_set(MPI_COMM_WORLD, MPI_ERRORS_RETURN);
MPI_Win_allocate(2 * sizeof(int), sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &baseptr, &win);
/* Initialize window buffer */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
baseptr[0] = 1;
baseptr[1] = 2;
MPI_Win_unlock(rank, win);
MPI_Barrier(MPI_COMM_WORLD);
/* Issue request-based get with testall. */
MPI_Win_lock_all(0, win);
MPI_Rget(&val1, 1, MPI_INT, 0, 0, 1, MPI_INT, win, &reqs[0]);
MPI_Rget(&val2, 1, MPI_INT, 0, 1, 1, MPI_INT, win, &reqs[1]);
do {
mpi_errno = MPI_Testall(2, reqs, &flag, stats);
} while (flag == 0);
/* Check get value. */
if (val1 != 1 || val2 != 2) {
printf("%d - Got val1 = %d, val2 = %d, expected 1, 2\n", rank, val1, val2);
fflush(stdout);
errs++;
}
/* Check return error code. */
if (mpi_errno != MPI_SUCCESS) {
printf("%d - Got return errno %d, expected MPI_SUCCESS(%d)\n",
rank, mpi_errno, MPI_SUCCESS);
fflush(stdout);
errs++;
}
MPI_Win_unlock_all(win);
MPI_Barrier(MPI_COMM_WORLD);
MPI_Win_free(&win);
MTest_Finalize(errs);
MPI_Finalize();
return errs != 0;
}
int main(int argc, char **argv) {
int i, rank, nproc;
int errors = 0, all_errors = 0;
TYPE_C *win_ptr, *res_ptr, *val_ptr;
MPI_Win win;
#if defined (GACC_TYPE_DERIVED)
MPI_Datatype derived_type;
#endif
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
win_ptr = malloc(sizeof(TYPE_C)*nproc*COUNT);
res_ptr = malloc(sizeof(TYPE_C)*nproc*COUNT);
val_ptr = malloc(sizeof(TYPE_C)*COUNT);
#if defined (GACC_TYPE_DERIVED)
MPI_Type_contiguous(1, TYPE_MPI_BASE, &derived_type);
MPI_Type_commit(&derived_type);
#endif
MPI_Win_create(win_ptr, sizeof(TYPE_C)*nproc*COUNT, sizeof(TYPE_C),
MPI_INFO_NULL, MPI_COMM_WORLD, &win);
/* Test self communication */
reset_bufs(win_ptr, res_ptr, val_ptr, 1, win);
for (i = 0; i < ITER; i++) {
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
MPI_Get_accumulate(val_ptr, COUNT, TYPE_MPI, res_ptr, COUNT, TYPE_MPI,
rank, 0, COUNT, TYPE_MPI, MPI_SUM, win);
MPI_Win_unlock(rank, win);
}
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
for (i = 0; i < COUNT; i++) {
if (win_ptr[i] != ITER) {
SQUELCH( printf("%d->%d -- SELF[%d]: expected "TYPE_FMT", got "TYPE_FMT"\n",
rank, rank, i, (TYPE_C) ITER, win_ptr[i]); );
errors++;
}
static int run_test()
{
int i, x, errs = 0, errs_total = 0;
MPI_Status stat;
int dst;
int winbuf_offset = 0;
double t0, avg_total_time = 0.0, t_total = 0.0;
double sum = 0.0;
dst = 1;
if (rank == 0) {
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, dst, MPI_MODE_NOCHECK, win);
DO_OP_LOOP(dst, SKIP);
MPI_Win_unlock(dst, win);
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, dst, MPI_MODE_NOCHECK, win);
t0 = MPI_Wtime();
DO_OP_LOOP(dst, ITER);
t_total = (MPI_Wtime() - t0) * 1000 * 1000; /*us */
t_total /= ITER;
MPI_Win_unlock(dst, win);
}
MPI_Barrier(MPI_COMM_WORLD);
if (rank == 0) {
#ifdef MTCORE
fprintf(stdout, "mtcore: iter %d %s num_op %d opsize %d nprocs %d nh %d total_time %.2lf\n",
ITER, OP_TYPE_NM[OP_TYPE], NOP, OP_SIZE, nprocs, MTCORE_NUM_H, t_total);
#else
fprintf(stdout, "orig: iter %d %s num_op %d opsize %d nprocs %d total_time %.2lf\n",
ITER, OP_TYPE_NM[OP_TYPE], NOP, OP_SIZE, nprocs, t_total);
#endif
}
exit:
return errs_total;
}
/** Lock a mutex.
*
* @param[in] hdl Mutex group that the mutex belongs to
* @param[in] mutex Desired mutex number [0..count-1]
* @param[in] proc Rank of process where the mutex lives
* @return MPI status
*/
int MPIX_Mutex_lock(MPIX_Mutex hdl, int mutex, int proc)
{
int rank, nproc, already_locked, i;
uint8_t *buf;
assert(mutex >= 0 && mutex < hdl->max_count);
MPI_Comm_rank(hdl->comm, &rank);
MPI_Comm_size(hdl->comm, &nproc);
assert(proc >= 0 && proc < nproc);
buf = malloc(nproc * sizeof(uint8_t));
assert(buf != NULL);
buf[rank] = 1;
/* Get all data from the lock_buf, except the byte belonging to
* me. Set the byte belonging to me to 1. */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->windows[mutex]);
MPI_Put(&buf[rank], 1, MPI_BYTE, proc, rank, 1, MPI_BYTE, hdl->windows[mutex]);
/* Get data to the left of rank */
if (rank > 0) {
MPI_Get(buf, rank, MPI_BYTE, proc, 0, rank, MPI_BYTE, hdl->windows[mutex]);
}
/* Get data to the right of rank */
if (rank < nproc - 1) {
MPI_Get(&buf[rank + 1], nproc - 1 - rank, MPI_BYTE, proc, rank + 1, nproc - 1 - rank,
MPI_BYTE, hdl->windows[mutex]);
}
MPI_Win_unlock(proc, hdl->windows[mutex]);
assert(buf[rank] == 1);
for (i = already_locked = 0; i < nproc; i++)
if (buf[i] && i != rank)
already_locked = 1;
/* Wait for notification */
if (already_locked) {
MPI_Status status;
debug_print("waiting for notification [proc = %d, mutex = %d]\n", proc, mutex);
MPI_Recv(NULL, 0, MPI_BYTE, MPI_ANY_SOURCE, MPIX_MUTEX_TAG + mutex, hdl->comm, &status);
}
debug_print("lock acquired [proc = %d, mutex = %d]\n", proc, mutex);
free(buf);
return MPI_SUCCESS;
}
/** Unlock a mutex.
*
* @param[in] hdl Mutex group that the mutex belongs to.
* @param[in] mutex Desired mutex number [0..count-1]
* @param[in] proc Rank of process where the mutex lives
* @return MPI status
*/
int MPIX_Mutex_unlock(MPIX_Mutex hdl, int mutex, int proc)
{
int rank, nproc, i;
uint8_t *buf;
assert(mutex >= 0 && mutex < hdl->max_count);
MPI_Comm_rank(hdl->comm, &rank);
MPI_Comm_size(hdl->comm, &nproc);
assert(proc >= 0 && proc < nproc);
buf = malloc(nproc * sizeof(uint8_t));
assert(buf != NULL);
buf[rank] = 0;
/* Get all data from the lock_buf, except the byte belonging to
* me. Set the byte belonging to me to 0. */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->windows[mutex]);
MPI_Put(&buf[rank], 1, MPI_BYTE, proc, rank, 1, MPI_BYTE, hdl->windows[mutex]);
/* Get data to the left of rank */
if (rank > 0) {
MPI_Get(buf, rank, MPI_BYTE, proc, 0, rank, MPI_BYTE, hdl->windows[mutex]);
}
/* Get data to the right of rank */
if (rank < nproc - 1) {
MPI_Get(&buf[rank + 1], nproc - 1 - rank, MPI_BYTE, proc, rank + 1, nproc - 1 - rank,
MPI_BYTE, hdl->windows[mutex]);
}
MPI_Win_unlock(proc, hdl->windows[mutex]);
assert(buf[rank] == 0);
/* Notify the next waiting process, starting to my right for fairness */
for (i = 1; i < nproc; i++) {
int p = (rank + i) % nproc;
if (buf[p] == 1) {
debug_print("notifying %d [proc = %d, mutex = %d]\n", p, proc, mutex);
MPI_Send(NULL, 0, MPI_BYTE, p, MPI_MUTEX_TAG + mutex, hdl->comm);
break;
}
}
debug_print("lock released [proc = %d, mutex = %d]\n", proc, mutex);
free(buf);
return MPI_SUCCESS;
}
void ompi_win_lock_f(MPI_Fint *lock_type, MPI_Fint *rank,
MPI_Fint *assert, MPI_Fint *win, MPI_Fint *ierr)
{
int c_ierr;
MPI_Win c_win = MPI_Win_f2c(*win);
c_ierr = MPI_Win_lock(OMPI_FINT_2_INT(*lock_type),
OMPI_FINT_2_INT(*rank),
OMPI_FINT_2_INT(*assert),
c_win);
if (NULL != ierr) *ierr = OMPI_INT_2_FINT(c_ierr);
}
/** Attempt to lock a mutex (non-blocking).
*
* @param[in] hdl Mutex group that the mutex belongs to.
* @param[in] mutex Desired mutex number [0..count-1]
* @param[in] world_proc Absolute ID of process where the mutex lives
* @return 0 on success, non-zero on failure
*/
int ARMCIX_Trylock_hdl(armcix_mutex_hdl_t hdl, int mutex, int world_proc) {
int rank, nproc, proc;
long lock_val, unlock_val, lock_out;
ARMCII_Assert(mutex >= 0);
MPI_Comm_rank(hdl->comm, &rank);
MPI_Comm_size(hdl->comm, &nproc);
/* User gives us the absolute ID. Translate to the rank in the mutex's group. */
proc = ARMCII_Translate_absolute_to_group(hdl->comm, world_proc);
ARMCII_Assert(proc >= 0);
lock_val = rank+1;
unlock_val = -1 * (rank+1);
/* mutex <- mutex + rank */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->window);
MPI_Accumulate(&lock_val, 1, MPI_LONG, proc, mutex, 1, MPI_LONG, MPI_SUM, hdl->window);
MPI_Win_unlock(proc, hdl->window);
/* read mutex value */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->window);
MPI_Get(&lock_out, 1, MPI_LONG, proc, mutex, 1, MPI_LONG, hdl->window);
MPI_Win_unlock(proc, hdl->window);
ARMCII_Assert(lock_out > 0);
ARMCII_Assert(lock_out <= nproc*(nproc+1)/2); // Must be < sum of all ranks
/* We are holding the mutex */
if (lock_out == rank+1)
return 0;
/* mutex <- mutex - rank */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->window);
MPI_Accumulate(&unlock_val, 1, MPI_LONG, proc, mutex, 1, MPI_LONG, MPI_SUM, hdl->window);
MPI_Win_unlock(proc, hdl->window);
return 1;
}
/** Lock a mutex.
*
* @param[in] hdl Mutex group that the mutex belongs to.
* @param[in] mutex Desired mutex number [0..count-1]
* @param[in] world_proc Absolute ID of process where the mutex lives
*/
void ARMCIX_Lock_hdl(armcix_mutex_hdl_t hdl, int mutex, int world_proc) {
int rank, nproc, already_locked, i, proc;
uint8_t *buf;
ARMCII_Assert(mutex >= 0 && mutex < hdl->max_count);
MPI_Comm_rank(hdl->grp.comm, &rank);
MPI_Comm_size(hdl->grp.comm, &nproc);
/* User gives us the absolute ID. Translate to the rank in the mutex's group. */
proc = ARMCII_Translate_absolute_to_group(&hdl->grp, world_proc);
ARMCII_Assert(proc >= 0);
buf = malloc(nproc*sizeof(uint8_t));
ARMCII_Assert(buf != NULL);
buf[rank] = 1;
/* Get all data from the lock_buf, except the byte belonging to
* me. Set the byte belonging to me to 1. */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->windows[mutex]);
MPI_Put(&buf[rank], 1, MPI_BYTE, proc, rank, 1, MPI_BYTE, hdl->windows[mutex]);
/* Get data to the left of rank */
if (rank > 0) {
MPI_Get(buf, rank, MPI_BYTE, proc, 0, rank, MPI_BYTE, hdl->windows[mutex]);
}
/* Get data to the right of rank */
if (rank < nproc - 1) {
MPI_Get(&buf[rank+1], nproc-1-rank, MPI_BYTE, proc, rank + 1, nproc-1-rank, MPI_BYTE, hdl->windows[mutex]);
}
MPI_Win_unlock(proc, hdl->windows[mutex]);
ARMCII_Assert(buf[rank] == 1);
for (i = already_locked = 0; i < nproc; i++)
if (buf[i] && i != rank)
already_locked = 1;
/* Wait for notification */
if (already_locked) {
MPI_Status status;
ARMCII_Dbg_print(DEBUG_CAT_MUTEX, "waiting for notification [proc = %d, mutex = %d]\n", proc, mutex);
MPI_Recv(NULL, 0, MPI_BYTE, MPI_ANY_SOURCE, ARMCI_MUTEX_TAG+mutex, hdl->grp.comm, &status);
}
ARMCII_Dbg_print(DEBUG_CAT_MUTEX, "lock acquired [proc = %d, mutex = %d]\n", proc, mutex);
free(buf);
}
/** Unlock a mutex.
*
* @param[in] hdl Mutex group that the mutex belongs to.
* @param[in] mutex Desired mutex number [0..count-1]
* @param[in] world_proc Absolute ID of process where the mutex lives
*/
void ARMCIX_Unlock_hdl(armcix_mutex_hdl_t hdl, int mutex, int world_proc) {
int rank, nproc, i, proc;
uint8_t *buf;
ARMCII_Assert(mutex >= 0 && mutex < hdl->max_count);
MPI_Comm_rank(hdl->grp.comm, &rank);
MPI_Comm_size(hdl->grp.comm, &nproc);
proc = ARMCII_Translate_absolute_to_group(&hdl->grp, world_proc);
ARMCII_Assert(proc >= 0);
buf = malloc(nproc*sizeof(uint8_t));
buf[rank] = 0;
/* Get all data from the lock_buf, except the byte belonging to
* me. Set the byte belonging to me to 0. */
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc, 0, hdl->windows[mutex]);
MPI_Put(&buf[rank], 1, MPI_BYTE, proc, rank, 1, MPI_BYTE, hdl->windows[mutex]);
/* Get data to the left of rank */
if (rank > 0) {
MPI_Get(buf, rank, MPI_BYTE, proc, 0, rank, MPI_BYTE, hdl->windows[mutex]);
}
/* Get data to the right of rank */
if (rank < nproc - 1) {
MPI_Get(&buf[rank+1], nproc-1-rank, MPI_BYTE, proc, rank + 1, nproc-1-rank, MPI_BYTE, hdl->windows[mutex]);
}
MPI_Win_unlock(proc, hdl->windows[mutex]);
ARMCII_Assert(buf[rank] == 0);
/* Notify the next waiting process, starting to my right for fairness */
for (i = 1; i < nproc; i++) {
int p = (rank + i) % nproc;
if (buf[p] == 1) {
ARMCII_Dbg_print(DEBUG_CAT_MUTEX, "notifying %d [proc = %d, mutex = %d]\n", p, proc, mutex);
MPI_Send(NULL, 0, MPI_BYTE, p, ARMCI_MUTEX_TAG+mutex, hdl->grp.comm);
break;
}
}
ARMCII_Dbg_print(DEBUG_CAT_MUTEX, "lock released [proc = %d, mutex = %d]\n", proc, mutex);
free(buf);
}
void RunPutLock(MPI_Win win, int destRank, int cnt, int sz)
{
int k, i, j, one = 1;
for (k = 0; k < MAX_RUNS; k++) {
MPI_Barrier(MPI_COMM_WORLD);
MPI_Win_lock(MPI_LOCK_SHARED, destRank, 0, win);
j = 0;
for (i = 0; i < cnt; i++) {
MPI_Put(&one, sz, MPI_INT, destRank, j, sz, MPI_INT, win);
j += sz;
}
MPI_Win_unlock(destRank, win);
}
}
void reset_vars(TYPE_C *val_ptr, TYPE_C *res_ptr, MPI_Win win) {
int i, rank, nproc;
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, 0, win);
for (i = 0; i < nproc; i++) {
val_ptr[i] = 0;
res_ptr[i] = -1;
}
MPI_Win_unlock(rank, win);
MPI_Barrier(MPI_COMM_WORLD);
}
void heartbeat_worker() {
MPI_Win win;
MPI_Win_create(NULL, 0, 1, MPI_INFO_NULL, MPI_COMM_WORLD, &win);
while (1) {
time_t current_time;
time(¤t_time);
long t = current_time;
MPI_Win_lock(MPI_LOCK_SHARE, 0, 0, &win);
MPI_Put(&t, 1, MPI_LONG, 0, rank - 1, 1, MPI_LONG, win);
MPI_Win_unlock(0, win);
sleep(1);
}
MPI_Win_free(&win);
}
void get_remote(double *buf, int I, int J)
{
int proc_owner;
int edge, size;
double t1;
proc_owner = block_owner(I, J);
edge = n%block_size;
if (edge == 0) {
edge = block_size;
}
if ((I == nblocks-1) && (J == nblocks-1)) {
size = edge*edge;
}
else if ((I == nblocks-1) || (J == nblocks-1)) {
size = edge*block_size;
}
else {
size = block_size*block_size;
}
size = size * sizeof(double);
t1 = MPI_Wtime();
#ifdef MPI2_ONESIDED
{
int target_disp = ( ((char*)(a[I+J*nblocks])) -
((char*)(ptr[proc_owner])) );
if(target_disp<0) {
printf("ERROR!: target disp is < 0, target_disp= %d\n", target_disp);
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, proc_owner, 0, win);
MPI_Get(buf, size, MPI_CHAR, proc_owner, target_disp, size,
MPI_CHAR, win);
MPI_Win_unlock(proc_owner, win);
}
#else
ARMCI_Get(a[I+J*nblocks], buf, size, proc_owner);
#endif
comm_time += MPI_Wtime() - t1;
get_cntr++;
}
