int PARMCI_Destroy_mutexes()
{
#ifdef LAPI /*fix to if cmpl handler for a pending unlock runs after destroy*/
int proc, mutex, i,factor=0;
#endif
if(num_mutexes==0)armci_die("armci_destroy_mutexes: not created",0);
if(armci_nproc == 1) return(0);
armci_msg_barrier();
#ifdef LAPI /*fix to if cmpl handler for a pending unlock runs after destroy*/
for(proc=0; proc<armci_nproc; proc++) {
for(mutex=0; mutex<glob_mutex[proc].count; mutex++) {
_dummy_server_work_ = 0.; /* must be global to fool the compiler */
while(!armci_mutex_free(mutex,proc)) {
for(i=0; i< SPINMAX *factor; i++) _dummy_server_work_ += 1.;
factor+=1;
}
}
}
#endif
num_mutexes=0;
# if defined(SERVER_LOCK)
armci_serv_mutex_close();
# endif
if(glob_mutex[armci_me].count)PARMCI_Free(glob_mutex[armci_me].token);
free(tickets);
free(glob_mutex);
free(mutex_mem_ar);
return(0);
}
int ARMCI_Free(void *ptr)
{
int rval;
static double stime, etime;
stime = TIME();
rval = PARMCI_Free(ptr);
etime = TIME();
ARMCI_Free_t += etime - stime;
return rval;
}
int ARMCI_Free(void *ptr)
{
int ret;
ret = PARMCI_Free(ptr);
return ret;
}
int ARMCI_Free(void *ptr) {
return PARMCI_Free(ptr);
}
int main(int argc, char *argv[])
{
int rank, size;
int provided;
double one = 1.0;
#if defined(__bgp__)
MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);
assert(provided==MPI_THREAD_MULTIPLE);
#else
MPI_Init_thread(&argc, &argv, MPI_THREAD_FUNNELED, &provided);
//assert(provided>MPI_THREAD_SINGLE);
#endif
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
assert( size > 1 );
PARMCI_Init_args(&argc, &argv);
int w, maxwinsize = ( argc > 1 ? atoi(argv[1]) : 1000000 );
if ( rank == 0 ) printf( "size = %d maxwinsize = %d doubles\n", size, maxwinsize );
for ( w = 1 ; w < maxwinsize ; w *= 2 )
{
double ** window;
window = (double **) PARMCI_Malloc_local( size * sizeof(double *) );
PARMCI_Malloc( (void **) window, w * sizeof(double) );
for (int i = 0; i < w; i++) window[rank][i] = 0.0;
double * buffer;
buffer = (double *) PARMCI_Malloc_local( w * sizeof(double) );
PARMCI_Barrier();
if (rank == 0)
for (int t=1; t<size; t++)
{
int bytes = w * sizeof(double);
for (int i = 0; i < w; i++) buffer[i] = (double)(t);
PARMCI_Acc( ARMCI_ACC_DBL, &one, buffer, window[t], bytes, t );
PARMCI_Fence( t );
for (int i = 0; i < w; i++) buffer[i] = 0.0;
PARMCI_Get( window[t], buffer, bytes, t );
int errors = 0;
for (int i = 0; i < w; i++)
if ( buffer[i] != (double)(t) ) errors++;
if ( errors > 0 )
for (int i = 0; i < w; i++)
printf("rank %d buffer[%d] = %lf \n", rank, i, buffer[i] );
}
PARMCI_Barrier();
if (rank != 0)
{
int errors = 0;
for (int i = 0; i < w; i++)
if ( window[rank][i] != (double)(rank) ) errors++;
if ( errors > 0 )
for (int i = 0; i < w; i++)
printf("rank %d window[%d][%d] = %lf \n", rank, rank, i, window[rank][i] );
}
PARMCI_Barrier();
if (rank == 0)
for (int t=1; t<size; t++)
{
int bytes = w * sizeof(double);
double t0, t1, t2, dt1, dt2, bw1, bw2;
for (int i = 0; i < w; i++) buffer[i] = (double)(-1);
t0 = MPI_Wtime();
PARMCI_Acc( ARMCI_ACC_DBL, &one, buffer, window[t], bytes, t );
t1 = MPI_Wtime();
PARMCI_Fence( t );
t2 = MPI_Wtime();
dt1 = t1 - t0;
dt2 = t2 - t0;
bw1 = bytes / dt1;
bw2 = bytes / dt2;
bw1 /= 1000000.0;
bw2 /= 1000000.0;
printf("PARMCI_Acc of from rank %4d to rank %4d of %9d bytes - local: %lf s (%lf MB/s) remote: %lf s (%lf MB/s) \n",
t, 0, bytes, dt1, bw1, dt2, bw2);
fflush(stdout);
}
PARMCI_Barrier();
PARMCI_Free_local( (void *) buffer );
PARMCI_Free( (void *) window[rank] );
PARMCI_Free_local( (void *) window );
}
PARMCI_Finalize();
printf("%d: all done \n", rank );
fflush(stdout);
MPI_Finalize();
return 0;
}
