int FormFunctionGradient (TAO_GA_APPLICATION gaapp, GAVec ga_X, double *f, GAVec ga_G, void *ptr)
{
AppCtx *user = (AppCtx *) ptr;
int lo, hi;
int taskId=user->me; //Which task am I running
int i;
int zero = 0;
/* reset atomicTask to nproc */
if (user->me == 0)
NGA_Put(user->atomicTask, &zero, &zero, &user->nproc, &zero);
for (i=0;i<user->natoms*user->ndim; i++)
user->grad[i] = 0.0;
*f = 0.0;
while (taskId < user->nBlocks) {
getBlock(ga_X, taskId, user);
if (user->ndim == 2)
LJFG_2D(taskId,f, user);
else
LJFG_3D(taskId,f, user);
/* Get next block */
taskId += user->nproc;
//NGA_Read_inc(user->atomicTask, &zero, 1);
}
/* gather function */
GA_Dgop(f, 1, "+");
/* gather gradient */
GA_Dgop(user->grad, user->natoms*user->ndim, "+");
NGA_Distribution(ga_G, user->me, &lo, &hi);
NGA_Put(ga_G, &lo, &hi, user->grad+lo, &hi);
GA_Sync();
return 0;
}
void test_io_dbl()
{
int n, ndim = NDIM;
double err, tt0, tt1, mbytes;
int g_a, g_b, d_a;
int i, itmp, j, req, loop;
int glo[MAXDIM],ghi[MAXDIM];
dra_size_t dlo[MAXDIM],dhi[MAXDIM];
dra_size_t ddims[MAXDIM],reqdims[MAXDIM];
dra_size_t m;
int index[MAXDIM], dims[MAXDIM];
int me, nproc, isize;
double *ptr;
double plus, minus;
int ld[MAXDIM], chunk[MAXDIM];
char filename[80];
FILE *fd;
n = SIZE;
m = ((dra_size_t)NFACTOR)*((dra_size_t)SIZE);
loop = 1;
for (i=0; i<ndim; i++) loop *= NFACTOR;
req = -1;
nproc = GA_Nnodes();
me = GA_Nodeid();
if (me == 0) {
printf("Creating temporary global arrays %d",n);
for (i=1; i<ndim; i++) {
printf(" x %d",n);
}
printf("\n");
}
if (me == 0) fflush(stdout);
GA_Sync();
for (i=0; i<ndim; i++) {
dims[i] = n;
chunk[i] = 1;
}
g_a = NGA_Create(MT_DBL, ndim, dims, "a", chunk);
if (!g_a) GA_Error("NGA_Create failed: a", 0);
g_b = NGA_Create(MT_DBL, ndim, dims, "b", chunk);
if (!g_b) GA_Error("NGA_Create failed: b", 0);
if (me == 0) printf("done\n");
if (me == 0) fflush(stdout);
/* initialize g_a, g_b with random values
... use ga_access to avoid allocating local buffers for ga_put */
GA_Sync();
NGA_Distribution(g_a, me, glo, ghi);
NGA_Access(g_a, glo, ghi, &ptr, ld);
isize = 1;
for (i=0; i<ndim; i++) isize *= (ghi[i]-glo[i]+1);
fill_random(ptr, isize);
GA_Sync();
GA_Zero(g_b);
/*.......................................................................*/
if (me == 0) {
printf("Creating Disk array %ld",m);
for (i=1; i<ndim; i++) {
printf(" x %ld",m);
}
printf("\n");
}
if (me == 0) fflush(stdout);
for (i=0; i<ndim; i++) {
ddims[i] = m;
reqdims[i] = (dra_size_t)n;
}
GA_Sync();
strcpy(filename,FNAME);
if (! (fd = fopen(filename, "w"))) {
strcpy(filename,FNAME_ALT);
if (! (fd = fopen(filename, "w"))) {
GA_Error("open failed",0);
}
}
fclose(fd);
if (NDRA_Create(MT_DBL, ndim, ddims, "A", filename, DRA_RW,
reqdims, &d_a) != 0) {
GA_Error("NDRA_Create failed(d_a): ",0);
}
if (me == 0) printf("testing write\n");
fflush(stdout);
tt1 = 0.0;
for (i=0; i<loop; i++) {
itmp=i;
for (j=0; j<ndim; j++) {
index[j] = itmp%NFACTOR;
itmp = (itmp - index[j])/NFACTOR;
}
for (j=0; j<ndim; j++) {
glo[j] = 0;
ghi[j] = SIZE - 1;
dlo[j] = ((dra_size_t)index[j])*((dra_size_t)SIZE);
dhi[j] = (((dra_size_t)index[j])+(dra_size_t)1)
* ((dra_size_t)SIZE) - (dra_size_t)1;
}
tt0 = MP_TIMER();
if (NDRA_Write_section(FALSE, g_a, glo, ghi,
d_a, dlo, dhi, &req) != 0) {
GA_Error("ndra_write_section failed:",0);
}
if (DRA_Wait(req) != 0) {
GA_Error("DRA_Wait failed(d_a): ",req);
}
tt1 += (MP_TIMER() - tt0);
}
GA_Dgop(&tt1,1,"+");
tt1 = tt1/((double)nproc);
mbytes = 1.e-6 * (double)(pow(m,ndim)*sizeof(double));
if (me == 0) {
printf("%11.2f MB time = %11.2f rate = %11.3f MB/s\n",
mbytes,tt1,mbytes/tt1);
}
if (DRA_Close(d_a) != 0) {
GA_Error("DRA_Close failed(d_a): ",d_a);
}
if (me == 0) printf("\n");
if (me == 0) printf("disk array closed\n");
if (me == 0) fflush(stdout);
/*..........................................................*/
if (me == 0) printf("\n");
if (me == 0) printf("opening disk array\n");
if (DRA_Open(filename, DRA_R, &d_a) != 0) {
GA_Error("DRA_Open failed",0);
}
if (me == 0) printf("testing read\n");
/* printf("testing read on proc %d\n",me); */
if (me == 0) fflush(stdout);
tt1 = 0.0;
for (i=0; i<loop; i++) {
itmp=i;
for (j=0; j<ndim; j++) {
index[j] = itmp%NFACTOR;
itmp = (itmp - index[j])/NFACTOR;
}
for (j=0; j<ndim; j++) {
glo[j] = 0;
ghi[j] = SIZE - 1;
dlo[j] = ((dra_size_t)index[j])*((dra_size_t)SIZE);
dhi[j] = (((dra_size_t)index[j])+(dra_size_t)1)
* ((dra_size_t)SIZE) - (dra_size_t)1;
}
tt0 = MP_TIMER();
if (NDRA_Read_section(FALSE, g_b, glo, ghi,
d_a, dlo, dhi, &req) != 0) {
GA_Error("ndra_read_section failed:",0);
}
if (DRA_Wait(req) != 0) {
GA_Error("DRA_Wait failed(d_a): ",req);
}
tt1 += (MP_TIMER() - tt0);
plus = 1.0;
minus = -1.0;
GA_Add(&plus, g_a, &minus, g_b, g_b);
err = GA_Ddot(g_b, g_b);
if (err != 0) {
if (me == 0) {
printf("BTW, we have error = %f on loop value %d\n",
err,i);
}
GA_Error(" bye",0);
}
}
GA_Dgop(&tt1,1,"+");
tt1 = tt1/((double)nproc);
if (me == 0) {
printf("%11.2f MB time = %11.2f rate = %11.3f MB/s\n",
mbytes,tt1,mbytes/tt1);
}
if (DRA_Delete(d_a) != 0) GA_Error("DRA_Delete failed",0);
/*.......................................................................*/
GA_Destroy(g_a);
GA_Destroy(g_b);
}