int main(int argc, char** argv)
{
int nprocs,myid,nprocssq;
int dims[2],chunk[2];
int i,j,k;
int stack = 100000, heap = 100000;
MPI_Init(&argc,&argv);
GA_Initialize();
MA_init(C_DBL,stack,heap);
nprocssq = GA_Nnodes();
nprocs = sqrt(nprocssq);
myid = GA_Nodeid();
dims[0] = N; dims[1] = N;
chunk[0] = N/nprocs;
chunk[1] = N/nprocs;
int g_a = NGA_Create(C_DBL,2,dims,"Array A",chunk);
int lo[2],hi[2];
NGA_Distribution(g_a,myid,lo,hi);
int ld[1] = {N/nprocs};
void *ptr;
double *local;
printf("Myid = %d, lo = [%d,%d] , hi = [%d,%d] , ld = %d \n",myid,lo[0],lo[1],hi[0],hi[1],ld[0]);
NGA_Access(g_a,lo,hi,&ptr,ld);
local = (double*) ptr;
printf("Myid = %d , local[0][0] = %f\n",*local);
GA_Sync();
GA_Destroy(g_a);
GA_Terminate();
MPI_Finalize();
return 0;
}
void FATR
ga_antisymmetrize_(Integer *g_a) {
DoublePrecision alpha = 0.5;
int i, me = GA_Nodeid();
extern void * FATR ga_malloc(Integer nelem, int type, char *name);
extern void FATR ga_free(void *ptr);
void FATR gai_subtr(int *lo, int *hi, void *a, void *b, DoublePrecision alpha,
int type, Integer nelem, int ndim);
int alo[GA_MAX_DIM], ahi[GA_MAX_DIM], lda[GA_MAX_DIM];
int blo[GA_MAX_DIM], bhi[GA_MAX_DIM], ldb[GA_MAX_DIM];
int ndim, dims[GA_MAX_DIM], type;
Integer nelem=1;
Logical have_data;
void *a_ptr, *b_ptr;
GA_Sync();
NGA_Inquire((int)(*g_a), &type, &ndim, dims);
if (dims[0] != dims[1])
GA_Error("ga_sym: can only sym square matrix", 0L);
/* Find the local distribution */
NGA_Distribution((int)(*g_a), me, alo, ahi);
have_data = ahi[0]>=0;
for(i=1; i<ndim; i++) have_data = have_data && ahi[i]>=0;
if(have_data) {
NGA_Access((int)(*g_a), alo, ahi, &a_ptr, lda);
for(i=0; i<ndim; i++) nelem *= ahi[i]-alo[i] +1;
b_ptr = (void *) ga_malloc(nelem, MT_C_DBL, "v");
for(i=2; i<ndim; i++) {bhi[i]=ahi[i]; blo[i]=alo[i]; }
/* switch rows and cols */
blo[1]=alo[0];
bhi[1]=ahi[0];
blo[0]=alo[1];
bhi[0]=ahi[1];
for (i=0; i < ndim-1; i++)
ldb[i] = bhi[i+1] - blo[i+1] + 1;
NGA_Get((int)(*g_a), blo, bhi, b_ptr, ldb);
}
GA_Sync();
if(have_data) {
gai_subtr(alo, ahi, a_ptr, b_ptr, alpha, type, nelem, ndim);
NGA_Release_update((int)(*g_a), alo, ahi);
ga_free(b_ptr);
}
GA_Sync();
}
PetscErrorCode testCreate2D()
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 1453, d2 = 1451;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal **v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym;
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[j][i] = 1.*i + d1 * j;
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Updated local portion with DAVec\n");
PetscBarrier(0);
{
double *da_ptr;
VecGetArray(vec, &da_ptr);
double *ptr;
int low[2],hi[2],ld;
NGA_Distribution(ga,GA_Nodeid(),low,hi);
NGA_Access(ga,low,hi,&ptr,&ld);
printf("[%d] ga:%p\tda:%p\tdiff:%p\n", GA_Nodeid(), ptr, da_ptr, (ptr-da_ptr) );
NGA_Release_update(ga,low,hi);
}
int lo[2],ld;
double val;
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = j;
lo[1] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j - val) > .1 )
printf(".");
// printf("[%d] (%3.0f,%3.0f)\n", GA_Nodeid(), 1.*i + d1*j, val);
}
}
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
int main(int argc, char **argv)
{
int size_dst = 15;
int g_a = 0;
int I_NEG_ONE = -1;
long L_NEG_ONE = -1;
long long LL_NEG_ONE = -1;
int FIVE = 5;
int TEN = 10;
int lo;
int hi;
int *ptr;
int i;
MP_INIT(argc,argv);
GA_INIT(argc,argv);
for (i=0; i<3; ++i) {
if (0 == i) {
g_a = NGA_Create(C_INT, 1, &size_dst, "dst", NULL);
GA_Fill(g_a, &I_NEG_ONE);
} else if (1 == i) {
g_a = NGA_Create(C_LONG, 1, &size_dst, "dst", NULL);
GA_Fill(g_a, &L_NEG_ONE);
} else if (2 == i) {
g_a = NGA_Create(C_LONGLONG, 1, &size_dst, "dst", NULL);
GA_Fill(g_a, &LL_NEG_ONE);
}
GA_Sync();
GA_Print(g_a);
NGA_Print_patch(g_a, &FIVE, &TEN, 0);
NGA_Print_patch(g_a, &FIVE, &TEN, 1);
NGA_Distribution(g_a, GA_Nodeid(), &lo, &hi);
NGA_Access(g_a, &lo, &hi, &ptr, NULL);
printf("[%d] (%d)=%d\n", GA_Nodeid(), lo, *ptr);
NGA_Release(g_a, &lo, &hi);
}
GA_Terminate();
MP_FINALIZE();
exit(EXIT_SUCCESS);
}
PetscErrorCode vizGA2DA()
{
PetscErrorCode ierr;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
int d1 = 40, d2 = 50;
DA da;
Vec vec;
const PetscInt *lx, *ly, *lz;
PetscInt m,n,p;
DALocalInfo info;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr);
ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr);
/**/
ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
for (int i = 0; i < m; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]);
}
for (int i = 0; i < n; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]);
}
/**/
int ga = GA_Create_handle();
int ndim = 2;
int dims[2] = {d2,d1};
GA_Set_data(ga,2,dims,MT_DBL);
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < n; i++ )
{
map[i] = ly[i-1] + map[i-1];
}
map[n] = 0;
for( int i = n+1; i < m+n; i++ )
{
map[i] = lx[i-n-1] + map[i-1];
}
/* correct ordering, but nodeid's dont line up with mpi rank for petsc's da
* DA: +---+---+ GA: +---+---+
* +-2-+-3-+ +-1-+-3-+
* +---+---+ +---+---+
* +-0-+-1-+ +-0-+-2-+
* +---+---+ +---+---+
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < m; i++ )
{
map[i] = lx[i] + map[i-1];
}
map[m] = 0;
for( int i = m+1; i < m+n; i++ )
{
map[i] = ly[i-m] + map[i-1];
}
*/
int block[2] = {n,m};
GA_Set_irreg_distr(ga,map,block);
ierr = GA_Allocate( ga );
if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr);
if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr);
if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1);
GA_Print_distribution(ga);
int lo[2], hi[2];
NGA_Distribution(ga,rank,lo,hi);
if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 ||
lo[0] != info.ys || hi[0] != info.ys+info.ym-1 )
{
PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n",
rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1);
}
PetscBarrier(0);
PetscSynchronizedFlush(PETSC_COMM_WORLD);
AO ao;
DAGetAO(da,&ao);
if( rank == 0 )
{
int *idx, len = d1*d2;
PetscReal *val;
PetscMalloc(sizeof(PetscReal)*len, &val);
PetscMalloc(sizeof(int)*len, &idx);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
idx[i + d1*j] = i + d1*j;
val[i + d1*j] = i + d1*j;
}
}
AOApplicationToPetsc(ao,len,idx);
VecSetValues(vec,len,idx,val,INSERT_VALUES);
int a[2], b[2],ld[1]={0};
double c = 0;
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
a[0] = j;
a[1] = i;
// printf("%5.0f ",c);
NGA_Put(ga,a,a,&c,ld);
c++;
}
}
}
// GA_Print(ga);
VecAssemblyBegin(vec);
VecAssemblyEnd(vec);
int ld;
double *ptr;
NGA_Access(ga,lo,hi,&ptr,&ld);
PetscReal **d;
int c=0;
ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr);
for (int j = info.ys; j < info.ys+info.ym; ++j)
{
for (int i = info.xs; i < info.xs+info.xm; ++i)
{
if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] )
GA_Error("DA array is not equal to GA array",1);
// printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]);
c++;
}
}
ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr);
c=0;
PetscReal *v;
int start, end;
VecGetOwnershipRange(vec, &start, &end);
VecGetArray( vec, &v );
for( int i = start; i < end; i++)
{
// printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") );
}
VecRestoreArray( vec, &v );
NGA_Release_update(ga,lo,hi);
Vec gada;
VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada);
VecView(gada,PETSC_VIEWER_STDOUT_SELF);
GA_Destroy(ga);
ierr = VecDestroy(vec); CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void do_work()
{
int ZERO=0; /* useful constants */
int g_a, g_b;
int n=N, ndim=2,type=MT_F_DBL,dims[2]={N,N},coord[2];
int me=GA_Nodeid(), nproc=GA_Nnodes();
int row, i, j;
int lo[2], hi[2];
/* Note: on all current platforms DoublePrecision = double */
DoublePrecision buf[N], *max_row=NULL;
MPI_Comm WORLD_COMM;
MPI_Comm ROW_COMM;
int ilo,ihi, jlo,jhi, ld, prow, pcol;
int root=0, grp_me=-1;
WORLD_COMM = GA_MPI_Comm_pgroup_default();
if(me==0)printf("Creating matrix A\n");
dims[0]=n; dims[1]=n;
g_a = NGA_Create(type, ndim, dims, "A", NULL);
if(!g_a) GA_Error("create failed: A",n);
if(me==0)printf("OK\n");
if(me==0)printf("Creating matrix B\n");
dims[0]=n;
g_b = NGA_Create(type, 1, dims, "B", NULL);
if(!g_b) GA_Error("create failed: B",n);
if(me==0)printf("OK\n");
GA_Zero(g_a); /* zero the matrix */
if(me==0)printf("Initializing matrix A\n");
/* fill in matrix A with values: A(i,j) = (i+j) */
for(row=me; row<n; row+= nproc){
/**
* simple load balancing:
* each process works on a different row in MIMD style
*/
for(i=0; i<n; i++) buf[i]=(DoublePrecision)(i+row+1);
lo[0]=hi[0]=row;
lo[1]=ZERO; hi[1]=n-1;
NGA_Put(g_a, lo, hi, buf, &n);
}
/* GA_print(&g_a);*/
NGA_Distribution(g_a, me, lo, hi);
ilo=lo[0]; ihi=hi[0];
jlo=lo[1]; jhi=hi[1];
GA_Sync();
if(ihi-ilo+1 >0){
max_row=(DoublePrecision*)malloc(sizeof(DoublePrecision)*(ihi-ilo+1));
if (!max_row) GA_Error("malloc 3 failed",(ihi-ilo+1));
for (i=0; i<(ihi-ilo+1); i++) {
max_row[i] = 0.0;
}
}
NGA_Proc_topology(g_a, me, coord); /* block coordinates */
prow = coord[0];
pcol = coord[1];
if(me==0)printf("Splitting comm according to distribution of A\n");
/* GA on SP1 requires synchronization before & after message-passing !!*/
GA_Sync();
if(me==0)printf("Computing max row elements\n");
/* create communicator for processes that 'own' A[:,jlo:jhi] */
MPI_Barrier(WORLD_COMM);
if(pcol < 0 || prow <0)
MPI_Comm_split(WORLD_COMM,MPI_UNDEFINED,MPI_UNDEFINED, &ROW_COMM);
else
MPI_Comm_split(WORLD_COMM, (int)pcol, (int)prow, &ROW_COMM);
if(ROW_COMM != MPI_COMM_NULL){
double *ptr;
MPI_Comm_rank(ROW_COMM, &grp_me);
/* each process computes max elements in the block it 'owns' */
lo[0]=ilo; hi[0]=ihi;
lo[1]=jlo; hi[1]=jhi;
NGA_Access(g_a, lo, hi, &ptr, &ld);
for(i=0; i<ihi-ilo+1; i++){
for(j=0; j<jhi-jlo+1; j++)
if(max_row[i] < ptr[i*ld + j]){
max_row[i] = ptr[i*ld + j];
}
}
MPI_Reduce(max_row, buf, ihi-ilo+1, MPI_DOUBLE, MPI_MAX,
root, ROW_COMM);
}else fprintf(stderr,"process %d not participating\n",me);
GA_Sync();
/* processes with rank=root in ROW_COMM put results into g_b */
ld = 1;
if(grp_me == root) {
lo[0]=ilo; hi[0]=ihi;
NGA_Put(g_b, lo, hi, buf, &ld);
}
GA_Sync();
if(me==0)printf("Checking the result\n");
if(me==0){
lo[0]=ZERO; hi[0]=n-1;
NGA_Get(g_b, lo, hi, buf, &n);
for(i=0; i< n; i++)if(buf[i] != (double)n+i){
fprintf(stderr,"error:%d max=%f should be:%d\n",i,buf[i],n+i);
GA_Error("terminating...",1);
}
}
if(me==0)printf("OK\n");
GA_Destroy(g_a);
GA_Destroy(g_b);
}
int main( int argc, char **argv ) {
int g_a, g_b, i, j, size, size_me;
int icnt, idx, jdx, ld;
int n=N, type=MT_C_INT, one;
int *values, *ptr;
int **indices;
int dims[2]={N,N};
int lo[2], hi[2];
int heap=3000000, stack=2000000;
int me, nproc;
int datatype, elements;
double *prealloc_mem;
MP_INIT(argc,argv);
#if 1
GA_INIT(argc,argv); /* initialize GA */
me=GA_Nodeid();
nproc=GA_Nnodes();
if(me==0) {
if(GA_Uses_fapi())GA_Error("Program runs with C array API only",1);
printf("\nUsing %ld processes\n",(long)nproc);
fflush(stdout);
}
heap /= nproc;
stack /= nproc;
if(! MA_init(MT_F_DBL, stack, heap))
GA_Error("MA_init failed",stack+heap); /* initialize memory allocator*/
/* Create a regular matrix. */
if(me==0)printf("\nCreating matrix A of size %d x %d\n",N,N);
g_a = NGA_Create(type, 2, dims, "A", NULL);
if(!g_a) GA_Error("create failed: A",n);
/* Fill matrix using scatter routines */
size = N*N;
if (size%nproc == 0) {
size_me = size/nproc;
} else {
i = size - size%nproc;
size_me = i/nproc;
if (me < size%nproc) size_me++;
}
/* Check that sizes are all okay */
i = size_me;
GA_Igop(&i,1,"+");
if (i != size) {
GA_Error("Sizes don't add up correctly: ",i);
} else if (me==0) {
printf("\nSizes add up correctly\n");
}
/* Allocate index and value arrays */
indices = (int**)malloc(size_me*sizeof(int*));
values = (int*)malloc(size_me*sizeof(int));
icnt = me;
for (i=0; i<size_me; i++) {
values[i] = icnt;
idx = icnt%N;
jdx = (icnt-idx)/N;
if (idx >= N || idx < 0) {
printf("p[%d] Bogus index i: %d\n",me,idx);
}
if (jdx >= N || jdx < 0) {
printf("p[%d] Bogus index j: %d\n",me,jdx);
}
indices[i] = (int*)malloc(2*sizeof(int));
(indices[i])[0] = idx;
(indices[i])[1] = jdx;
icnt += nproc;
}
/* Scatter values into g_a */
NGA_Scatter(g_a, values, indices, size_me);
GA_Sync();
/* Check to see if contents of g_a are correct */
NGA_Distribution( g_a, me, lo, hi );
NGA_Access(g_a, lo, hi, &ptr, &ld);
for (i=lo[0]; i<hi[0]; i++) {
idx = i-lo[0];
for (j=lo[1]; j<hi[1]; j++) {
jdx = j-lo[1];
if (ptr[idx*ld+jdx] != j*N+i) {
printf("p[%d] (Scatter) expected: %d actual: %d\n",me,j*N+i,ptr[idx*ld+jdx]);
}
}
}
if (me==0) printf("\nCompleted test of NGA_Scatter\n");
for (i=0; i<size_me; i++) {
values[i] = 0;
}
GA_Sync();
NGA_Gather(g_a, values, indices, size_me);
icnt = me;
for (i=0; i<size_me; i++) {
if (icnt != values[i]) {
printf("p[%d] (Gather) expected: %d actual: %d\n",me,icnt,values[i]);
}
icnt += nproc;
}
if (me==0) printf("\nCompleted test of NGA_Gather\n");
GA_Sync();
/* Scatter-accumulate values back into GA*/
one = 1;
NGA_Scatter_acc(g_a, values, indices, size_me, &one);
GA_Sync();
/* Check to see if contents of g_a are correct */
for (i=lo[0]; i<hi[0]; i++) {
idx = i-lo[0];
for (j=lo[1]; j<hi[1]; j++) {
jdx = j-lo[1];
if (ptr[idx*ld+jdx] != 2*(j*N+i)) {
printf("p[%d] (Scatter_acc) expected: %d actual: %d\n",me,2*(j*N+i),ptr[idx*ld+jdx]);
}
}
}
if (me==0) printf("\nCompleted test of NGA_Scatter_acc\n");
NGA_Release(g_a, lo, hi);
/* Test fixed buffer size */
NGA_Alloc_gatscat_buf(size_me);
/* Scatter-accumulate values back into GA*/
GA_Sync();
NGA_Scatter_acc(g_a, values, indices, size_me, &one);
GA_Sync();
/* Check to see if contents of g_a are correct */
for (i=lo[0]; i<hi[0]; i++) {
idx = i-lo[0];
for (j=lo[1]; j<hi[1]; j++) {
jdx = j-lo[1];
if (ptr[idx*ld+jdx] != 3*(j*N+i)) {
printf("p[%d] (Scatter_acc) expected: %d actual: %d\n",me,3*(j*N+i),ptr[idx*ld+jdx]);
}
}
}
if (me==0) printf("\nCompleted test of NGA_Scatter_acc using fixed buffers\n");
NGA_Release(g_a, lo, hi);
NGA_Free_gatscat_buf();
GA_Destroy(g_a);
if(me==0)printf("\nSuccess\n");
GA_Terminate();
#endif
MP_FINALIZE();
return 0;
}
void test_io_dbl()
{
int n, ndim = NDIM;
double err, tt0, tt1, mbytes;
int g_a, g_b, d_a;
int i, itmp, j, req, loop;
int glo[MAXDIM],ghi[MAXDIM];
dra_size_t dlo[MAXDIM],dhi[MAXDIM];
dra_size_t ddims[MAXDIM],reqdims[MAXDIM];
dra_size_t m;
int index[MAXDIM], dims[MAXDIM];
int me, nproc, isize;
double *ptr;
double plus, minus;
int ld[MAXDIM], chunk[MAXDIM];
char filename[80];
FILE *fd;
n = SIZE;
m = ((dra_size_t)NFACTOR)*((dra_size_t)SIZE);
loop = 1;
for (i=0; i<ndim; i++) loop *= NFACTOR;
req = -1;
nproc = GA_Nnodes();
me = GA_Nodeid();
if (me == 0) {
printf("Creating temporary global arrays %d",n);
for (i=1; i<ndim; i++) {
printf(" x %d",n);
}
printf("\n");
}
if (me == 0) fflush(stdout);
GA_Sync();
for (i=0; i<ndim; i++) {
dims[i] = n;
chunk[i] = 1;
}
g_a = NGA_Create(MT_DBL, ndim, dims, "a", chunk);
if (!g_a) GA_Error("NGA_Create failed: a", 0);
g_b = NGA_Create(MT_DBL, ndim, dims, "b", chunk);
if (!g_b) GA_Error("NGA_Create failed: b", 0);
if (me == 0) printf("done\n");
if (me == 0) fflush(stdout);
/* initialize g_a, g_b with random values
... use ga_access to avoid allocating local buffers for ga_put */
GA_Sync();
NGA_Distribution(g_a, me, glo, ghi);
NGA_Access(g_a, glo, ghi, &ptr, ld);
isize = 1;
for (i=0; i<ndim; i++) isize *= (ghi[i]-glo[i]+1);
fill_random(ptr, isize);
GA_Sync();
GA_Zero(g_b);
/*.......................................................................*/
if (me == 0) {
printf("Creating Disk array %ld",m);
for (i=1; i<ndim; i++) {
printf(" x %ld",m);
}
printf("\n");
}
if (me == 0) fflush(stdout);
for (i=0; i<ndim; i++) {
ddims[i] = m;
reqdims[i] = (dra_size_t)n;
}
GA_Sync();
strcpy(filename,FNAME);
if (! (fd = fopen(filename, "w"))) {
strcpy(filename,FNAME_ALT);
if (! (fd = fopen(filename, "w"))) {
GA_Error("open failed",0);
}
}
fclose(fd);
if (NDRA_Create(MT_DBL, ndim, ddims, "A", filename, DRA_RW,
reqdims, &d_a) != 0) {
GA_Error("NDRA_Create failed(d_a): ",0);
}
if (me == 0) printf("testing write\n");
fflush(stdout);
tt1 = 0.0;
for (i=0; i<loop; i++) {
itmp=i;
for (j=0; j<ndim; j++) {
index[j] = itmp%NFACTOR;
itmp = (itmp - index[j])/NFACTOR;
}
for (j=0; j<ndim; j++) {
glo[j] = 0;
ghi[j] = SIZE - 1;
dlo[j] = ((dra_size_t)index[j])*((dra_size_t)SIZE);
dhi[j] = (((dra_size_t)index[j])+(dra_size_t)1)
* ((dra_size_t)SIZE) - (dra_size_t)1;
}
tt0 = MP_TIMER();
if (NDRA_Write_section(FALSE, g_a, glo, ghi,
d_a, dlo, dhi, &req) != 0) {
GA_Error("ndra_write_section failed:",0);
}
if (DRA_Wait(req) != 0) {
GA_Error("DRA_Wait failed(d_a): ",req);
}
tt1 += (MP_TIMER() - tt0);
}
GA_Dgop(&tt1,1,"+");
tt1 = tt1/((double)nproc);
mbytes = 1.e-6 * (double)(pow(m,ndim)*sizeof(double));
if (me == 0) {
printf("%11.2f MB time = %11.2f rate = %11.3f MB/s\n",
mbytes,tt1,mbytes/tt1);
}
if (DRA_Close(d_a) != 0) {
GA_Error("DRA_Close failed(d_a): ",d_a);
}
if (me == 0) printf("\n");
if (me == 0) printf("disk array closed\n");
if (me == 0) fflush(stdout);
/*..........................................................*/
if (me == 0) printf("\n");
if (me == 0) printf("opening disk array\n");
if (DRA_Open(filename, DRA_R, &d_a) != 0) {
GA_Error("DRA_Open failed",0);
}
if (me == 0) printf("testing read\n");
/* printf("testing read on proc %d\n",me); */
if (me == 0) fflush(stdout);
tt1 = 0.0;
for (i=0; i<loop; i++) {
itmp=i;
for (j=0; j<ndim; j++) {
index[j] = itmp%NFACTOR;
itmp = (itmp - index[j])/NFACTOR;
}
for (j=0; j<ndim; j++) {
glo[j] = 0;
ghi[j] = SIZE - 1;
dlo[j] = ((dra_size_t)index[j])*((dra_size_t)SIZE);
dhi[j] = (((dra_size_t)index[j])+(dra_size_t)1)
* ((dra_size_t)SIZE) - (dra_size_t)1;
}
tt0 = MP_TIMER();
if (NDRA_Read_section(FALSE, g_b, glo, ghi,
d_a, dlo, dhi, &req) != 0) {
GA_Error("ndra_read_section failed:",0);
}
if (DRA_Wait(req) != 0) {
GA_Error("DRA_Wait failed(d_a): ",req);
}
tt1 += (MP_TIMER() - tt0);
plus = 1.0;
minus = -1.0;
GA_Add(&plus, g_a, &minus, g_b, g_b);
err = GA_Ddot(g_b, g_b);
if (err != 0) {
if (me == 0) {
printf("BTW, we have error = %f on loop value %d\n",
err,i);
}
GA_Error(" bye",0);
}
}
GA_Dgop(&tt1,1,"+");
tt1 = tt1/((double)nproc);
if (me == 0) {
printf("%11.2f MB time = %11.2f rate = %11.3f MB/s\n",
mbytes,tt1,mbytes/tt1);
}
if (DRA_Delete(d_a) != 0) GA_Error("DRA_Delete failed",0);
/*.......................................................................*/
GA_Destroy(g_a);
GA_Destroy(g_b);
}
int main(int argc, char **argv) {
int nmax, nprocs, me, me_plus;
int g_a_data, g_a_i, g_a_j, isize;
int gt_a_data, gt_a_i, gt_a_j;
int g_b, g_c;
int i, j, jj, k, one, jcnt;
int chunk, kp1, ld;
int *p_i, *p_j;
double *p_data, *p_b, *p_c;
double t_beg, t_beg2, t_ga_tot, t_get, t_mult, t_cnstrct, t_mpi_in, t_ga_in;
double t_hypre_strct, t_ga_trans, t_gp_get;
double t_get_blk_csr, t_trans_blk_csr, t_trans_blk, t_create_csr_ga, t_beg3;
double t_gp_tget, t_gp_malloc, t_gp_assign, t_beg4;
double prdot, dotga, dothypre, tempc;
double prtot, gatot, hypretot, gatot2, hypretot2;
double prdot2, prtot2;
int status;
int idim, jdim, kdim, idum, memsize;
int lsize, ntot;
int heap=200000, fudge=100, stack=200000, ma_heap;
double *cbuf, *vector;
int pdi, pdj, pdk, ip, jp, kp, ncells;
int lo[3],hi[3];
int blo[3], bhi[3];
int ld_a, ld_b, ld_c, ld_i, ld_j, irows, ioff, joff, total_procs;
int iproc, iblock, btot;
double *amat, *bvec;
int *ivec, *jvec;
int *proclist, *proc_inv, *icnt;
int *voffset, *offset, *mapc;
int iloop, lo_bl, hi_bl;
char *buf, **buf_ptr;
int *iparams, *jval, *ival;
double *rval, *rvalt;
int imin, imax, jmin, jmax, irow, icol, nnz;
int nrows, kmin, kmax, lmin, lmax, jdx;
int LOOPNUM = 100;
void **blk_ptr;
void *blk;
int blk_size, tsize, zero;
int *iblk, *jblk, *blkidx;
int *tblk_ptr;
int *ivalt, *jvalt, *iparamst;
int *iblk_t, *jblk_t, *blkidx_t;
/*
Hypre declarations
*/
int ierr;
#if USE_HYPRE
HYPRE_StructGrid grid;
HYPRE_StructStencil stencil;
HYPRE_StructMatrix matrix;
HYPRE_StructVector vec_x, vec_y;
int i4, j4, ndim, nelems, offsets[7][3];
int stencil_indices[7], hlo[3], hhi[3];
double weights[7];
double *values;
double alpha, beta;
int *rows, *cols;
#endif
/*
*** Intitialize a message passing library
*/
zero = 0;
one = 1;
ierr = MPI_Init(&argc, &argv);
/*
*** Initialize GA
There are 2 choices: ga_initialize or ga_initialize_ltd.
In the first case, there is no explicit limit on memory usage.
In the second, user can set limit (per processor) in bytes.
*/
t_beg = GA_Wtime();
NGA_Initialize();
GP_Initialize();
t_ga_in = GA_Wtime() - t_beg;
NGA_Dgop(&t_ga_in,one,"+");
t_ga_tot = 0.0;
t_ga_trans = 0.0;
t_get_blk_csr = 0.0;
t_create_csr_ga = 0.0;
t_trans_blk_csr = 0.0;
t_trans_blk = 0.0;
t_gp_get = 0.0;
t_gp_malloc = 0.0;
t_gp_assign = 0.0;
t_mult = 0.0;
t_get = 0.0;
t_gp_tget = 0.0;
t_hypre_strct = 0.0;
prtot = 0.0;
prtot2 = 0.0;
gatot = 0.0;
hypretot = 0.0;
me = NGA_Nodeid();
me_plus = me + 1;
nprocs = NGA_Nnodes();
if (me == 0) {
printf("Time to initialize GA: %12.4f\n",
t_ga_in/((double)nprocs));
}
/*
we can also use GA_set_memory_limit BEFORE first ga_create call
*/
ma_heap = heap + fudge;
/* call GA_set_memory_limit(util_mdtob(ma_heap)) */
if (me == 0) {
printf("\nNumber of cores used: %d\n\nGA initialized\n\n",nprocs);
}
/*
*** Initialize the MA package
MA must be initialized before any global array is allocated
*/
if (!MA_init(MT_DBL, stack, ma_heap)) NGA_Error("ma_init failed",-1);
/*
create a sparse LMAX x LMAX matrix and two vectors of length
LMAX. The matrix is stored in compressed row format.
One of the vectors is filled with random data and the other
is filled with zeros.
*/
idim = IMAX;
jdim = JMAX;
kdim = KMAX;
ntot = idim*jdim*kdim;
if (me == 0) {
printf("\nDimension of matrix: %d\n\n",ntot);
}
t_beg = GA_Wtime();
grid_factor(nprocs,idim,jdim,kdim,&pdi,&pdj,&pdk);
if (me == 0) {
printf("\nProcessor grid configuration\n");
printf(" PDX: %d\n",pdi);
printf(" PDY: %d\n",pdj);
printf(" PDZ: %d\n\n",pdk);
printf(" Number of Loops: %d\n",LOOPNUM);
}
create_laplace_mat(idim,jdim,kdim,pdi,pdj,pdk,&g_a_data,&g_a_j,&g_a_i,&mapc);
t_cnstrct = GA_Wtime() - t_beg;
g_b = NGA_Create_handle();
NGA_Set_data(g_b,one,&ntot,MT_DBL);
NGA_Set_irreg_distr(g_b,mapc,&nprocs);
status = NGA_Allocate(g_b);
/*
fill g_b with random values
*/
NGA_Distribution(g_b,me,blo,bhi);
NGA_Access(g_b,blo,bhi,&p_b,&ld);
ld = bhi[0]-blo[0]+1;
btot = ld;
vector = (double*)malloc(ld*sizeof(double));
for (i=0; i<ld; i++) {
idum = 0;
p_b[i] = ran3(&idum);
vector[i] = p_b[i];
}
NGA_Release(g_b,blo,bhi);
NGA_Sync();
g_c = NGA_Create_handle();
NGA_Set_data(g_c,one,&ntot,MT_DBL);
NGA_Set_irreg_distr(g_c,mapc,&nprocs);
status = NGA_Allocate(g_c);
NGA_Zero(g_c);
#if USE_HYPRE
/*
Assemble HYPRE grid and use that to create matrix. Start by creating
grid partition
*/
ndim = 3;
i = me;
ip = i%pdi;
i = (i-ip)/pdi;
jp = i%pdj;
kp = (i-jp)/pdj;
lo[0] = (int)(((double)idim)*((double)ip)/((double)pdi));
if (ip < pdi-1) {
hi[0] = (int)(((double)idim)*((double)(ip+1))/((double)pdi)) - 1;
} else {
hi[0] = idim - 1;
}
lo[1] = (int)(((double)jdim)*((double)jp)/((double)pdj));
if (jp < pdj-1) {
hi[1] = (int)(((double)jdim)*((double)(jp+1))/((double)pdj)) - 1;
} else {
hi[1] = jdim - 1;
}
lo[2] = (int)(((double)kdim)*((double)kp)/((double)pdk));
if (kp < pdk-1) {
hi[2] = (int)(((double)kdim)*((double)(kp+1))/((double)pdk)) - 1;
} else {
hi[2] = kdim - 1;
}
/*
Create grid
*/
hlo[0] = lo[0];
hlo[1] = lo[1];
hlo[2] = lo[2];
hhi[0] = hi[0];
hhi[1] = hi[1];
hhi[2] = hi[2];
ierr = HYPRE_StructGridCreate(MPI_COMM_WORLD, ndim, &grid);
ierr = HYPRE_StructGridSetExtents(grid, hlo, hhi);
ierr = HYPRE_StructGridAssemble(grid);
/*
Create stencil
*/
offsets[0][0] = 0;
offsets[0][1] = 0;
offsets[0][2] = 0;
offsets[1][0] = 1;
offsets[1][1] = 0;
offsets[1][2] = 0;
offsets[2][0] = 0;
offsets[2][1] = 1;
offsets[2][2] = 0;
offsets[3][0] = 0;
offsets[3][1] = 0;
offsets[3][2] = 1;
offsets[4][0] = -1;
offsets[4][1] = 0;
offsets[4][2] = 0;
offsets[5][0] = 0;
offsets[5][1] = -1;
offsets[5][2] = 0;
offsets[6][0] = 0;
offsets[6][1] = 0;
offsets[6][2] = -1;
nelems = 7;
ierr = HYPRE_StructStencilCreate(ndim, nelems, &stencil);
for (i=0; i<nelems; i++) {
ierr = HYPRE_StructStencilSetElement(stencil, i, offsets[i]);
}
ncells = (hi[0]-lo[0]+1)*(hi[1]-lo[1]+1)*(hi[2]-lo[2]+1);
jcnt = 7*ncells;
values = (double*)malloc(jcnt*sizeof(double));
jcnt = 0;
weights[0] = 6.0;
weights[1] = -1.0;
weights[2] = -1.0;
weights[3] = -1.0;
weights[4] = -1.0;
weights[5] = -1.0;
weights[6] = -1.0;
for (i=0; i<ncells; i++) {
for (j=0; j<7; j++) {
values[jcnt] = weights[j];
jcnt++;
}
}
ierr = HYPRE_StructMatrixCreate(MPI_COMM_WORLD, grid, stencil, &matrix);
ierr = HYPRE_StructMatrixInitialize(matrix);
for (i=0; i<7; i++) {
stencil_indices[i] = i;
}
ierr = HYPRE_StructMatrixSetBoxValues(matrix, hlo, hhi, 7, stencil_indices, values);
free(values);
/*
Check all six sides of current box to see if any are boundaries.
Set values to zero if they are.
*/
if (hi[0] == idim-1) {
ncells = (hi[1]-lo[1]+1)*(hi[2]-lo[2]+1);
hlo[0] = idim-1;
hhi[0] = idim-1;
hlo[1] = lo[1];
hhi[1] = hi[1];
hlo[2] = lo[2];
hhi[2] = hi[2];
values = (double*)malloc(ncells*sizeof(double));
for (i=0; i<ncells; i++) values[i] = 0.0;
i4 = 1;
j4 = 1;
ierr = HYPRE_StructMatrixSetBoxValues(matrix, hlo, hhi, i4, &j4, values);
free(values);
}
if (hi[1] == jdim-1) {
ncells = (hi[0]-lo[0]+1)*(hi[2]-lo[2]+1);
hlo[0] = lo[0];
hhi[0] = hi[0];
hlo[1] = jdim-1;
hhi[1] = jdim-1;
hlo[2] = lo[2];
hhi[2] = hi[2];
values = (double*)malloc(ncells*sizeof(double));
for (i=0; i<ncells; i++) values[i] = 0.0;
i4 = 1;
j4 = 2;
ierr = HYPRE_StructMatrixSetBoxValues(matrix, hlo, hhi, i4, &j4, values);
free(values);
}
if (hi[2] == kdim-1) {
ncells = (hi[0]-lo[0]+1)*(hi[1]-lo[1]+1);
hlo[0] = lo[0];
hhi[0] = hi[0];
hlo[1] = lo[1];
hhi[1] = hi[1];
hlo[2] = kdim-1;
hhi[2] = kdim-1;
values = (double*)malloc(ncells*sizeof(double));
for (i=0; i<ncells; i++) values[i] = 0.0;
i4 = 1;
j4 = 3;
ierr = HYPRE_StructMatrixSetBoxValues(matrix, hlo, hhi, i4, &j4, values);
free(values);
}
if (lo[0] == 0) {
ncells = (hi[1]-lo[1]+1)*(hi[2]-lo[2]+1);
hlo[0] = 0;
hhi[0] = 0;
hlo[1] = lo[1];
hhi[1] = hi[1];
hlo[2] = lo[2];
hhi[2] = hi[2];
values = (double*)malloc(ncells*sizeof(double));
for (i=0; i<ncells; i++) values[i] = 0.0;
i4 = 1;
j4 = 4;
ierr = HYPRE_StructMatrixSetBoxValues(matrix, hlo, hhi, i4, &j4, values);
free(values);
}
if (lo[1] == 0) {
ncells = (hi[0]-lo[0]+1)*(hi[2]-lo[2]+1);
hlo[0] = lo[0];
hhi[0] = hi[0];
hlo[1] = 0;
hhi[1] = 0;
hlo[2] = lo[2];
hhi[2] = hi[2];
values = (double*)malloc(ncells*sizeof(double));
for (i=0; i<ncells; i++) values[i] = 0.0;
i4 = 1;
j4 = 5;
ierr = HYPRE_StructMatrixSetBoxValues(matrix, hlo, hhi, i4, &j4, values);
free(values);
}
if (lo[2] == 1) {
ncells = (hi[1]-lo[1]+1)*(hi[2]-lo[2]+1);
hlo[0] = lo[0];
hhi[0] = hi[0];
hlo[1] = lo[1];
hhi[1] = hi[1];
hlo[2] = 0;
hhi[2] = 0;
values = (double*)malloc(ncells*sizeof(double));
for (i=0; i<ncells; i++) values[i] = 0.0;
i4 = 1;
j4 = 6;
ierr = HYPRE_StructMatrixSetBoxValues(matrix, hlo, hhi, i4, &j4, values);
free(values);
}
ierr = HYPRE_StructMatrixAssemble(matrix);
/*
Create vectors for matrix-vector multiply
*/
ierr = HYPRE_StructVectorCreate(MPI_COMM_WORLD, grid, &vec_x);
ierr = HYPRE_StructVectorInitialize(vec_x);
hlo[0] = lo[0];
hlo[1] = lo[1];
hlo[2] = lo[2];
hhi[0] = hi[0];
hhi[1] = hi[1];
hhi[2] = hi[2];
ierr = HYPRE_StructVectorSetBoxValues(vec_x, hlo, hhi, vector);
ierr = HYPRE_StructVectorAssemble(vec_x);
NGA_Distribution(g_a_i,me,blo,bhi);
if (bhi[1] > ntot-1) {
bhi[1] = ntot-1;
}
btot = (hi[0]-lo[0]+1)*(hi[1]-lo[1]+1)*(hi[2]-lo[2]+1);
for (i=0; i<btot; i++) vector[i] = 0.0;
hlo[0] = lo[0];
hlo[1] = lo[1];
hlo[2] = lo[2];
hhi[0] = hi[0];
hhi[1] = hi[1];
hhi[2] = hi[2];
ierr = HYPRE_StructVectorGetBoxValues(vec_x, hlo, hhi, vector);
for (i=0; i<btot; i++) vector[i] = 0.0;
ierr = HYPRE_StructVectorCreate(MPI_COMM_WORLD, grid, &vec_y);
ierr = HYPRE_StructVectorInitialize(vec_y);
ierr = HYPRE_StructVectorSetBoxValues(vec_y, hlo, hhi, vector);
ierr = HYPRE_StructVectorAssemble(vec_y);
#endif
/* Multiply sparse matrix. Start by accessing pointers to local portions of
g_a_data, g_a_j, g_a_i */
NGA_Sync();
for (iloop=0; iloop<LOOPNUM; iloop++) {
t_beg2 = GA_Wtime();
NGA_Distribution(g_c,me,blo,bhi);
NGA_Access(g_c,blo,bhi,&p_c,&ld_c);
for (i = 0; i<bhi[0]-blo[0]+1; i++) {
p_c[i] = 0.0;
}
/* get number of matrix blocks coupled to this process */
NGA_Get(g_a_i,&me,&me,&lo_bl,&one);
#if 1
NGA_Get(g_a_i,&me_plus,&me_plus,&hi_bl,&one);
hi_bl--;
total_procs = hi_bl - lo_bl + 1;
blk_ptr = (void**)malloc(sizeof(void*));
/* Loop through matrix blocks */
ioff = 0;
for (iblock = 0; iblock<total_procs; iblock++) {
t_beg = GA_Wtime();
jdx = lo_bl+iblock;
#if 0
GP_Access_element(g_a_data, &jdx, &blk_ptr[0], &isize);
#endif
#if 1
GP_Get_size(g_a_data, &jdx, &jdx, &isize);
#endif
blk = (void*)malloc(isize);
#if 1
GP_Get(g_a_data, &jdx, &jdx, blk, blk_ptr, &one, &blk_size, &one, &tsize, 0);
#endif
t_gp_get = t_gp_get + GA_Wtime() - t_beg;
iparams = (int*)blk_ptr[0];
rval = (double*)(iparams+7);
imin = iparams[0];
imax = iparams[1];
jmin = iparams[2];
jmax = iparams[3];
irow = iparams[4];
icol = iparams[5];
nnz = iparams[6];
jval = (int*)(rval+nnz);
ival = (int*)(jval+nnz);
nrows = imax - imin + 1;
bvec = (double*)malloc((jmax-jmin+1)*sizeof(double));
j = 0;
t_beg = GA_Wtime();
NGA_Get(g_b,&jmin,&jmax,bvec,&j);
t_get = t_get + GA_Wtime() - t_beg;
t_beg = GA_Wtime();
for (i=0; i<nrows; i++) {
kmin = ival[i];
kmax = ival[i+1]-1;
tempc = 0.0;
for (j = kmin; j<=kmax; j++) {
jj = jval[j];
tempc = tempc + rval[j]*bvec[jj];
}
p_c[i] = p_c[i] + tempc;
}
t_mult = t_mult + GA_Wtime() - t_beg;
free(bvec);
free(blk);
}
NGA_Sync();
t_ga_tot = t_ga_tot + GA_Wtime() - t_beg2;
NGA_Distribution(g_c,me,blo,bhi);
NGA_Release(g_c,blo,bhi);
#if USE_HYPRE
alpha = 1.0;
beta = 0.0;
t_beg = GA_Wtime();
ierr = HYPRE_StructMatrixMatvec(alpha, matrix, vec_x, beta, vec_y);
t_hypre_strct = t_hypre_strct + GA_Wtime() - t_beg;
hlo[0] = lo[0];
hlo[1] = lo[1];
hlo[2] = lo[2];
hhi[0] = hi[0];
hhi[1] = hi[1];
hhi[2] = hi[2];
ierr = HYPRE_StructVectorGetBoxValues(vec_y, hlo, hhi, vector);
NGA_Distribution(g_c,me,hlo,hhi);
cbuf = (double*)malloc((hhi[0]-hlo[0]+1)*sizeof(double));
NGA_Get(g_c,hlo,hhi,cbuf,&one);
prdot = 0.0;
dotga = 0.0;
dothypre = 0.0;
for (i=0; i<(hhi[0]-hlo[0]+1); i++) {
dothypre = dothypre + vector[i]*vector[i];
dotga = dotga + cbuf[i]*cbuf[i];
prdot = prdot + (vector[i]-cbuf[i])*(vector[i]-cbuf[i]);
}
NGA_Dgop(&dotga,1,"+");
NGA_Dgop(&dothypre,1,"+");
NGA_Dgop(&prdot,1,"+");
gatot += sqrt(dotga);
hypretot += sqrt(dothypre);
prtot += sqrt(prdot);
free(cbuf);
#endif
/* Transpose matrix. Start by making local copies of ival and jval arrays for
the sparse matrix of blocks stored in the GP array */
#if 1
t_beg2 = GA_Wtime();
t_beg3 = GA_Wtime();
iblk = (int*)malloc((nprocs+1)*sizeof(int));
iblk_t = (int*)malloc((nprocs+1)*sizeof(int));
#if 0
NGA_Get(g_a_i,&zero,&nprocs,iblk,&one);
#else
if (me == 0) {
NGA_Get(g_a_i,&zero,&nprocs,iblk,&one);
} else {
for (i=0; i<nprocs+1; i++) {
iblk[i] = 0;
}
}
GA_Igop(iblk,nprocs+1,"+");
#endif
jblk = (int*)malloc(iblk[nprocs]*sizeof(int));
jblk_t = (int*)malloc(iblk[nprocs]*sizeof(int));
iblock = iblk[nprocs]-1;
#if 0
NGA_Get(g_a_j,&zero,&iblock,jblk,&one);
#else
if (me == 0) {
NGA_Get(g_a_j,&zero,&iblock,jblk,&one);
} else {
for (i=0; i<iblock+1; i++) {
jblk[i] = 0;
}
}
GA_Igop(jblk,iblock+1,"+");
#endif
iblock++;
blkidx = (int*)malloc(iblk[nprocs]*sizeof(int));
blkidx_t = (int*)malloc(iblk[nprocs]*sizeof(int));
for (i=0; i<iblock; i++) {
blkidx[i] = i;
}
iblock = nprocs;
t_get_blk_csr = t_get_blk_csr + GA_Wtime() - t_beg3;
t_beg3 = GA_Wtime();
stran(iblock, iblock, iblk, jblk, blkidx, iblk_t, jblk_t, blkidx_t);
t_trans_blk_csr = t_trans_blk_csr + GA_Wtime() - t_beg3;
t_beg3 = GA_Wtime();
gt_a_data = GP_Create_handle();
i = iblk_t[nprocs];
GP_Set_dimensions(gt_a_data, one, &i);
GP_Set_irreg_distr(gt_a_data, iblk_t, &nprocs);
GP_Allocate(gt_a_data);
gt_a_j = NGA_Create_handle();
i = iblk_t[nprocs];
NGA_Set_data(gt_a_j, one, &i, C_INT);
NGA_Set_irreg_distr(gt_a_j, iblk_t, &nprocs);
NGA_Allocate(gt_a_j);
gt_a_i = NGA_Create_handle();
i = nprocs+1;
NGA_Set_data(gt_a_i,one,&i,C_INT);
for (i=0; i<nprocs; i++) mapc[i] = i;
NGA_Set_irreg_distr(gt_a_i, mapc, &nprocs);
NGA_Allocate(gt_a_i);
/* copy i and j arrays of transposed matrix into distributed arrays */
if (me==0) {
lo_bl = 0;
hi_bl = nprocs;
NGA_Put(gt_a_i,&lo_bl,&hi_bl,iblk_t,&one);
lo_bl = 0;
hi_bl = iblk_t[nprocs]-1;
NGA_Put(gt_a_j,&lo_bl,&hi_bl,jblk_t,&one);
}
NGA_Sync();
lo_bl = iblk[me];
hi_bl = iblk[me+1];
total_procs = hi_bl - lo_bl + 1;
total_procs = hi_bl - lo_bl;
t_create_csr_ga = t_create_csr_ga + GA_Wtime() - t_beg3;
for (iblock = lo_bl; iblock < hi_bl; iblock++) {
t_beg4 = GA_Wtime();
jdx = blkidx_t[iblock];
GP_Get_size(g_a_data, &jdx, &jdx, &isize);
blk = (void*)malloc(isize);
GP_Get(g_a_data, &jdx, &jdx, blk, blk_ptr, &one, &blk_size, &one, &tsize, 0);
/* Parameters for original block */
iparams = (int*)blk_ptr[0];
rval = (double*)(iparams+7);
imin = iparams[0];
imax = iparams[1];
jmin = iparams[2];
jmax = iparams[3];
irow = iparams[4];
icol = iparams[5];
nnz = iparams[6];
jval = (int*)(rval+nnz);
ival = (int*)(jval+nnz);
/* Create transposed block */
isize = 7*sizeof(int) + nnz*(sizeof(double)+sizeof(int))
+ (jmax-jmin+2)*sizeof(int);
t_gp_tget = t_gp_tget + GA_Wtime() - t_beg4;
t_beg4 = GA_Wtime();
tblk_ptr = (int*)GP_Malloc(isize);
t_gp_malloc = t_gp_malloc + GA_Wtime() - t_beg4;
t_beg3 = GA_Wtime();
iparamst = (int*)tblk_ptr;
rvalt = (double*)(iparamst+7);
jvalt = (int*)(rvalt+nnz);
ivalt = (int*)(jvalt+nnz);
iparamst[0] = jmin;
iparamst[1] = jmax;
iparamst[2] = imin;
iparamst[3] = imax;
iparamst[4] = icol;
iparamst[5] = irow;
iparamst[6] = nnz;
i = imax-imin+1;
j = jmax-jmin+1;
stranr(i, j, ival, jval, rval, ivalt, jvalt, rvalt);
t_trans_blk = t_trans_blk + GA_Wtime() - t_beg3;
t_beg4 = GA_Wtime();
GP_Assign_local_element(gt_a_data, &iblock, (void*)tblk_ptr, isize);
t_gp_assign = t_gp_assign + GA_Wtime() - t_beg4;
#if 1
free(blk);
#endif
}
/* Clean up after transpose */
#if 1
free(iblk);
free(iblk_t);
free(jblk);
free(jblk_t);
free(blkidx);
free(blkidx_t);
#endif
NGA_Sync();
t_ga_trans = t_ga_trans + GA_Wtime() - t_beg2;
#if USE_HYPRE
alpha = 1.0;
beta = 0.0;
ierr = HYPRE_StructMatrixMatvec(alpha, matrix, vec_x, beta, vec_y);
hlo[0] = lo[0];
hlo[1] = lo[1];
hlo[2] = lo[2];
hhi[0] = hi[0];
hhi[1] = hi[1];
hhi[2] = hi[2];
ierr = HYPRE_StructVectorGetBoxValues(vec_y, hlo, hhi, vector);
NGA_Distribution(g_c,me,hlo,hhi);
cbuf = (double*)malloc((hhi[0]-hlo[0]+1)*sizeof(double));
NGA_Get(g_c,hlo,hhi,cbuf,&one);
dothypre = 0.0;
dotga = 0.0;
prdot2 = 0.0;
for (i=0; i<(hhi[0]-hlo[0]+1); i++) {
dothypre = dothypre + vector[i]*vector[i];
dotga = dotga + cbuf[i]*cbuf[i];
if (fabs(vector[i]-cbuf[i]) > 1.0e-10) {
printf("p[%d] i: %d vector: %f cbuf: %f\n",me,i,vector[i],cbuf[i]);
}
prdot2 = prdot2 + (vector[i]-cbuf[i])*(vector[i]-cbuf[i]);
}
NGA_Dgop(&dotga,1,"+");
NGA_Dgop(&dothypre,1,"+");
NGA_Dgop(&prdot2,1,"+");
prtot2 += sqrt(prdot2);
gatot2 += sqrt(dotga);
hypretot2 += sqrt(dothypre);
free(cbuf);
free(blk_ptr);
#endif
/* Clean up transposed matrix */
GP_Distribution(gt_a_data,me,blo,bhi);
for (i=blo[0]; i<bhi[0]; i++) {
GP_Free(GP_Free_local_element(gt_a_data,&i));
}
GP_Destroy(gt_a_data);
NGA_Destroy(gt_a_i);
NGA_Destroy(gt_a_j);
#endif
#endif
}
free(vector);
#if USE_HYPRE
if (me == 0) {
printf("Magnitude of GA solution: %e\n",
gatot/((double)LOOPNUM));
printf("Magnitude of HYPRE solution: %e\n",
hypretot/((double)LOOPNUM));
printf("Magnitude of GA solution(2): %e\n",
gatot2/((double)LOOPNUM));
printf("Magnitude of HYPRE solution(2): %e\n",
hypretot2/((double)LOOPNUM));
printf("Difference between GA and HYPRE (Struct) results: %e\n",
prtot/((double)LOOPNUM));
printf("Difference between transpose and HYPRE results: %e\n",
prtot2/((double)LOOPNUM));
}
#endif
/*
Clean up arrays
*/
NGA_Destroy(g_b);
NGA_Destroy(g_c);
GP_Distribution(g_a_data,me,blo,bhi);
for (i=blo[0]; i<bhi[0]; i++) {
GP_Free(GP_Free_local_element(g_a_data,&i));
}
GP_Destroy(g_a_data);
NGA_Destroy(g_a_i);
NGA_Destroy(g_a_j);
#if USE_HYPRE
ierr = HYPRE_StructStencilDestroy(stencil);
ierr = HYPRE_StructGridDestroy(grid);
ierr = HYPRE_StructMatrixDestroy(matrix);
ierr = HYPRE_StructVectorDestroy(vec_x);
ierr = HYPRE_StructVectorDestroy(vec_y);
#endif
NGA_Dgop(&t_cnstrct,1,"+");
NGA_Dgop(&t_get,1,"+");
NGA_Dgop(&t_gp_get,1,"+");
NGA_Dgop(&t_mult,1,"+");
NGA_Dgop(&t_ga_tot,1,"+");
NGA_Dgop(&t_ga_trans,1,"+");
NGA_Dgop(&t_get_blk_csr,1,"+");
NGA_Dgop(&t_trans_blk_csr,1,"+");
NGA_Dgop(&t_trans_blk,1,"+");
NGA_Dgop(&t_create_csr_ga,1,"+");
NGA_Dgop(&t_gp_tget,1,"+");
NGA_Dgop(&t_gp_malloc,1,"+");
NGA_Dgop(&t_gp_assign,1,"+");
#if USE_HYPRE
NGA_Dgop(&t_hypre_strct,1,"+");
#endif
free(mapc);
if (me == 0) {
printf("Time to create sparse matrix: %12.4f\n",
t_cnstrct/((double)(nprocs*LOOPNUM)));
printf("Time to get right hand side vector: %12.4f\n",
t_get/((double)(nprocs*LOOPNUM)));
printf("Time to get GP blocks: %12.4f\n",
t_gp_get/((double)(nprocs*LOOPNUM)));
printf("Time for sparse matrix block multiplication: %12.4f\n",
t_mult/((double)(nprocs*LOOPNUM)));
printf("Time for total sparse matrix multiplication: %12.4f\n",
t_ga_tot/((double)(nprocs*LOOPNUM)));
#if USE_HYPRE
printf("Total time for HYPRE (Struct) matrix-vector multiply:%12.4f\n",
t_hypre_strct/((double)(nprocs*LOOPNUM)));
#endif
printf("Time to get block CSR distribution: %12.4f\n",
t_get_blk_csr/((double)(nprocs*LOOPNUM)));
printf("Time for transposing block CSR distribution: %12.4f\n",
t_trans_blk_csr/((double)(nprocs*LOOPNUM)));
printf("Time for creating transposed block CSR GA: %12.4f\n",
t_create_csr_ga/((double)(nprocs*LOOPNUM)));
printf("Time for transposing blocks: %12.4f\n",
t_trans_blk/((double)(nprocs*LOOPNUM)));
printf("Time to get GP blocks for transpose: %12.4f\n",
t_gp_tget/((double)(nprocs*LOOPNUM)));
printf("Time to malloc GP blocks for transpose: %12.4f\n",
t_gp_malloc/((double)(nprocs*LOOPNUM)));
printf("Time to assign GP blocks for transpose: %12.4f\n",
t_gp_assign/((double)(nprocs*LOOPNUM)));
printf("Time for total sparse matrix transpose: %12.4f\n",
t_ga_trans/((double)(nprocs*LOOPNUM)));
}
if (me==0) {
printf("Terminating GA library\n");
}
NGA_Terminate();
/*
*** Tidy up after message-passing library
*/
ierr = MPI_Finalize();
}
void do_work()
{
int g_a, g_b;
int me=GA_Nodeid(), nproc=GA_Nnodes(), proc, loop;
int dims[NDIM], lo[NDIM], hi[NDIM], block[NDIM], ld[NDIM-1];
int i,d,*proclist, offset;
int adims[NDIM], ndim,type;
typedef struct {
int lo[NDIM];
int hi[NDIM];
} patch_t;
patch_t *regions;
int *map;
double *buf;
/***** create array A with default distribution *****/
if(me==0){printf("Creating array A\n"); fflush(stdout);}
for(i = 0; i<NDIM; i++)dims[i] = N*(i+1);
#ifdef NEW_API
g_a = GA_Create_handle();
GA_Set_data(g_a,NDIM,dims,MT_F_DBL);
GA_Set_array_name(g_a,"array A");
(void)GA_Allocate(g_a);
#else
g_a = NGA_Create(MT_F_DBL, NDIM, dims, "array A", NULL);
#endif
if(!g_a) GA_Error("create failed: A",0);
if(me==0)printf("OK\n\n");
/* print info about array we got */
NGA_Inquire(g_a, &type, &ndim, adims);
GA_Print_distribution(g_a);
GA_Sync();
/* duplicate array A with ga_create irreg rather than ga_duplicate
* -- want to show distribution control
* -- with ga_duplicate it would be g_b=GA_Duplicate(g_a,name)
*/
if(me==0)printf("\nReconstructing distribution description for A\n");
/* get memory for arrays describing distribution */
proclist = (int*)malloc(nproc*sizeof(int));
if(!proclist)GA_Error("malloc failed for proclist",0);
regions = (patch_t*)malloc(nproc*sizeof(patch_t));
if(!regions)GA_Error("malloc failed for regions",0);
map = (int*)malloc((nproc+ndim)*sizeof(int)); /* ubound= nproc+mdim */
if(!map)GA_Error("malloc failed for map",0);
/* first find out how array g_a is distributed */
for(i=0;i<ndim;i++)lo[i]=BASE;
for(i=0;i<ndim;i++)hi[i]=adims[i] -1 + BASE;
proc = NGA_Locate_region(g_a, lo, hi, (int*)regions, proclist);
if(proc<1) GA_Error("error in NGA_Locate_region",proc);
/* determine blocking for each dimension */
for(i=0;i<ndim;i++)block[i]=0;
for(i=0;i<ndim;i++)adims[i]=0;
offset =0;
for(d=0; d<ndim; d++)
for(i=0;i<proc;i++)
if( regions[i].hi[d]>adims[d] ){
map[offset] = regions[i].lo[d];
offset++;
block[d]++;
adims[d]= regions[i].hi[d];
}
if(me==0){
printf("Distribution map contains %d elements\n",offset);
print_subscript("number of blocks for each dimension",ndim,block,"\n");
print_subscript("distribution map",offset,map,"\n\n");
fflush(stdout);
}
if(me==0)printf("Creating array B applying distribution of A\n");
# ifdef USE_DUPLICATE
g_b = GA_Duplicate(g_a,"array B");
# else
g_b = NGA_Create_irreg(MT_F_DBL, NDIM, dims, "array B", block,map);
# endif
if(!g_b) GA_Error("create failed: B",0);
if(me==0)printf("OK\n\n");
free(proclist); free(regions); free(map);
GA_Print_distribution(g_b);
GA_Sync();
if(me==0){
printf("\nCompare distributions of A and B\n");
if(GA_Compare_distr(g_a,g_b))
printf("Failure: distributions NOT identical\n");
else
printf("Success: distributions identical\n");
fflush(stdout);
}
if(me==0){
printf("\nAccessing local elements of A: set them to the owner process id\n");
fflush(stdout);
}
GA_Sync();
NGA_Distribution(g_a,me,lo,hi);
if(hi[0]>=0){/* -1 means no elements stored on this processor */
double *ptr;
int locdim[NDIM];
NGA_Access(g_a, lo,hi, &ptr, ld);
for(i=0;i<ndim;i++)locdim[i]=hi[i]-lo[i]+1;
fill_patch(ptr, locdim, ld, ndim,(double)me);
}
for(i=0;i<nproc; i++){
if(me==i && hi[0]>=0){
char msg[100];
sprintf(msg,"%d: leading dimensions",me);
print_subscript(msg,ndim-1,ld,"\n");
fflush(stdout);
}
GA_Sync();
}
GA_Sync();
if(me==0)printf("\nRandomly checking the update using ga_get on array sections\n");
GA_Sync();
/* show ga_get working and verify array updates
* every process does N random gets
* for simplicity get only a single row at a time
*/
srand(me); /* different seed for every process */
hi[ndim-1]=adims[ndim-1] -1 + BASE;
for(i=1;i<ndim-1; i++)ld[i]=1; ld[ndim-2]=adims[ndim-1] -1 + BASE;
/* get buffer memory */
buf = (double*)malloc(adims[ndim-1]*sizeof(double));
if(!buf)GA_Error("malloc failed for buf",0);
/* half of the processes check the result */
if(me<=nproc/2)
for(loop = 0; loop< N; loop++){ /* task parallel loop */
lo[ndim-1]=BASE;
for (i= 0; i < ndim -1; i ++){
lo[i] = hi[i] = rand()%adims[i]+BASE;
}
/* print_subscript("getting",ndim,lo,"\n");*/
NGA_Get(g_a,lo,hi,buf,ld);
/* check values */
for(i=0;i<adims[ndim-1]; i++){
int p = NGA_Locate(g_a, lo);
if((double)p != buf[i]) {
char msg[100];
sprintf(msg,"%d: wrong value: %d != %lf a",me, p, buf[i]);
print_subscript(msg,ndim,lo,"\n");
GA_Error("Error - bye",i);
}
lo[ndim-1]++;
}
}
free(buf);
GA_Sync();
if(me==0)printf("OK\n");
GA_Destroy(g_a);
GA_Destroy(g_b);
}
