validate_transpose(int g_A, int g_B, int* lo, int* hi, int ld)
{
int i, j;
int local_A[SIZE][SIZE], local_B[SIZE][SIZE];
// int **local_A=NULL, **local_B=NULL;
/*
local_A=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_value[i]=(int*)malloc(SIZE*sizeof(int));
local_B=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_value[i]=(int*)malloc(SIZE*sizeof(int));
*/
NGA_Get(g_A, lo, hi, local_A, &ld);
NGA_Get(g_B, lo, hi, local_B, &ld);
for(i=1; i<SIZE; i++)
{
for(j=1; j<SIZE; j++)
{
if(local_B[j][i]!=local_A[i][j])
GA_Error("ERROR : in passing values", DIM);
}
}
}
/*
* Check to see if inversion is correct. Start by copying g_a into local
* buffer a, and g_b into local buffer b.
*/
void verify(int g_a, int g_b) {
int i, type, ndim, dims[MAXDIM], lo[MAXDIM], hi[MAXDIM], ld[MAXDIM];
int a[MAXPROC*TOTALELEMS],b[MAXPROC*TOTALELEMS];
/* Get dimensions of GA */
NGA_Inquire(g_a, &type, &ndim, dims);
lo[0] = 0;
hi[0] = dims[0]-1;
/* ### copy the block of data described by the arrays "lo" and "hi" from
* ### the global array "g_a" into the local array "a". Copy the same block
* ### of data from "g_b" into the local array "b". Use the array of strides
* ### "ld" to describe the physical layout of "a" and "b". */
NGA_Get(g_a,lo,hi,a,ld);
NGA_Get(g_b,lo,hi,b,ld);
for(i=0; i<dims[0]; i++)
if (a[i] != b[dims[0]-i-1])
{
printf("Mismatch: a[%d]=%d is not equal to b[%d]=%d\n",
i, a[i], dims[0]-i-1, b[dims[0]-i-1]);
GA_Error("verify failed",0);
}
printf(" Transpose OK\n");
}
fillandscale(int rank, int nprocs)
{
int g_A, val1=5, val2=5, local_A[SIZE][SIZE], i, j;
int dims[DIM]={SIZE,SIZE}, alo[DIM]={1,1}, ahi[DIM]={2,2}, ld=5;
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
GA_Zero(g_A);
NGA_Fill_patch(g_A, alo, ahi, &val1);
GA_Print(g_A);
GA_Scale(g_A, &val2);
GA_Print(g_A);
NGA_Get(g_A, alo, ahi, local_A, &ld);
if(rank == 1)
{
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++) if(local_A[i][j]!=val1*val2)
printf(" GA ERROR: \n");
}
}
GA_Destroy(g_A);
}
void FATR
ga_antisymmetrize_(Integer *g_a) {
DoublePrecision alpha = 0.5;
int i, me = GA_Nodeid();
extern void * FATR ga_malloc(Integer nelem, int type, char *name);
extern void FATR ga_free(void *ptr);
void FATR gai_subtr(int *lo, int *hi, void *a, void *b, DoublePrecision alpha,
int type, Integer nelem, int ndim);
int alo[GA_MAX_DIM], ahi[GA_MAX_DIM], lda[GA_MAX_DIM];
int blo[GA_MAX_DIM], bhi[GA_MAX_DIM], ldb[GA_MAX_DIM];
int ndim, dims[GA_MAX_DIM], type;
Integer nelem=1;
Logical have_data;
void *a_ptr, *b_ptr;
GA_Sync();
NGA_Inquire((int)(*g_a), &type, &ndim, dims);
if (dims[0] != dims[1])
GA_Error("ga_sym: can only sym square matrix", 0L);
/* Find the local distribution */
NGA_Distribution((int)(*g_a), me, alo, ahi);
have_data = ahi[0]>=0;
for(i=1; i<ndim; i++) have_data = have_data && ahi[i]>=0;
if(have_data) {
NGA_Access((int)(*g_a), alo, ahi, &a_ptr, lda);
for(i=0; i<ndim; i++) nelem *= ahi[i]-alo[i] +1;
b_ptr = (void *) ga_malloc(nelem, MT_C_DBL, "v");
for(i=2; i<ndim; i++) {bhi[i]=ahi[i]; blo[i]=alo[i]; }
/* switch rows and cols */
blo[1]=alo[0];
bhi[1]=ahi[0];
blo[0]=alo[1];
bhi[0]=ahi[1];
for (i=0; i < ndim-1; i++)
ldb[i] = bhi[i+1] - blo[i+1] + 1;
NGA_Get((int)(*g_a), blo, bhi, b_ptr, ldb);
}
GA_Sync();
if(have_data) {
gai_subtr(alo, ahi, a_ptr, b_ptr, alpha, type, nelem, ndim);
NGA_Release_update((int)(*g_a), alo, ahi);
ga_free(b_ptr);
}
GA_Sync();
}
// -------------------------------------------------------------
// MatGetValues_DenseGA
// -------------------------------------------------------------
static
PetscErrorCode
MatGetValues_DenseGA(Mat mat,
PetscInt m, const PetscInt idxm[], PetscInt n, const PetscInt idxn[],
PetscScalar v[])
{
PetscErrorCode ierr = 0;
struct MatGACtx *ctx;
int i, j, idx;
PetscScalar vij;
int lo[2], hi[2], ld[2] = {1, 1};
ierr = MatShellGetContext(mat, (void *)&ctx); CHKERRQ(ierr);
idx = 0;
for (i = 0; i < m; ++i) {
for (j = 0; j < n; ++j, ++idx) {
lo[0] = idxm[i];
hi[0] = idxm[i];
lo[1] = idxn[j];
hi[1] = idxn[j];
NGA_Get(ctx->ga, lo, hi, (void *)&vij, ld);
v[idx] = vij;
}
}
return ierr;
}
int FormFunctionGradient (TAO_GA_APPLICATION gaapp, GAVec ga_X, double *f, GAVec ga_G, void *ptr)
{
int lo, hi; //the global coordinates
AppCtx *user = (AppCtx *) ptr;
int i,j;
double *g, *x;
double xx,yy,zz,temp,rij;
MA_get_pointer(user->memHandle, &x);
g = x + user->ndim*user->natoms;
lo=0;
hi=user->n-1; /* range of array indices */
NGA_Get(ga_X, &lo, &hi, x, &hi);
*f = 0;
for (i=0; i < user->n; i++)
g[i] = 0.0;
if (user->ndim == 2) {
for (j=1; j < user->natoms; j++) {
for (i=0; i<j; i++) {
xx = x[2*j] - x[2*i];
yy = x[2*j+1] - x[2*i+1];
rij = xx*xx + yy*yy;
temp = 1.0/rij/rij/rij;
*f += temp*(temp-2.0);
temp *= 12.0*(temp-1.0)/rij;
g[2*j] -= xx*temp;
g[2*j+1] -= yy*temp;
g[2*i] += xx*temp;
g[2*i+1] += yy*temp;
}
}
} else if (user->ndim == 3) {
for (j=1; j < user->natoms; j++) {
for (i=0; i < j; i++) {
xx = x[3*j] - x[3*i];
yy = x[3*j+1] - x[3*i+1];
zz = x[3*j+2] - x[3*i+2];
rij = xx*xx + yy*yy + zz*zz;
temp = 1.0/rij/rij/rij;
*f += temp*(temp-2.0);
temp *= 12.0*(temp-1.0)/rij;
g[3*j] -= xx*temp;
g[3*j+1] -= yy*temp;
g[3*j+2] -= zz*temp;
g[3*i] += xx*temp;
g[3*i+1] += yy*temp;
g[3*i+2] += zz*temp;
}
}
}
NGA_Put(ga_G, &lo, &hi, g, &hi);
return 0;
}
int getBlock(GAVec g_x, int taskId, AppCtx *user)
{
int lo, hi;
int size;
size = user->BlockSize * user->ndim;
/* get the coordinates of the atoms in the corresponding rows in the block */
lo = user->btopo[taskId].x*size;
hi = lo + size -1;
NGA_Get(g_x, &lo, &hi, user->x1, &hi);
/* get the coordinates of the atoms in the corresponding cols in the block */
lo = user->btopo[taskId].y*size;
hi = lo + size - 1;
NGA_Get(g_x, &lo, &hi, user->x2, &hi);
return 0;
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, local_C[DIM][DIM], dims[DIM]={5,5};
int val_A=5, val_B=3, ld=DIM, max;
int lo[DIM]={2,2}, hi[DIM]={4,4}, blo[DIM]={0,0}, bhi[DIM]={2,2}, clo[DIM]={1,1}, chi[DIM]={3,3};
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = NGA_Create(C_INT, DIM, dims, "array_B", NULL);
g_C = NGA_Create(C_INT, DIM, dims, "array_C", NULL);
GA_Fill(g_A, &val_A);
GA_Fill(g_B, &val_B);
GA_Zero(g_C);
GA_Elem_maximum_patch(g_A, lo, hi, g_B, blo, bhi, g_C, clo, chi);
GA_Print(g_C);
GA_Sync();
NGA_Get(g_C, clo, chi, local_C, &ld);
if(rank==1)
{
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
if(val_A>val_B) max=val_A;
else max=val_B;
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)
if(local_C[i][j]!=max) printf("GA Error : \n");
}
}
GA_Sync();
if(rank == 0)
printf("Test Completed \n");
GA_Terminate();
MPI_Finalize();
}
main(int argc, char **argv)
{
int rank, nprocs;
int g_A, g_V, val1=5, val2=5, local_A[SIZE][SIZE], dims_V=SIZE, local_V[dims_V];
int dims[DIM]={SIZE,SIZE}, dims2[DIM], lo[DIM]={1,1}, hi[DIM]={2,2}, ld=5, i, j;
int loV=0, hiV=dims_V-1;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_V = NGA_Create(C_INT, 1, &dims_V, "array_A", NULL);
GA_Fill(g_A, &val1);
GA_Print(g_A);
printf("\n");
GA_Scale(g_A, &val2);
GA_Print(g_A);
GA_Get_diag(g_A, g_V);
GA_Print(g_V);
NGA_Get(g_A, lo, hi, local_A, &ld);
NGA_Get(g_V, &loV, &hiV, local_V, &ld);
if(rank==1)
{
for(i=0; i<dims_V; i++)
if(local_V[i]!=val1*val2) printf(" GA Error: \n");
}
if(rank == 0)
printf("Test Completed \n");
GA_Terminate();
MPI_Finalize();
}
int workq_append_task_single_(
int *task_id,
int *tile_dim,
int *g_a,
int *g_b,
int *ld,
double *bufa,
double *bufb
) {
struct bench_buf buf;
int lo[2], hi[2], myld[1], tile_size;
buf.mtype = *task_id+3;
buf.task_id = *task_id;
buf.tile_dim = *tile_dim;
tile_size = *tile_dim * (*tile_dim);
lo[1] = *task_id*tile_size;
hi[1] = lo[1] + tile_size - 1;
lo[0] = 0;
hi[0] = 0;
myld[0] = *ld;
//printf("lo={%d,%d}, hi={%d,%d}\n", lo[0], lo[1], hi[0], hi[1]);
NGA_Get(*g_a, lo, hi, shmdata+offset, myld);
offset += tile_size;
NGA_Get(*g_b, lo, hi, shmdata+offset, myld);
offset += tile_size;
tot_size += sizeof(double)*(tile_size*2);
//int i,j;
//printf("bufa: ");
//for (i=0; i<*tile_dim; i++) {
// for (j=0; j<*tile_dim; j++) {
// printf("%f ", bufa[*tile_dim*i+j]);
// }
//}
bench_bufs[num_microtasks++] = buf;
return 0;
}
int InitializeVariables(GAVec ga_X, AppCtx *user)
{
double *x;
double xx, yy, zz;
int isqrtn, icrtn, left, i, j, k, il, jl, ctr;
int lo, hi;
lo = 0; hi = user->n - 1; /* range of array indices */
MA_get_pointer(user->memHandle, &x);
NGA_Get (ga_X, &lo, &hi, x, &hi);
if (user->ndim == 2) {
isqrtn = (int) sqrt( (double) user->natoms);
left = user->natoms - isqrtn * isqrtn;
xx = 0.0;
yy = 0.0;
for (j=0; j<=isqrtn + left/isqrtn; j++) {
for (i=0; i < TaoMin(isqrtn, user->natoms - j*isqrtn); i++) {
ctr = j*isqrtn + i;
x[2*ctr] = xx;
x[2*ctr+1] = yy;
xx += 1.0;
}
yy += 1.0;
xx = 0.0;
}
}
else if (user->ndim == 3) {
icrtn = (int) pow((user->natoms + 0.5),1.0/3.0);
left = user->natoms - icrtn * icrtn * icrtn;
xx = yy = zz = 0.0;
for (k=0; k <= icrtn + left; k++) {
jl = TaoMin(icrtn, (user->natoms - k*icrtn*icrtn)/icrtn+1);
for (j=0; j<jl; j++) {
il = TaoMin(icrtn, user->natoms - k*icrtn*icrtn - j*icrtn);
for (i=0; i<il; i++) {
ctr = k*icrtn*icrtn + j*icrtn + i;
x[3*ctr] = xx;
x[3*ctr+1] = yy;
x[3*ctr+2] = zz;
xx += 1.0;
}
yy += 1.0;
xx = 0.0;
}
zz += 1.0;
yy = 0.0;
}
}
NGA_Put(ga_X, &lo, &hi, x, &hi);
return 0;
}
PetscErrorCode testCreate2D()
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 1453, d2 = 1451;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal **v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym;
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[j][i] = 1.*i + d1 * j;
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"Updated local portion with DAVec\n");
PetscBarrier(0);
{
double *da_ptr;
VecGetArray(vec, &da_ptr);
double *ptr;
int low[2],hi[2],ld;
NGA_Distribution(ga,GA_Nodeid(),low,hi);
NGA_Access(ga,low,hi,&ptr,&ld);
printf("[%d] ga:%p\tda:%p\tdiff:%p\n", GA_Nodeid(), ptr, da_ptr, (ptr-da_ptr) );
NGA_Release_update(ga,low,hi);
}
int lo[2],ld;
double val;
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = j;
lo[1] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j - val) > .1 )
printf(".");
// printf("[%d] (%3.0f,%3.0f)\n", GA_Nodeid(), 1.*i + d1*j, val);
}
}
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, **local_A=NULL, **local_B=NULL;
int dims[DIM]={SIZE,SIZE}, dims2[DIM], lo[DIM]={SIZE-SIZE,SIZE-SIZE}, hi[DIM]={SIZE-1,SIZE-1}, ld=5, value=5;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
local_A=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
{
local_A[i]=(int*)malloc(SIZE*sizeof(int));
for(j=0; j<SIZE; j++) local_A[i][j]=rand()%10;
}
local_B=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
{
local_B[i]=(int*)malloc(SIZE*sizeof(int));
for(j=0; j<SIZE; j++) local_B[i][j]=rand()%10;
}
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
GA_Zero(g_A);
if(rank==0)
{
NGA_Put(g_A, lo, hi, local_A, &ld);
NGA_Get(g_A, lo, hi, local_B, &ld);
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)
if(local_A[i][j]!=local_B[i][j]) GA_ERROR_MSG();
}
}
GA_Sync();
GA_Destroy(g_A);
if(rank == 0) GA_PRINT_MSG();
GA_Terminate();
MPI_Finalize();
}
void coefs_ga_get_3d(ga_coefs_t *ga_coefs,void *mini_cube,int x,int y,int z)
{
int nsplines = ga_coefs->nsplines;
int lo[4],hi[4],ld[4];
lo[0]=x;lo[1]=y;lo[2]=z;lo[3]=0;
hi[0]=x+3;hi[1]=y+3;hi[2]=z+3;hi[3]=nsplines-1;
ld[0]=ld[1]=4;ld[2]=nsplines;
double begin=MPI_Wtime()*1000;
NGA_Get(ga_coefs->g_a,lo,hi,mini_cube,ld);
double end=MPI_Wtime()*1000;
ga_coefs->amount++;
ga_coefs->sumt+=end-begin;
return;
}
// -------------------------------------------------------------
// MatConvertGAtoDense
// -------------------------------------------------------------
PetscErrorCode
MatConvertGAToDense(Mat A, Mat *B)
{
PetscErrorCode ierr = 0;
MPI_Comm comm;
int nproc;
struct MatGACtx *ctx;
PetscInt lrows, grows, lcols, gcols, lo, hi;
ierr = PetscObjectGetComm((PetscObject)A, &comm); CHKERRQ(ierr);
ierr = MPI_Comm_size(comm, &nproc);
ierr = MatShellGetContext(A, &ctx); CHKERRQ(ierr);
ierr = MatGetSize(A, &grows, &gcols); CHKERRQ(ierr);
ierr = MatGetLocalSize(A, &lrows, &lcols); CHKERRQ(ierr);
ierr = MatCreateDense(comm, lrows, lcols, grows, gcols, NULL, B); CHKERRQ(ierr);
ierr = MatCreate(comm, B); CHKERRXX(ierr);
ierr = MatSetSizes(*B, lrows, lcols, grows, gcols); CHKERRXX(ierr);
if (nproc == 1) {
ierr = MatSetType(*B, MATSEQDENSE); CHKERRXX(ierr);
ierr = MatSeqDenseSetPreallocation(*B, PETSC_NULL); CHKERRXX(ierr);
} else {
ierr = MatSetType(*B, MATDENSE); CHKERRXX(ierr);
ierr = MatMPIDenseSetPreallocation(*B, PETSC_NULL); CHKERRXX(ierr);
}
ierr = MatGetOwnershipRange(*B, &lo, &hi); CHKERRQ(ierr);
std::vector<PetscInt> cidx(gcols);
for (PetscInt c = 0; c < gcols; ++c) {
cidx[c] = c;
}
std::vector<PetscScalar> rowvals(gcols);
for (PetscInt r = lo; r < hi; ++r) {
int glo[2] = {r, 0};
int ghi[2] = {r, gcols - 1};
int ld[2] = {1,1};
NGA_Get(ctx->ga, glo, ghi, &rowvals[0], ld);
ierr = MatSetValues(*B, 1, &r, gcols, &cidx[0], &rowvals[0], INSERT_VALUES); CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(*B, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
ierr = MatAssemblyEnd(*B, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
return ierr;
}
// -------------------------------------------------------------
// GA2Vec
// -------------------------------------------------------------
static
int
GA2Vec(int ga, Vec x)
{
int lrows, rows;
PetscErrorCode ierr = 0;
ierr = VecGetLocalSize(x, &lrows); CHKERRQ(ierr);
ierr = VecGetSize(x, &rows); CHKERRQ(ierr);
PetscInt vlo, vhi;
ierr = VecGetOwnershipRange(x, &vlo, &vhi); CHKERRQ(ierr);
PetscScalar *v;
ierr = VecGetArray(x, &v); CHKERRQ(ierr);
int lo[2] = {vlo,0}, hi[2] = {vhi-1,0}, ld[2] = {1,1};
NGA_Get(ga, lo, hi, v, ld);
ierr = VecRestoreArray(x, &v); CHKERRQ(ierr);
return ierr;
}
PetscErrorCode testCreate3D( )
{
int ga;
DA da;
DALocalInfo info;
Vec vec;
PetscErrorCode ierr;
PetscFunctionBegin;
int d1 = 229, d2 = 229, d3 = 229;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
ierr = DACreate3d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,d3,
PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1,1,
0,0,0, &da); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr);
PetscReal ***v;
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
int xe = info.xs+info.xm,
ye = info.ys+info.ym,
ze = info.zs+info.zm;
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
v[k][j][i] = 1.*i + d1*j + d1*d2*k;
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Sequential values filled in petsc vec.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
int lo[3],ld, p = 10;
int patch[10][10][10];
double val;
for (int k = 0; k < d3; k+=p) {
for (int j = 0; j < d2; j+=p) {
for (int i = 0; i < d1; i+=p) {
lo[0] = k;
lo[1] = j;
lo[2] = i;
NGA_Get(ga,lo,lo,&val,&ld);
if( PetscAbs( i + d1*j + d1*d2*k - val) > .1 )
// printf(".");
printf("(%3.0f,%3.0f) ", 1.*i + d1*j + d1*d2*k, val);
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Get() test.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
if( rank == 0 )
{
for (int k = 0; k < d3; ++k) {
printf(">%d\n",k);
for (int j = 0; j < d2; ++j) {
for (int i = 0; i < d1; ++i) {
lo[0] = k; lo[1] = j; lo[2] = i;
val = 1.*i + d1*j + d1*d2*k;
val *= -1;
NGA_Put(ga,lo,lo,&val,&ld);
}
}
}
}
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended NGA_Put() negative seq values.\n"); CHKERRQ(ierr);
ierr = PetscBarrier(0); CHKERRQ(ierr);
ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr);
for (int k = info.zs; k < ze; ++k) {
for (int j = info.ys; j < ye; ++j) {
for (int i = info.xs; i < xe; ++i) {
val = -1 * (1.*i + d1*j + d1*d2*k);
if( PetscAbs( val - v[k][j][i] ) > .1 )
printf(".");
}
}
}
ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD, "Ended petsc vec update test.\n"); CHKERRQ(ierr);
if( rank == 0 )
GA_Print_stats();
ierr = VecDestroy(vec); CHKERRQ(ierr);
GA_Destroy(ga);
PetscFunctionReturn(0);
}
/* Square matrix-matrix multiplication */
void matrix_multiply(int M, int N, int K,
int blockX_len, int blockY_len)
{
/* Local buffers and Global arrays declaration */
double *a=NULL, *b=NULL, *c=NULL;
int dims[NDIMS], ld[NDIMS], chunks[NDIMS];
int lo[NDIMS], hi[NDIMS], cdims[NDIMS]; /* dim of blocks */
int g_a, g_b, g_c, g_cnt, g_cnt2;
int offset;
double alpha = 1.0, beta=0.0;
int count_p = 0, next_p = 0;
int count_gac = 0, next_gac = 0;
double t1,t2,seconds;
ga_nbhdl_t nbh;
int count_acc = 0;
/* Find local processor ID and the number of processes */
int proc=GA_Nodeid(), nprocs=GA_Nnodes();
if ((M % blockX_len) != 0 || (M % blockY_len) != 0 || (N % blockX_len) != 0 || (N % blockY_len) != 0
|| (K % blockX_len) != 0 || (K % blockY_len) != 0)
GA_Error("Dimension size M/N/K is not divisible by X/Y block sizes", 101);
/* Allocate/Set process local buffers */
a = malloc (blockX_len * blockY_len * sizeof(double));
b = malloc (blockX_len * blockY_len * sizeof(double));
c = malloc (blockX_len * blockY_len * sizeof(double));
cdims[0] = blockX_len;
cdims[1] = blockY_len;
/* Configure array dimensions */
for(int i = 0; i < NDIMS; i++) {
dims[i] = N;
chunks[i] = -1;
ld[i] = cdims[i]; /* leading dimension/stride of the local buffer */
}
/* create a global array g_a and duplicate it to get g_b and g_c*/
g_a = NGA_Create(C_DBL, NDIMS, dims, "array A", chunks);
if (!g_a)
GA_Error("NGA_Create failed: A", NDIMS);
#if DEBUG>1
if (proc == 0)
printf(" Created Array A\n");
#endif
/* Ditto for C and B */
g_b = GA_Duplicate(g_a, "array B");
g_c = GA_Duplicate(g_a, "array C");
if (!g_b || !g_c)
GA_Error("GA_Duplicate failed",NDIMS);
if (proc == 0)
printf("Created Arrays B and C\n");
/* Subscript array for read-incr, which is nothing but proc */
int * rdcnt = malloc (nprocs * sizeof(int));
memset (rdcnt, 0, nprocs * sizeof(int));
int * rdcnt2 = malloc (nprocs * sizeof(int));
memset (rdcnt2, 0, nprocs * sizeof(int));
/* Create global array of nprocs elements for nxtval */
int counter_dim[1];
counter_dim[0] = nprocs;
g_cnt = NGA_Create(C_INT, 1, counter_dim, "Shared counter", NULL);
if (!g_cnt)
GA_Error("Shared counter failed",1);
g_cnt2 = GA_Duplicate(g_cnt, "another shared counter");
if (!g_cnt2)
GA_Error("Another shared counter failed",1);
GA_Zero(g_cnt);
GA_Zero(g_cnt2);
#if DEBUG>1
/* initialize data in matrices a and b */
if(proc == 0)
printf("Initializing local buffers - a and b\n");
#endif
int w = 0;
int l = 7;
for(int i = 0; i < cdims[0]; i++) {
for(int j = 0; j < cdims[1]; j++) {
a[i*cdims[1] + j] = (double)(++w%29);
b[i*cdims[1] + j] = (double)(++l%37);
}
}
/* Copy data to global arrays g_a and g_b from local buffers */
next_p = NGA_Read_inc(g_cnt2,&rdcnt[proc],(long)1);
for (int i = 0; i < N; i+=cdims[0])
{
if (next_p == count_p) {
for (int j = 0; j < N; j+=cdims[1])
{
/* Indices of patch */
lo[0] = i;
lo[1] = j;
hi[0] = lo[0] + cdims[0];
hi[1] = lo[1] + cdims[1];
hi[0] = hi[0]-1;
hi[1] = hi[1]-1;
#if DEBUG>1
printf ("%d: PUT_GA_A_B: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_Put(g_a, lo, hi, a, ld);
NGA_Put(g_b, lo, hi, b, ld);
}
next_p = NGA_Read_inc(g_cnt2,&rdcnt[proc],(long)1);
}
count_p++;
}
#if DEBUG>1
printf ("After NGA_PUT to global - A and B arrays\n");
#endif
/* Synchronize all processors to make sure puts from
nprocs has finished before proceeding with dgemm */
GA_Sync();
t1 = GA_Wtime();
next_gac = NGA_Read_inc(g_cnt,&rdcnt2[proc],(long)1);
for (int m = 0; m < N; m+=cdims[0])
{
for (int k = 0; k < N; k+=cdims[0])
{
if (next_gac == count_gac)
{
/* A = m x k */
lo[0] = m; lo[1] = k;
hi[0] = cdims[0] + lo[0]; hi[1] = cdims[1] + lo[1];
hi[0] = hi[0]-1; hi[1] = hi[1]-1;
#if DEBUG>3
printf ("%d: GET GA_A: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_Get(g_a, lo, hi, a, ld);
for (int n = 0; n < N; n+=cdims[1])
{
memset (c, 0, sizeof(double) * cdims[0] * cdims[1]);
/* B = k x n */
lo[0] = k; lo[1] = n;
hi[0] = cdims[0] + lo[0]; hi[1] = cdims[1] + lo[1];
hi[0] = hi[0]-1; hi[1] = hi[1]-1;
#if DEBUG>3
printf ("%d: GET_GA_B: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_Get(g_b, lo, hi, b, ld);
//_my_dgemm_ (a, local_N, b, local_N, c, local_N, local_N, local_N, local_N, alpha, beta=1.0);
/* TODO I am assuming square matrix blocks, further testing/work
required for rectangular matrices */
cblas_dgemm ( CblasRowMajor, CblasNoTrans, /* TransA */CblasNoTrans, /* TransB */
cdims[0] /* M */, cdims[1] /* N */, cdims[0] /* K */, alpha,
a, cdims[0], /* lda */ b, cdims[1], /* ldb */
beta=1.0, c, cdims[0] /* ldc */);
NGA_NbWait(&nbh);
/* C = m x n */
lo[0] = m; lo[1] = n;
hi[0] = cdims[0] + lo[0]; hi[1] = cdims[1] + lo[1];
hi[0] = hi[0]-1; hi[1] = hi[1]-1;
#if DEBUG>3
printf ("%d: ACC_GA_C: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_NbAcc(g_c, lo, hi, c, ld, &alpha, &nbh);
count_acc += 1;
} /* END LOOP N */
next_gac = NGA_Read_inc(g_cnt,&rdcnt2[proc],(long)1);
} /* ENDIF if count == next */
count_gac++;
} /* END LOOP K */
} /* END LOOP M */
GA_Sync();
t2 = GA_Wtime();
seconds = t2 - t1;
if (proc == 0)
printf("Time taken for MM (secs):%lf \n", seconds);
printf("Number of ACC: %d\n", count_acc);
/* Correctness test - modify data again before this function */
for (int i = 0; i < NDIMS; i++) {
lo[i] = 0;
hi[i] = dims[i]-1;
ld[i] = dims[i];
}
verify(g_a, g_b, g_c, lo, hi, ld, N);
/* Clear local buffers */
free(a);
free(b);
free(c);
free(rdcnt);
free(rdcnt2);
GA_Sync();
/* Deallocate arrays */
GA_Destroy(g_a);
GA_Destroy(g_b);
GA_Destroy(g_c);
GA_Destroy(g_cnt);
GA_Destroy(g_cnt2);
}
int schwartz_screening(PFock_t pfock, BasisSet_t basis)
{
int myrank;
MPI_Comm_rank(MPI_COMM_WORLD, &myrank);
// create shell pairs values
//ERD_t erd;
int nthreads = omp_get_max_threads();
//CInt_createERD(basis, &erd, nthreads);
int nshells = pfock->nshells;
// create global arrays for screening
int nprow = pfock->nprow;
int npcol = pfock->npcol;
int dims[2];
int block[2];
int map[nprow + npcol];
for (int i = 0; i < nprow; i++) {
map[i] = pfock->rowptr_sh[i];
}
for (int i = 0; i < npcol; i++) {
map[i + nprow] = pfock->colptr_sh[i];
}
dims[0] = nshells;
dims[1] = nshells;
block[0] = nprow;
block[1] = npcol;
pfock->ga_screening =
NGA_Create_irreg(C_DBL, 2, dims, "array Screening", block, map);
if (0 == pfock->ga_screening) {
return -1;
}
// compute the max shell value
double *sq_values = (double *)PFOCK_MALLOC(sizeof(double) *
pfock->nshells_row * pfock->nshells_col);
if (NULL == sq_values) {
return -1;
}
int startM = pfock->sshell_row;
int startN = pfock->sshell_col;
int endM = pfock->eshell_row;
int endN = pfock->eshell_col;
double maxtmp = 0.0;
#pragma omp parallel
{
int tid = omp_get_thread_num();
#pragma omp for reduction(max:maxtmp)
for (int M = startM; M <= endM; M++) {
int dimM = CInt_getShellDim(basis, M);
for (int N = startN; N <= endN; N++) {
int dimN = CInt_getShellDim(basis, N);
double *integrals;
int nints=
ComputeShellQuartet(basis,tid,M,N,M,N,&integrals);
//CInt_computeShellQuartet(basis, erd, tid, M, N, M, N,
// &integrals, &nints);
double maxvalue = 0.0;
if (nints != 0) {
for (int iM = 0; iM < dimM; iM++) {
for (int iN = 0; iN < dimN; iN++) {
int index =
iM * (dimN*dimM*dimN+dimN) + iN * (dimM*dimN+1);
if (maxvalue < fabs(integrals[index])) {
maxvalue = fabs(integrals[index]);
}
}
}
}
sq_values[(M - startM) * (endN - startN + 1) + (N - startN)]
= maxvalue;
if (maxvalue > maxtmp) {
maxtmp = maxvalue;
}
}
}
}
int lo[2];
int hi[2];
lo[0] = startM;
hi[0] = endM;
lo[1] = startN;
hi[1] = endN;
int ld = endN - startN + 1;
NGA_Put(pfock->ga_screening, lo, hi, sq_values, &ld);
// max value
MPI_Allreduce(&maxtmp, &(pfock->maxvalue), 1,
MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD);
//CInt_destroyERD(erd);
PFOCK_FREE(sq_values);
// init shellptr
sq_values = (double *)PFOCK_MALLOC(sizeof(double) * nshells);
if (NULL == sq_values) {
return -1;
}
int nnz = 0;
double eta = pfock->tolscr2 / pfock->maxvalue;
pfock->shellptr = (int *)PFOCK_MALLOC(sizeof(int) * (nshells + 1));
pfock->mem_cpu += 1.0 * sizeof(int) * (nshells + 1);
if (NULL == pfock->shellptr) {
return -1;
}
memset(pfock->shellptr, 0, sizeof(int) * (nshells + 1));
for (int M = 0; M < nshells; M++) {
pfock->shellptr[M] = nnz;
lo[0] = M;
hi[0] = M;
lo[1] = 0;
hi[1] = nshells - 1;
ld = nshells;
NGA_Get(pfock->ga_screening, lo, hi, sq_values, &ld);
for (int N = 0; N < nshells; N++) {
double maxvalue = sq_values[N];
if (maxvalue > eta) {
if (M > N && (M + N) % 2 == 1 || M < N && (M + N) % 2 == 0) {
continue;
} else {
nnz++;
}
}
}
pfock->shellptr[M + 1] = nnz;
}
pfock->nnz = nnz;
double maxvalue;
pfock->shellvalue = (double *)PFOCK_MALLOC(sizeof(double) * nnz);
pfock->shellid = (int *)PFOCK_MALLOC(sizeof(int) * nnz);
pfock->shellrid = (int *)PFOCK_MALLOC(sizeof(int) * nnz);
pfock->mem_cpu += 1.0 * sizeof(double) * nnz + 2.0 * sizeof(int) * nnz;
nshells = pfock->nshells;
if (pfock->shellvalue == NULL ||
pfock->shellid == NULL ||
pfock->shellrid == NULL) {
return -1;
}
nnz = 0;
for (int A = 0; A < nshells; A++) {
pfock->shellptr[A] = nnz;
lo[0] = A;
hi[0] = A;
lo[1] = 0;
hi[1] = nshells - 1;
ld = nshells;
NGA_Get(pfock->ga_screening, lo, hi, sq_values, &ld);
for (int B = 0; B < nshells; B++) {
maxvalue = sq_values[B];
if (maxvalue > eta) {
if (A > B && (A + B) % 2 == 1 || A < B && (A + B) % 2 == 0)
continue;
if (A == B) {
pfock->shellvalue[nnz] = maxvalue;
} else {
pfock->shellvalue[nnz] = -maxvalue;
}
pfock->shellid[nnz] = B;
pfock->shellrid[nnz] = A;
nnz++;
}
}
}
PFOCK_FREE(sq_values);
GA_Destroy(pfock->ga_screening);
return 0;
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, local_C[DIM][DIM], dims[DIM]={5,5}, val1=5, val2=4, alpha=3, beta=2, ld=5;
int alo[DIM]={2,2}, ahi[DIM]={3,3}, blo[DIM]={2,2}, bhi[DIM]={3,3}, clo[DIM]={1,1}, chi[DIM]={2,2};
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = GA_Duplicate(g_A, "array_B");
g_C = GA_Duplicate(g_A, "array_C");
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
GA_Zero(g_C);
NGA_Add_patch(&alpha, g_A, clo, chi, &beta, g_B, blo, bhi, g_C, clo, chi);
GA_Sync();
GA_Print(g_A);
GA_Print(g_B);
GA_Print(g_C);
NGA_Get(g_C, clo, chi, local_C, &ld);
//printf("check 1 \n");
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
if(rank == 0)
{
printf("check 2\n");
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)
if(local_C[i][j]!=(alpha*val1)+(beta*val2)) printf("GA Error : \n");
}
}
if(rank==0)
GA_PRINT_MSG();
GA_Sync();
/*
GA_Destroy(g_A);
GA_Destroy(g_B);
GA_Destroy(g_C);
*/
//*******************************************************************
/* what would be the possible reason for GA_destroy to get failed ..,
* solve this before consolidate the whole
*/
GA_Terminate();
MPI_Finalize();
}
void do_work()
{
int ZERO=0; /* useful constants */
int g_a, g_b;
int n=N, ndim=2,type=MT_F_DBL,dims[2]={N,N},coord[2];
int me=GA_Nodeid(), nproc=GA_Nnodes();
int row, i, j;
int lo[2], hi[2];
/* Note: on all current platforms DoublePrecision = double */
DoublePrecision buf[N], *max_row=NULL;
MPI_Comm WORLD_COMM;
MPI_Comm ROW_COMM;
int ilo,ihi, jlo,jhi, ld, prow, pcol;
int root=0, grp_me=-1;
WORLD_COMM = GA_MPI_Comm_pgroup_default();
if(me==0)printf("Creating matrix A\n");
dims[0]=n; dims[1]=n;
g_a = NGA_Create(type, ndim, dims, "A", NULL);
if(!g_a) GA_Error("create failed: A",n);
if(me==0)printf("OK\n");
if(me==0)printf("Creating matrix B\n");
dims[0]=n;
g_b = NGA_Create(type, 1, dims, "B", NULL);
if(!g_b) GA_Error("create failed: B",n);
if(me==0)printf("OK\n");
GA_Zero(g_a); /* zero the matrix */
if(me==0)printf("Initializing matrix A\n");
/* fill in matrix A with values: A(i,j) = (i+j) */
for(row=me; row<n; row+= nproc){
/**
* simple load balancing:
* each process works on a different row in MIMD style
*/
for(i=0; i<n; i++) buf[i]=(DoublePrecision)(i+row+1);
lo[0]=hi[0]=row;
lo[1]=ZERO; hi[1]=n-1;
NGA_Put(g_a, lo, hi, buf, &n);
}
/* GA_print(&g_a);*/
NGA_Distribution(g_a, me, lo, hi);
ilo=lo[0]; ihi=hi[0];
jlo=lo[1]; jhi=hi[1];
GA_Sync();
if(ihi-ilo+1 >0){
max_row=(DoublePrecision*)malloc(sizeof(DoublePrecision)*(ihi-ilo+1));
if (!max_row) GA_Error("malloc 3 failed",(ihi-ilo+1));
for (i=0; i<(ihi-ilo+1); i++) {
max_row[i] = 0.0;
}
}
NGA_Proc_topology(g_a, me, coord); /* block coordinates */
prow = coord[0];
pcol = coord[1];
if(me==0)printf("Splitting comm according to distribution of A\n");
/* GA on SP1 requires synchronization before & after message-passing !!*/
GA_Sync();
if(me==0)printf("Computing max row elements\n");
/* create communicator for processes that 'own' A[:,jlo:jhi] */
MPI_Barrier(WORLD_COMM);
if(pcol < 0 || prow <0)
MPI_Comm_split(WORLD_COMM,MPI_UNDEFINED,MPI_UNDEFINED, &ROW_COMM);
else
MPI_Comm_split(WORLD_COMM, (int)pcol, (int)prow, &ROW_COMM);
if(ROW_COMM != MPI_COMM_NULL){
double *ptr;
MPI_Comm_rank(ROW_COMM, &grp_me);
/* each process computes max elements in the block it 'owns' */
lo[0]=ilo; hi[0]=ihi;
lo[1]=jlo; hi[1]=jhi;
NGA_Access(g_a, lo, hi, &ptr, &ld);
for(i=0; i<ihi-ilo+1; i++){
for(j=0; j<jhi-jlo+1; j++)
if(max_row[i] < ptr[i*ld + j]){
max_row[i] = ptr[i*ld + j];
}
}
MPI_Reduce(max_row, buf, ihi-ilo+1, MPI_DOUBLE, MPI_MAX,
root, ROW_COMM);
}else fprintf(stderr,"process %d not participating\n",me);
GA_Sync();
/* processes with rank=root in ROW_COMM put results into g_b */
ld = 1;
if(grp_me == root) {
lo[0]=ilo; hi[0]=ihi;
NGA_Put(g_b, lo, hi, buf, &ld);
}
GA_Sync();
if(me==0)printf("Checking the result\n");
if(me==0){
lo[0]=ZERO; hi[0]=n-1;
NGA_Get(g_b, lo, hi, buf, &n);
for(i=0; i< n; i++)if(buf[i] != (double)n+i){
fprintf(stderr,"error:%d max=%f should be:%d\n",i,buf[i],n+i);
GA_Error("terminating...",1);
}
}
if(me==0)printf("OK\n");
GA_Destroy(g_a);
GA_Destroy(g_b);
}
/*
* test ga_dgemm
* Note: - change nummax for large arrays
* - turn off "dgemm_verify" for large arrays due to memory
* limitations, as dgemm_verify=1 for large arrays produces
* segfault, dumps core,or any crap.
*/
int main(int argc, char **argv)
{
int num_m;
int num_n;
int num_k;
int i;
int ii;
double *h0;
int g_c;
int g_b;
int g_a;
double a;
double t1;
double mf;
double avg_t[ntrans];
double avg_mf[ntrans];
int itime;
int ntimes;
int nums_m[/*howmany*/] = {512,1024};
int nums_n[/*howmany*/] = {512,1024};
int nums_k[/*howmany*/] = {512,1024};
char transa[/*ntrans*/] = "ntnt";
char transb[/*ntrans*/] = "nntt";
char ta;
char tb;
double *tmpa;
double *tmpb;
double *tmpc;
int ndim;
int dims[2];
#ifdef BLOCK_CYCLIC
int block_size[2];
#endif
#if defined(USE_ELEMENTAL)
// initialize Elemental (which will initialize MPI)
ElInitialize( &argc, &argv );
ElMPICommRank( MPI_COMM_WORLD, &me );
ElMPICommSize( MPI_COMM_WORLD, &nproc );
// instantiate el::global array
ElGlobalArraysConstruct_d( &eldga );
// initialize global arrays
ElGlobalArraysInitialize_d( eldga );
#else
MP_INIT(argc,argv);
if (!MA_init(MT_DBL,1,20000000)) {
GA_Error("failed: ma_init(MT_DBL,1,20000000)",10);
}
GA_INIT(argc,argv);
me = GA_Nodeid();
#endif
h0 = (double*)malloc(sizeof(double) * nummax*nummax);
tmpa = (double*)malloc(sizeof(double) * nummax*nummax);
tmpb = (double*)malloc(sizeof(double) * nummax*nummax);
tmpc = (double*)malloc(sizeof(double) * nummax*nummax);
ii = 0;
for (i=0; i<nummax*nummax; i++) {
ii = ii + 1;
if (ii > nummax) {
ii = 0;
}
h0[i] = ii;
}
/* Compute times assuming 500 mflops and 5 second target time */
/* ntimes = max(3.0d0,5.0d0/(4.0d-9*num**3)); */
ntimes = 5;
for (ii=0; ii<howmany; ii++) {
num_m = nums_m[ii];
num_n = nums_n[ii];
num_k = nums_k[ii];
a = 0.5/(num_m*num_n);
if (num_m > nummax || num_n > nummax || num_k > nummax) {
GA_Error("Insufficient memory: check nummax", 1);
}
#ifndef BLOCK_CYCLIC
ndim = 2;
/*
dims[0] = num_m;
dims[1] = num_n;
*/
dims[1] = num_m;
dims[0] = num_n;
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_d( eldga, ndim, dims, "g_c", NULL, &g_c );
#else
if (!((g_c = NGA_Create(MT_DBL,ndim,dims,"g_c",NULL)))) {
GA_Error("failed: create g_c",20);
}
#endif
/*
dims[0] = num_k;
dims[1] = num_n;
*/
dims[1] = num_k;
dims[0] = num_n;
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_d( eldga, ndim, dims, "g_b", NULL, &g_b );
#else
if (!((g_b = NGA_Create(MT_DBL,ndim,dims,"g_b",NULL)))) {
GA_Error("failed: create g_b",30);
}
#endif
/*
dims[0] = num_m;
dims[1] = num_k;
*/
dims[1] = num_m;
dims[0] = num_k;
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_d( eldga, ndim, dims, "g_a", NULL, &g_a );
#else
if (!((g_a = NGA_Create(MT_DBL,ndim,dims,"g_a",NULL)))) {
GA_Error("failed: create g_a",40);
}
#endif
#else
ndim = 2;
block_size[0] = 128;
block_size[1] = 128;
dims[0] = num_m;
dims[1] = num_n;
g_c = GA_Create_handle();
GA_Set_data(g_c,ndim,dims,MT_DBL);
GA_Set_array_name(g_c,"g_c");
GA_Set_block_cyclic(g_c,block_size);
if (!GA_Allocate(g_c)) {
GA_Error("failed: create g_c",40);
}
dims[0] = num_k;
dims[1] = num_n;
g_b = GA_Create_handle();
GA_Set_data(g_b,ndim,dims,MT_DBL);
GA_Set_array_name(g_b,"g_b");
GA_Set_block_cyclic(g_b,block_size);
if (!ga_allocate(g_b)) {
GA_Error("failed: create g_b",40);
}
dims[0] = num_m;
dims[1] = num_k;
g_a = GA_Create_handle();
GA_Set_data(g_a,ndim,dims,MT_DBL);
GA_Set_array_name(g_a,"g_a");
GA_Set_block_cyclic(g_a,block_size);
if (!ga_allocate(g_a)) {
GA_Error('failed: create g_a',40);
}
#endif
/* Initialize matrices A and B */
if (me == 0) {
load_ga(g_a, h0, num_m, num_k);
load_ga(g_b, h0, num_k, num_n);
}
#if defined(USE_ELEMENTAL)
double zero = 0.0;
ElGlobalArraysFill_d( eldga, g_c, &zero );
ElGlobalArraysSync_d( eldga );
#else
GA_Zero(g_c);
GA_Sync();
#endif
#if defined(USE_ELEMENTAL)
if (me == 0) {
#else
if (GA_Nodeid() == 0) {
#endif
printf("\nMatrix Multiplication on C = A[%ld,%ld]xB[%ld,%ld]\n",
(long)num_m, (long)num_k, (long)num_k, (long)num_n);
fflush(stdout);
}
for (i=0; i<ntrans; i++) {
avg_t[i] = 0.0;
avg_mf[i] = 0.0;
}
for (itime=0; itime<ntimes; itime++) {
for (i=0; i<ntrans; i++) {
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_d( eldga );
#else
GA_Sync();
#endif
ta = transa[i];
tb = transb[i];
t1 = MP_TIMER();
#if defined(USE_ELEMENTAL)
ElGlobalArraysDgemm_d( eldga, ta, tb, num_m, num_n, num_k, 1.0, g_a, g_b, 0.0, g_c );
#else
GA_Dgemm(ta,tb,num_m,num_n,num_k,1.0, g_a, g_b, 0.0, g_c);
#endif
t1 = MP_TIMER() - t1;
#if defined(USE_ELEMENTAL)
if (me == 0) {
#else
if (GA_Nodeid() == 0) {
#endif
#if defined(USE_ELEMENTAL)
mf = 2e0*num_m*num_n*num_k/t1*1e-6/nproc;
#else
mf = 2e0*num_m*num_n*num_k/t1*1e-6/GA_Nnodes();
#endif
avg_t[i] = avg_t[i]+t1;
avg_mf[i] = avg_mf[i] + mf;
printf("%15s%2d: %12.4f seconds %12.1f mflops/proc %c %c\n",
"Run#", itime, t1, mf, ta, tb);
fflush(stdout);
if (dgemm_verify && itime == 0) {
/* recall the C API swaps the matrix order */
/* we swap it here for the Fortran-based verify */
verify_ga_dgemm(tb, ta, num_n, num_m, num_k, 1.0,
g_b, g_a, 0.0, g_c, tmpb, tmpa, tmpc);
}
}
}
}
#if defined(USE_ELEMENTAL)
if (me == 0) {
#else
if (GA_Nodeid() == 0) {
#endif
printf("\n");
for (i=0; i<ntrans; i++) {
printf("%17s: %12.4f seconds %12.1f mflops/proc %c %c\n",
"Average", avg_t[i]/ntimes, avg_mf[i]/ntimes,
transa[i], transb[i]);
}
if(dgemm_verify) {
printf("All GA_Dgemms are verified...O.K.\n");
}
fflush(stdout);
}
/*
GA_Print(g_a);
GA_Print(g_b);
GA_Print(g_c);
*/
#if defined(USE_ELEMENTAL)
ElGlobalArraysDestroy_d( eldga, g_a );
ElGlobalArraysDestroy_d( eldga, g_b );
ElGlobalArraysDestroy_d( eldga, g_c );
#else
GA_Destroy(g_c);
GA_Destroy(g_b);
GA_Destroy(g_a);
#endif
}
/* ???
format(a15, i2, ': ', e12.4, ' seconds ',f12.1,
. ' mflops/proc ', 3a2)
*/
#if defined(USE_ELEMENTAL)
if (me == 0) {
#else
if (GA_Nodeid() == 0) {
#endif
printf("All tests successful\n");
}
free(h0);
free(tmpa);
free(tmpb);
free(tmpc);
#if defined(USE_ELEMENTAL)
// call el::global arrays destructor
ElGlobalArraysTerminate_d( eldga );
ElGlobalArraysDestruct_d( eldga );
ElFinalize();
#else
GA_Terminate();
MP_FINALIZE();
#endif
return 0;
}
/*
* Verify for correctness. Process 0 computes BLAS dgemm
* locally. For larger arrays, disbale this test as memory
* might not be sufficient
*/
void verify_ga_dgemm(char xt1, char xt2, int num_m, int num_n, int num_k,
double alpha, int g_a, int g_b, double beta, int g_c,
double *tmpa, double *tmpb, double *tmpc)
{
int i,j,type,ndim,dims[2],lo[2],hi[2];
double abs_value;
for (i=0; i<num_n; i++) {
for (j=0; j<num_m; j++) {
tmpc[j+i*num_m] = -1.0;
tmpa[j+i*num_m] = -2.0;
}
}
#if defined(USE_ELEMENTAL)
ElGlobalArraysInquire_d( eldga, g_a, &ndim, dims );
#else
NGA_Inquire(g_a, &type, &ndim, dims);
#endif
lo[0] = 0;
lo[1] = 0;
hi[0] = dims[0]-1;
hi[1] = dims[1]-1;
#if defined(USE_ELEMENTAL)
ElGlobalArraysGet_d( eldga, g_a, lo, hi, tmpa, &dims[1] );
#else
NGA_Get(g_a, lo, hi, tmpa, &dims[1]);
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysInquire_d( eldga, g_a, &ndim, dims );
#else
NGA_Inquire(g_a, &type, &ndim, dims);
#endif
lo[0] = 0;
lo[1] = 0;
hi[0] = dims[0]-1;
hi[1] = dims[1]-1;
#if defined(USE_ELEMENTAL)
ElGlobalArraysGet_d( eldga, g_b, lo, hi, tmpb, &dims[1] );
#else
NGA_Get(g_b, lo, hi, tmpb, &dims[1]);
#endif
/* compute dgemm sequentially */
#if defined(USE_ELEMENTAL)
cblas_dgemm ( CblasRowMajor, ( xt1 == 'n'? CblasNoTrans: CblasTrans ),
( xt2 == 'n'? CblasNoTrans: CblasTrans ),
num_m /* M */, num_n /* N */, num_k /* K */,
alpha, tmpa, num_m, /* lda */
tmpb, num_k, /* ldb */ beta,
tmpc, num_m /* ldc */);
#else
xb_dgemm(&xt1, &xt2, &num_m, &num_n, &num_k,
&alpha, tmpa, &num_m,
tmpb, &num_k, &beta,
tmpc, &num_m);
#endif
/* after computing c locally, verify it with the values in g_c */
#if defined(USE_ELEMENTAL)
ElGlobalArraysInquire_d( eldga, g_a, &ndim, dims );
#else
NGA_Inquire(g_a, &type, &ndim, dims);
#endif
lo[0] = 0;
lo[1] = 0;
hi[0] = dims[0]-1;
hi[1] = dims[1]-1;
#if defined(USE_ELEMENTAL)
ElGlobalArraysGet_d( eldga, g_c, lo, hi, tmpa, &dims[1] );
#else
NGA_Get(g_c, lo, hi, tmpa, &dims[1]);
#endif
for (i=0; i<num_n; i++) {
for (j=0; j<num_m; j++) {
abs_value = fabs(tmpc[j+i*num_m]-tmpa[j+i*num_m]);
if(abs_value > 1.0 || abs_value < -1.0) {
printf("Values are = %f %f\n",
tmpc[j+i*num_m], tmpa[j+i*num_m]);
printf("Values are = %f %f\n",
fabs(tmpc[j+i*num_m]-tmpa[j*i*num_m]), abs_value);
fflush(stdout);
GA_Error("verify ga_dgemm failed", 1);
}
}
}
}
/**
* called by process '0' (or your master process )
*/
void load_ga(int handle, double *f, int dim1, int dim2)
{
int lo[2], hi[2];
if (dim1 < 0 || dim2 < 0) {
return;
}
lo[0] = 0;
lo[1] = 0;
hi[0] = dim1-1;
hi[1] = dim2-1;
#if defined(USE_ELEMENTAL)
ElGlobalArraysPut_d( eldga, handle, lo, hi, f, &dim1 );
#else
NGA_Put(handle, lo, hi, f, &dim1);
#endif
}
int main(int argc, char **argv)
{
int me;
int nproc;
int status;
int g_a;
int dims[NDIM];
int chunk[NDIM];
int pg_world;
size_t num = 10;
double *p1 = NULL;
double *p2 = NULL;
size_t i;
int num_mutex;
int lo[1];
int hi[1];
int ld[1]={1};
MPI_Comm comm;
MP_INIT(argc,argv);
GA_INIT(argc,argv);
me = GA_Nodeid();
nproc = GA_Nnodes();
comm = GA_MPI_Comm_pgroup_default();
printf("%d: Hello world!\n",me);
if (me==0) printf("%d: GA_Initialize\n",me);
/*if (me==0) printf("%d: ARMCI_Init\n",me);*/
/*ARMCI_Init();*/
/*if (me==0) printf("%d: MA_Init\n",me);*/
/*MA_init(MT_DBL, 8*1024*1024, 2*1024*1024);*/
if (me==0) printf("%d: GA_Create_handle\n",me);
g_a = GA_Create_handle();
if (me==0) printf("%d: GA_Set_array_name\n",me);
GA_Set_array_name(g_a,"test array A");
dims[0] = 30;
if (me==0) printf("%d: GA_Set_data\n",me);
GA_Set_data(g_a,NDIM,dims,MT_DBL);
chunk[0] = -1;
if (me==0) printf("%d: GA_Set_chunk\n",me);
GA_Set_chunk(g_a,chunk);
if (me==0) printf("%d: GA_Pgroup_get_world\n",me);
pg_world = GA_Pgroup_get_world();
if (me==0) printf("%d: GA_Set_pgroup\n",me);
GA_Set_pgroup(g_a,pg_world);
if (me==0) printf("%d: GA_Allocate\n",me);
status = GA_Allocate(g_a);
if(0 == status) MPI_Abort(comm,100);
if (me==0) printf("%d: GA_Zero\n",me);
GA_Zero(g_a);
if (me==0) printf("%d: GA_Sync\n",me);
GA_Sync();
num = 10;
p1 = malloc(num*sizeof(double));
/*double* p1 = ARMCI_Malloc_local(num*sizeof(double));*/
if (p1==NULL) MPI_Abort(comm,1000);
p2 = malloc(num*sizeof(double));
/*double* p2 = ARMCI_Malloc_local(num*sizeof(double));*/
if (p2==NULL) MPI_Abort(comm,2000);
for ( i=0 ; i<num ; i++ ) p1[i] = 7.0;
for ( i=0 ; i<num ; i++ ) p2[i] = 3.0;
num_mutex = 17;
status = GA_Create_mutexes(num_mutex);
if (me==0) printf("%d: GA_Create_mutexes = %d\n",me,status);
/***************************************************************/
if (me==0) {
printf("%d: before GA_Lock\n",me);
GA_Lock(0);
lo[0] = 0;
hi[0] = num-1;
GA_Init_fence();
NGA_Put(g_a,lo,hi,p1,ld);
GA_Fence();
GA_Unlock(0);
printf("%d: after GA_Unlock\n",me);
}
GA_Print(g_a);
if (me==1) {
printf("%d: before GA_Lock\n",me);
GA_Lock(0);
lo[0] = 0;
hi[0] = num-1;
GA_Init_fence();
NGA_Get(g_a,lo,hi,p2,ld);
GA_Fence();
GA_Unlock(0);
printf("%d: after GA_Unlock\n",me);
for ( i=0 ; i<num ; i++ ) printf("p2[%2lu] = %20.10f\n",
(long unsigned)i,p2[i]);
}
/***************************************************************/
status = GA_Destroy_mutexes();
if (me==0) printf("%d: GA_Destroy_mutexes = %d\n",me,status);
/*ARMCI_Free(p2);*/
/*ARMCI_Free(p1);*/
free(p2);
free(p1);
if (me==0) printf("%d: GA_Destroy\n",me);
GA_Destroy(g_a);
/*if (me==0) printf("%d: ARMCI_Finalize\n",me);*/
/*ARMCI_Finalize();*/
if (me==0) printf("%d: GA_Terminate\n",me);
GA_Terminate();
if (me==0) printf("%d: MPI_Finalize\n",me);
MPI_Finalize();
return(0);
}
void
test(int data_type) {
int me=GA_Nodeid();
int nproc = GA_Nnodes();
int g_a, g_b, g_c;
int ndim = 2;
int dims[2]={N,N};
int lo[2]={0,0};
int hi[2]={N-1,N-1};
int block_size[2]={NB,NB-1};
int proc_grid[2];
int i,j,l,k,m,n, ld;
double alpha_dbl = 1.0, beta_dbl = 0.0;
double dzero = 0.0;
double ddiff;
float alpha_flt = 1.0, beta_flt = 0.0;
float fzero = 0.0;
float fdiff;
float ftmp;
double dtmp;
SingleComplex ctmp;
DoubleComplex ztmp;
DoubleComplex alpha_dcpl = {1.0, 0.0} , beta_dcpl = {0.0, 0.0};
DoubleComplex zzero = {0.0,0.0};
DoubleComplex zdiff;
SingleComplex alpha_scpl = {1.0, 0.0} , beta_scpl = {0.0, 0.0};
SingleComplex czero = {0.0,0.0};
SingleComplex cdiff;
void *alpha=NULL, *beta=NULL;
void *abuf=NULL, *bbuf=NULL, *cbuf=NULL, *c_ptr=NULL;
switch (data_type) {
case C_FLOAT:
alpha = (void *)&alpha_flt;
beta = (void *)&beta_flt;
abuf = (void*)malloc(N*N*sizeof(float));
bbuf = (void*)malloc(N*N*sizeof(float));
cbuf = (void*)malloc(N*N*sizeof(float));
if(me==0) printf("Single Precision: Testing GA_Sgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
case C_DBL:
alpha = (void *)&alpha_dbl;
beta = (void *)&beta_dbl;
abuf = (void*)malloc(N*N*sizeof(double));
bbuf = (void*)malloc(N*N*sizeof(double));
cbuf = (void*)malloc(N*N*sizeof(double));
if(me==0) printf("Double Precision: Testing GA_Dgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
case C_DCPL:
alpha = (void *)&alpha_dcpl;
beta = (void *)&beta_dcpl;
abuf = (void*)malloc(N*N*sizeof(DoubleComplex));
bbuf = (void*)malloc(N*N*sizeof(DoubleComplex));
cbuf = (void*)malloc(N*N*sizeof(DoubleComplex));
if(me==0) printf("Double Complex: Testing GA_Zgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
case C_SCPL:
alpha = (void *)&alpha_scpl;
beta = (void *)&beta_scpl;
abuf = (void*)malloc(N*N*sizeof(SingleComplex));
bbuf = (void*)malloc(N*N*sizeof(SingleComplex));
cbuf = (void*)malloc(N*N*sizeof(SingleComplex));
if(me==0) printf("Single Complex: Testing GA_Cgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
default:
GA_Error("wrong data type", data_type);
}
if (me==0) printf("\nCreate A, B, C\n");
#ifdef USE_REGULAR
g_a = NGA_Create(data_type, ndim, dims, "array A", NULL);
#endif
#ifdef USE_SIMPLE_CYCLIC
g_a = NGA_Create_handle();
NGA_Set_data(g_a,ndim,dims,data_type);
NGA_Set_array_name(g_a,"array A");
NGA_Set_block_cyclic(g_a,block_size);
if (!GA_Allocate(g_a)) {
GA_Error("Failed: create: g_a",40);
}
#endif
#ifdef USE_SCALAPACK
g_a = NGA_Create_handle();
NGA_Set_data(g_a,ndim,dims,data_type);
NGA_Set_array_name(g_a,"array A");
grid_factor(nproc,&i,&j);
proc_grid[0] = i;
proc_grid[1] = j;
NGA_Set_block_cyclic_proc_grid(g_a,block_size,proc_grid);
if (!GA_Allocate(g_a)) {
GA_Error("Failed: create: g_a",40);
}
#endif
#ifdef USE_TILED
g_a = NGA_Create_handle();
NGA_Set_data(g_a,ndim,dims,data_type);
NGA_Set_array_name(g_a,"array A");
grid_factor(nproc,&i,&j);
proc_grid[0] = i;
proc_grid[1] = j;
NGA_Set_tiled_proc_grid(g_a,block_size,proc_grid);
if (!GA_Allocate(g_a)) {
GA_Error("Failed: create: g_a",40);
}
#endif
g_b = GA_Duplicate(g_a, "array B");
g_c = GA_Duplicate(g_a, "array C");
if(!g_a || !g_b || !g_c) GA_Error("Create failed: a, b or c",1);
ld = N;
if (me==0) printf("\nInitialize A\n");
/* Set up matrix A */
if (me == 0) {
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
((float*)abuf)[i*N+j] = (float)(i*N+j);
break;
case C_DBL:
((double*)abuf)[i*N+j] = (double)(i*N+j);
break;
case C_DCPL:
((DoubleComplex*)abuf)[i*N+j].real = (double)(i*N+j);
((DoubleComplex*)abuf)[i*N+j].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)abuf)[i*N+j].real = (float)(i*N+j);
((SingleComplex*)abuf)[i*N+j].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_a,lo,hi,abuf,&ld);
}
GA_Sync();
if (me==0) printf("\nInitialize B\n");
/* Set up matrix B */
if (me == 0) {
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
((float*)bbuf)[i*N+j] = (float)(j*N+i);
break;
case C_DBL:
((double*)bbuf)[i*N+j] = (double)(j*N+i);
break;
case C_DCPL:
((DoubleComplex*)bbuf)[i*N+j].real = (double)(j*N+i);
((DoubleComplex*)bbuf)[i*N+j].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)bbuf)[i*N+j].real = (float)(j*N+i);
((SingleComplex*)bbuf)[i*N+j].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_b,lo,hi,bbuf,&ld);
}
GA_Sync();
if (me==0) printf("\nPerform matrix multiply\n");
switch (data_type) {
case C_FLOAT:
NGA_Matmul_patch('N','N',&alpha_flt,&beta_flt,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DBL:
NGA_Matmul_patch('N','N',&alpha_dbl,&beta_dbl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_SCPL:
NGA_Matmul_patch('N','N',&alpha_scpl,&beta_scpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DCPL:
NGA_Matmul_patch('N','N',&alpha_dcpl,&beta_dcpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
default:
GA_Error("wrong data type", data_type);
}
GA_Sync();
#if 0
if (me==0) printf("\nCheck answer\n");
/*
GA_Print(g_a);
if (me == 0) printf("\n\n\n\n");
GA_Print(g_b);
if (me == 0) printf("\n\n\n\n");
GA_Print(g_c);
*/
/* Check answer */
NGA_Get(g_a,lo,hi,abuf,&ld);
NGA_Get(g_b,lo,hi,bbuf,&ld);
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N+j] = fzero;
break;
case C_DBL:
((double*)cbuf)[i*N+j] = dzero;
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N+j] = zzero;
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N+j] = czero;
break;
default:
GA_Error("wrong data type", data_type);
}
for (k=0; k<N; k++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N+j] += ((float*)abuf)[i*N+k]
*((float*)bbuf)[k*N+j];
break;
case C_DBL:
((double*)cbuf)[i*N+j] += ((double*)abuf)[i*N+k]
*((double*)bbuf)[k*N+j];
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N+j].real +=
(((DoubleComplex*)abuf)[i*N+k].real
*((DoubleComplex*)bbuf)[k*N+j].real
-(((DoubleComplex*)abuf)[i*N+k].imag
*((DoubleComplex*)bbuf)[k*N+j].imag));
((DoubleComplex*)cbuf)[i*N+j].imag +=
(((DoubleComplex*)abuf)[i*N+k].real
*((DoubleComplex*)bbuf)[k*N+j].imag
+(((DoubleComplex*)abuf)[i*N+k].imag
*((DoubleComplex*)bbuf)[k*N+j].real));
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N+j].real +=
(((SingleComplex*)abuf)[i*N+k].real
*((SingleComplex*)bbuf)[k*N+j].real
-(((SingleComplex*)abuf)[i*N+k].imag
*((SingleComplex*)bbuf)[k*N+j].imag));
((SingleComplex*)cbuf)[i*N+j].imag +=
(((SingleComplex*)abuf)[i*N+k].real
*((SingleComplex*)bbuf)[k*N+j].imag
+(((SingleComplex*)abuf)[i*N+k].imag
*((SingleComplex*)bbuf)[k*N+j].real));
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
}
GA_Sync();
if (me == 0) {
NGA_Get(g_c,lo,hi,abuf,&ld);
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
fdiff = ((float*)abuf)[i*N+j]-((float*)cbuf)[i*N+j];
if (((float*)abuf)[i*N+j] != 0.0) {
fdiff /= ((float*)abuf)[i*N+j];
}
if (fabs(fdiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: %f Expected: %f\n",me,i,j,
((float*)abuf)[i*N+j],((float*)cbuf)[i*N+j]);
}
break;
case C_DBL:
ddiff = ((double*)abuf)[i*N+j]-((double*)cbuf)[i*N+j];
if (((double*)abuf)[i*N+j] != 0.0) {
ddiff /= ((double*)abuf)[i*N+j];
}
if (fabs(ddiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: %f Expected: %f\n",me,i,j,
((double*)abuf)[i*N+j],((double*)cbuf)[i*N+j]);
}
break;
case C_DCPL:
zdiff.real = ((DoubleComplex*)abuf)[i*N+j].real
-((DoubleComplex*)cbuf)[i*N+j].real;
zdiff.imag = ((DoubleComplex*)abuf)[i*N+j].imag
-((DoubleComplex*)cbuf)[i*N+j].imag;
if (((DoubleComplex*)abuf)[i*N+j].real != 0.0 ||
((DoubleComplex*)abuf)[i*N+j].imag != 0.0) {
ztmp = ((DoubleComplex*)abuf)[i*N+j];
ddiff = sqrt((zdiff.real*zdiff.real+zdiff.imag*zdiff.imag)
/(ztmp.real*ztmp.real+ztmp.imag*ztmp.imag));
} else {
ddiff = sqrt(zdiff.real*zdiff.real+zdiff.imag*zdiff.imag);
}
if (fabs(ddiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: (%f,%f) Expected: (%f,%f)\n",me,i,j,
((DoubleComplex*)abuf)[i*N+j].real,
((DoubleComplex*)abuf)[i*N+j].imag,
((DoubleComplex*)cbuf)[i*N+j].real,
((DoubleComplex*)cbuf)[i*N+j].imag);
}
break;
case C_SCPL:
cdiff.real = ((SingleComplex*)abuf)[i*N+j].real
-((SingleComplex*)cbuf)[i*N+j].real;
cdiff.imag = ((SingleComplex*)abuf)[i*N+j].imag
-((SingleComplex*)cbuf)[i*N+j].imag;
if (((SingleComplex*)abuf)[i*N+j].real != 0.0 ||
((SingleComplex*)abuf)[i*N+j].imag != 0.0) {
ctmp = ((SingleComplex*)abuf)[i*N+j];
fdiff = sqrt((cdiff.real*cdiff.real+cdiff.imag*cdiff.imag)
/(ctmp.real*ctmp.real+ctmp.imag*ctmp.imag));
} else {
fdiff = sqrt(cdiff.real*cdiff.real+cdiff.imag*cdiff.imag);
}
if (fabs(fdiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: (%f,%f) Expected: (%f,%f)\n",me,i,j,
((SingleComplex*)abuf)[i*N+j].real,
((SingleComplex*)abuf)[i*N+j].imag,
((SingleComplex*)cbuf)[i*N+j].real,
((SingleComplex*)cbuf)[i*N+j].imag);
}
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
}
GA_Sync();
/* copy cbuf back to g_a */
if (me == 0) {
NGA_Put(g_a,lo,hi,cbuf,&ld);
}
GA_Sync();
/* Get norm of g_a */
switch (data_type) {
case C_FLOAT:
ftmp = GA_Fdot(g_a,g_a);
break;
case C_DBL:
dtmp = GA_Ddot(g_a,g_a);
break;
case C_DCPL:
ztmp = GA_Zdot(g_a,g_a);
break;
case C_SCPL:
ctmp = GA_Cdot(g_a,g_a);
break;
default:
GA_Error("wrong data type", data_type);
}
/* subtract C from A and put the results in B */
beta_flt = -1.0;
beta_dbl = -1.0;
beta_scpl.real = -1.0;
beta_dcpl.real = -1.0;
GA_Zero(g_b);
GA_Add(alpha,g_a,beta,g_c,g_b);
/* evaluate the norm of the difference between the two matrices */
switch (data_type) {
case C_FLOAT:
fdiff = GA_Fdot(g_b, g_b);
if (ftmp != 0.0) {
fdiff /= ftmp;
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(fdiff), TOLERANCE);
GA_Error("GA_Sgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Sgemm OK\n\n");
}
break;
case C_DBL:
ddiff = GA_Ddot(g_b, g_b);
if (dtmp != 0.0) {
ddiff /= dtmp;
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(ddiff), TOLERANCE);
GA_Error("GA_Dgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Dgemm OK\n\n");
}
break;
case C_DCPL:
zdiff = GA_Zdot(g_b, g_b);
if (ztmp.real != 0.0 || ztmp.imag != 0.0) {
ddiff = sqrt((zdiff.real*zdiff.real+zdiff.imag*zdiff.imag)
/(ztmp.real*ztmp.real+ztmp.imag*ztmp.imag));
} else {
ddiff = sqrt(zdiff.real*zdiff.real+zdiff.imag*zdiff.imag);
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(zdiff.real), TOLERANCE);
GA_Error("GA_Zgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Zgemm OK\n\n");
}
break;
case C_SCPL:
cdiff = GA_Cdot(g_b, g_b);
if (ctmp.real != 0.0 || ctmp.imag != 0.0) {
fdiff = sqrt((cdiff.real*cdiff.real+cdiff.imag*cdiff.imag)
/(ctmp.real*ctmp.real+ctmp.imag*ctmp.imag));
} else {
fdiff = sqrt(cdiff.real*cdiff.real+cdiff.imag*cdiff.imag);
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(cdiff.real), TOLERANCE);
GA_Error("GA_Cgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Cgemm OK\n\n");
}
break;
default:
GA_Error("wrong data type", data_type);
}
#endif
free(abuf);
free(bbuf);
free(cbuf);
switch (data_type) {
case C_FLOAT:
abuf = (void*)malloc(N*N*sizeof(float)/4);
bbuf = (void*)malloc(N*N*sizeof(float)/4);
cbuf = (void*)malloc(N*N*sizeof(float)/4);
break;
case C_DBL:
abuf = (void*)malloc(N*N*sizeof(double)/4);
bbuf = (void*)malloc(N*N*sizeof(double)/4);
cbuf = (void*)malloc(N*N*sizeof(double)/4);
break;
case C_DCPL:
abuf = (void*)malloc(N*N*sizeof(DoubleComplex)/4);
bbuf = (void*)malloc(N*N*sizeof(DoubleComplex)/4);
cbuf = (void*)malloc(N*N*sizeof(DoubleComplex)/4);
break;
case C_SCPL:
abuf = (void*)malloc(N*N*sizeof(SingleComplex)/4);
bbuf = (void*)malloc(N*N*sizeof(SingleComplex)/4);
cbuf = (void*)malloc(N*N*sizeof(SingleComplex)/4);
break;
default:
GA_Error("wrong data type", data_type);
}
/* Test multiply on a fraction of matrix. Start by reinitializing
* A and B */
GA_Zero(g_a);
GA_Zero(g_b);
GA_Zero(g_c);
if (me==0) printf("\nTest patch multiply\n");
lo[0] = N/4;
lo[1] = N/4;
hi[0] = 3*N/4-1;
hi[1] = 3*N/4-1;
ld = N/2;
/* Set up matrix A */
if (me==0) printf("\nInitialize A\n");
if (me == 0) {
for (i=N/4; i<3*N/4; i++) {
for (j=N/4; j<3*N/4; j++) {
switch (data_type) {
case C_FLOAT:
((float*)abuf)[(i-N/4)*N/2+(j-N/4)] = (float)(i*N+j);
break;
case C_DBL:
((double*)abuf)[(i-N/4)*N/2+(j-N/4)] = (double)(i*N+j);
break;
case C_DCPL:
((DoubleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].real = (double)(i*N+j);
((DoubleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].real = (float)(i*N+j);
((SingleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_a,lo,hi,abuf,&ld);
}
GA_Sync();
if (me==0) printf("\nInitialize B\n");
/* Set up matrix B */
if (me == 0) {
for (i=N/4; i<3*N/4; i++) {
for (j=N/4; j<3*N/4; j++) {
switch (data_type) {
case C_FLOAT:
((float*)bbuf)[(i-N/4)*N/2+(j-N/4)] = (float)(j*N+i);
break;
case C_DBL:
((double*)bbuf)[(i-N/4)*N/2+(j-N/4)] = (double)(j*N+i);
break;
case C_DCPL:
((DoubleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].real = (double)(j*N+i);
((DoubleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].real = (float)(j*N+i);
((SingleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_b,lo,hi,bbuf,&ld);
}
GA_Sync();
beta_flt = 0.0;
beta_dbl = 0.0;
beta_scpl.real = 0.0;
beta_dcpl.real = 0.0;
if (me==0) printf("\nPerform matrix multiply on sub-blocks\n");
switch (data_type) {
case C_FLOAT:
NGA_Matmul_patch('N','N',&alpha_flt,&beta_flt,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DBL:
NGA_Matmul_patch('N','N',&alpha_dbl,&beta_dbl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_SCPL:
NGA_Matmul_patch('N','N',&alpha_scpl,&beta_scpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DCPL:
NGA_Matmul_patch('N','N',&alpha_dcpl,&beta_dcpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
default:
GA_Error("wrong data type", data_type);
}
GA_Sync();
#if 0
if (0) {
/*
if (data_type != C_SCPL && data_type != C_DCPL) {
*/
if (me==0) printf("\nCheck answer\n");
/* Multiply buffers by hand */
if (me == 0) {
for (i=0; i<N/2; i++) {
for (j=0; j<N/2; j++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N/2+j] = fzero;
break;
case C_DBL:
((double*)cbuf)[i*N/2+j] = dzero;
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N/2+j] = zzero;
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N/2+j] = czero;
break;
default:
GA_Error("wrong data type", data_type);
}
for (k=0; k<N/2; k++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N/2+j] += ((float*)abuf)[i*N/2+k]
*((float*)bbuf)[k*N/2+j];
break;
case C_DBL:
((double*)cbuf)[i*N/2+j] += ((double*)abuf)[i*N/2+k]
*((double*)bbuf)[k*N/2+j];
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N/2+j].real +=
(((DoubleComplex*)abuf)[i*N/2+k].real
*((DoubleComplex*)bbuf)[k*N/2+j].real
-(((DoubleComplex*)abuf)[i*N/2+k].imag
*((DoubleComplex*)bbuf)[k*N/2+j].imag));
((DoubleComplex*)cbuf)[i*N/2+j].imag +=
(((DoubleComplex*)abuf)[i*N/2+k].real
*((DoubleComplex*)bbuf)[k*N/2+j].imag
+(((DoubleComplex*)abuf)[i*N/2+k].imag
*((DoubleComplex*)bbuf)[k*N/2+j].real));
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N/2+j].real +=
(((SingleComplex*)abuf)[i*N/2+k].real
*((SingleComplex*)bbuf)[k*N/2+j].real
-(((SingleComplex*)abuf)[i*N/2+k].imag
*((SingleComplex*)bbuf)[k*N/2+j].imag));
((SingleComplex*)cbuf)[i*N/2+j].imag +=
(((SingleComplex*)abuf)[i*N/2+k].real
*((SingleComplex*)bbuf)[k*N/2+j].imag
+(((SingleComplex*)abuf)[i*N/2+k].imag
*((SingleComplex*)bbuf)[k*N/2+j].real));
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
}
NGA_Put(g_a,lo,hi,cbuf,&ld);
}
if (me == 0) printf("\n\n\n\n");
/* Get norm of g_a */
switch (data_type) {
case C_FLOAT:
ftmp = NGA_Fdot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
case C_DBL:
dtmp = NGA_Ddot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
case C_DCPL:
ztmp = NGA_Zdot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
case C_SCPL:
ctmp = NGA_Cdot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
default:
GA_Error("wrong data type", data_type);
}
/* subtract C from A and put the results in B */
beta_flt = -1.0;
beta_dbl = -1.0;
beta_scpl.real = -1.0;
beta_dcpl.real = -1.0;
NGA_Zero_patch(g_b,lo,hi);
NGA_Add_patch(alpha,g_a,lo,hi,beta,g_c,lo,hi,g_b,lo,hi);
/* evaluate the norm of the difference between the two matrices */
switch (data_type) {
case C_FLOAT:
fdiff = NGA_Fdot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (ftmp != 0.0) {
fdiff /= ftmp;
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(fdiff), TOLERANCE);
GA_Error("GA_Sgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Sgemm OK\n\n");
}
break;
case C_DBL:
ddiff = NGA_Ddot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (dtmp != 0.0) {
ddiff /= dtmp;
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(ddiff), TOLERANCE);
GA_Error("GA_Dgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Dgemm OK\n\n");
}
break;
case C_DCPL:
zdiff = NGA_Zdot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (ztmp.real != 0.0 || ztmp.imag != 0.0) {
ddiff = sqrt((zdiff.real*zdiff.real+zdiff.imag*zdiff.imag)
/(ztmp.real*ztmp.real+ztmp.imag*ztmp.imag));
} else {
ddiff = sqrt(zdiff.real*zdiff.real+zdiff.imag*zdiff.imag);
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(zdiff.real), TOLERANCE);
GA_Error("GA_Zgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Zgemm OK\n\n");
}
break;
case C_SCPL:
cdiff = NGA_Cdot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (ctmp.real != 0.0 || ctmp.imag != 0.0) {
fdiff = sqrt((cdiff.real*cdiff.real+cdiff.imag*cdiff.imag)
/(ctmp.real*ctmp.real+ctmp.imag*ctmp.imag));
} else {
fdiff = sqrt(cdiff.real*cdiff.real+cdiff.imag*cdiff.imag);
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(cdiff.real), TOLERANCE);
GA_Error("GA_Cgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Cgemm OK\n\n");
}
break;
default:
GA_Error("wrong data type", data_type);
}
}
#endif
free(abuf);
free(bbuf);
free(cbuf);
GA_Destroy(g_a);
GA_Destroy(g_b);
GA_Destroy(g_c);
}
int main(int argc, char **argv)
{
int rank, nprocs;
int g_A;
int *local_A=NULL, *local_B=NULL, *output_A=NULL;
int dims[DIM]={SIZE,SIZE}, dims2[DIM], lo[DIM]={SIZE-SIZE,SIZE-SIZE}, hi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
int value=SIZE;
//int value=0;
#if defined(USE_ELEMENTAL)
// initialize Elemental (which will initialize MPI)
ElInitialize( &argc, &argv );
ElMPICommRank( MPI_COMM_WORLD, &rank );
ElMPICommSize( MPI_COMM_WORLD, &nprocs );
// instantiate el::global array
ElGlobalArraysConstruct_i( &eliga );
// initialize global arrays
ElGlobalArraysInitialize_i( eliga );
#else
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
#endif
local_A=(int*)malloc(SIZE*SIZE*sizeof(int));
output_A=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (output_A, 0, SIZE*SIZE*sizeof(int));
for(int j=0; j<SIZE; j++)
for(int i=0; i<SIZE; i++) local_A[i+j*ld]=(i + j);
local_B=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (local_B, 0, SIZE*SIZE*sizeof(int));
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_i( eliga, DIM, dims, "array_A", NULL, NULL, &g_A );
ElGlobalArraysFill_i( eliga, g_A, &value );
ElGlobalArraysPrint_i( eliga, g_A );
// acc data
ElGlobalArraysPut_i( eliga, g_A, lo, hi, local_A, &ld );
ElGlobalArraysSync_i( eliga );
// get
ElGlobalArraysGet_i( eliga, g_A, lo, hi, local_B, &ld );
ElGlobalArraysSync_i( eliga );
ElGlobalArraysPrint_i( eliga, g_A );
#else
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
GA_Fill(g_A, &value);
GA_Print(g_A);
NGA_Put(g_A, lo, hi, local_A, &ld);
GA_Sync();
NGA_Get(g_A, lo, hi, local_B, &ld);
GA_Sync();
GA_Print(g_A);
#endif
// updated output
MPI_Reduce (local_A, output_A, SIZE*SIZE, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD);
if(rank==0)
{
printf(" Original local buffer to be accumulated: \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_A[i*ld+j]);
printf("\n");
}
printf("\n");
printf(" Get returns: \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_B[i*ld + j]);
printf("\n");
}
printf("\n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
{
if(local_B[i*ld+j]!=output_A[i*ld+j])
GA_Error("ERROR", -99);
}
}
}
#if defined(USE_ELEMENTAL)
ElGlobalArraysDestroy_i( eliga, g_A );
#else
GA_Destroy(g_A);
#endif
if(rank == 0)
printf ("OK. Test passed\n");
free (local_A);
free (local_B);
free (output_A);
#if defined(USE_ELEMENTAL)
ElGlobalArraysTerminate_i( eliga );
// call el::global arrays destructor
ElGlobalArraysDestruct_i( eliga );
ElFinalize();
#else
GA_Terminate();
MPI_Finalize();
#endif
}
void matrix_multiply() {
int dims[NDIM], chunk[NDIM], ld[NDIM];
int lo[NDIM], hi[NDIM], lo1[NDIM], hi1[NDIM];
int lo2[NDIM], hi2[NDIM], lo3[NDIM], hi3[NDIM];
int g_a, g_b, g_c, i, j, k, l;
int me, nprocs;
/* Find local processor ID and the number of processors */
/* ### assign processor ID to the int variable "me" and the total number
* ### of processors to the int variable "nprocs" */
me = GA_Nodeid();
nprocs = GA_Nnodes();
/* Configure array dimensions. Force an unequal data distribution */
for(i=0; i<NDIM; i++) {
dims[i] = TOTALELEMS;
ld[i] = dims[i];
chunk[i] = TOTALELEMS/nprocs-1; /*minimum block size on each process*/
}
/* create a global array g_a and duplicate it to get g_b and g_c*/
/* ### create GA of doubles with dimension "NDIM" and size "dims" with
* ### minimum block size "chunk" and assign the handle to the
* ### integer variable "g_a". Then create remaining global arrays,
* ### assigned to the integer handle "g_b" and "g_c" by duplicating
* ### g_a. Assign the names "Array A", "Array B" and "Array C" to
* ### "g_a", "g_b", and "g_c". */
g_a=NGA_Create(C_DBL, NDIM, dims, "array A", chunk);
if (!g_a) GA_Error("create failed: A", NDIM);
if (me==0) printf(" Created Array A\n");
g_b=GA_Duplicate(g_a,"array B");
g_c=GA_Duplicate(g_a,"array C");
if (!g_b || !g_c) GA_Error("duplicate failed",NDIM);
if (me==0) printf(" Created Arrays B and C\n");
/* initialize data in matrices a and b */
if(me==0)printf(" Initializing matrix A and B\n");
k = 0; l = 7;
for(i=0; i<dims[0]; i++) {
for(j=0; j<dims[1]; j++) {
a[i][j] = (double)(++k%29);
b[i][j] = (double)(++l%37);
}
}
/* Copy data to global arrays g_a and g_b */
lo1[0] = 0;
lo1[1] = 0;
hi1[0] = dims[0]-1;
hi1[1] = dims[1]-1;
if (me==0) {
/* ### copy the contents of array "a" into the portion of global array
* ### "g_a" described by "lo1" and "hi1". Similarly, copy the contents
* ### of the array "b" into corresponding portion of global array "g_b".
* ### Use the array of strides "ld" to describe the physical layout of
* ### arrays "a" and "b". */
NGA_Put(g_a,lo1,hi1,a,ld);
NGA_Put(g_b,lo1,hi1,b,ld);
}
/* Synchronize all processors to make sure everyone has data */
/* ### synchronize all processors */
GA_Sync();
/* Determine which block of data is locally owned. Note that
the same block is locally owned for all GAs. */
/* ### find out which block of data my node ("me") owns for the global
* ### array "g_c" and store the contents in the integer arrays "lo" and
* ### "hi". */
NGA_Distribution(g_c,me,lo,hi);
/* Get the blocks from g_a and g_b needed to compute this block in
g_c and copy them into the local buffers a and b. */
lo2[0] = lo[0];
lo2[1] = 0;
hi2[0] = hi[0];
hi2[1] = dims[0]-1;
/* ### copy the block of data described by the arrays "lo2" and "hi2" from
* ### the global array "g_a" into the local array "a". Use the array of
* ### strides "ld" to describe the physical layout of "a". */
NGA_Get(g_a,lo2,hi2,a,ld);
lo3[0] = 0;
lo3[1] = lo[1];
hi3[0] = dims[1]-1;
hi3[1] = hi[1];
/* ### copy the block of data described by the arrays "lo3" and "hi3" from
* ### the global array "g_b" into the local array "b". Use the array of
* ### strides "ld" to describe the physical layout of "b". */
NGA_Get(g_b,lo3,hi3,b,ld);
/* Do local matrix multiplication and store the result in local
buffer c. Start by evaluating the transpose of b. */
for(i=0; i < hi3[0]-lo3[0]+1; i++)
for(j=0; j < hi3[1]-lo3[1]+1; j++)
btrns[j][i] = b[i][j];
/* Multiply a and b to get c */
for(i=0; i < hi[0] - lo[0] + 1; i++) {
for(j=0; j < hi[1] - lo[1] + 1; j++) {
c[i][j] = 0.0;
for(k=0; k<dims[0]; k++)
c[i][j] = c[i][j] + a[i][k]*btrns[j][k];
}
}
/* Copy c back to g_c */
/* ### copy data from the local array "c" into the block of the global
* ### array "g_c" described by the integer arrays "lo" and "hi". Use
* ### the array of strides "ld" to describe the physical layout of "c". */
NGA_Put(g_c,lo,hi,c,ld);
verify(g_a, g_b, g_c, lo1, hi1, ld);
/* Deallocate arrays */
/* ### destroy the global arrays "g_a", "g_b", "g_c" */
GA_Destroy(g_a);
GA_Destroy(g_b);
GA_Destroy(g_c);
}
int main(int argc, char **argv)
{
int rank, nprocs;
int g_A;
int *local_A=NULL, *local_B=NULL, *output_A=NULL;
int dims[DIM]={SIZE,SIZE}, dims2[DIM], lo[DIM]={SIZE-SIZE,SIZE-SIZE}, hi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
int value=SIZE;
#if defined(USE_ELEMENTAL)
// initialize Elemental (which will initialize MPI)
ElInitialize( &argc, &argv );
ElMPICommRank( MPI_COMM_WORLD, &rank );
ElMPICommSize( MPI_COMM_WORLD, &nprocs );
// instantiate el::global array
ElGlobalArraysConstruct_i( &eliga );
// initialize global arrays
ElGlobalArraysInitialize_i( eliga );
#else
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
#endif
local_A=(int*)malloc(SIZE*SIZE*sizeof(int));
output_A=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (output_A, 0, SIZE*SIZE*sizeof(int));
for(int j=0; j<SIZE; j++)
for(int i=0; i<SIZE; i++) local_A[i+j*ld]=(i + j);
//for(int i=0; i<SIZE; i++) local_A[i+j*ld]=(rand()%10);
local_B=(int*)malloc(SIZE*SIZE*sizeof(int));
memset (local_B, 0, SIZE*SIZE*sizeof(int));
// nb handle
#if defined(USE_ELEMENTAL)
typedef ElInt ga_nbhdl_t;
#endif
ga_nbhdl_t nbnb;
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_i( eliga, DIM, dims, "array_A", NULL, &g_A );
ElGlobalArraysFill_i( eliga, g_A, &value );
#else
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
GA_Fill(g_A, &value);
#endif
if (rank == 0) printf ("Initial global array:\n");
#if defined(USE_ELEMENTAL)
ElGlobalArraysPrint_i( eliga, g_A );
#else
GA_Print(g_A);
#endif
for (int i = 0; i < NITERS; i++)
{
// acc data
#if defined(USE_ELEMENTAL)
ElGlobalArraysNBAccumulate_i( eliga, g_A, lo, hi, local_A, &ld, &value, &nbnb );
#else
NGA_NbAcc(g_A, lo, hi, local_A, &ld, &value, &nbnb);
#endif
// updated output
MPI_Reduce (local_A, output_A, SIZE*SIZE, MPI_INT, MPI_SUM, 0, MPI_COMM_WORLD);
#if defined(USE_ELEMENTAL)
ElGlobalArraysNBWait_i( eliga, &nbnb );
#else
NGA_NbWait (&nbnb);
#endif
// get
if (rank == 0) printf ("Get in iter #%d\n", i);
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
ElGlobalArraysGet_i( eliga, g_A, lo, hi, local_B, &ld );
ElGlobalArraysPrint_i( eliga, g_A );
#else
GA_Sync();
NGA_Get(g_A, lo, hi, local_B, &ld);
GA_Print(g_A);
#endif
} // end of iters
if(rank==0)
{
printf(" Alpha (multiplier): %d\n", value);
printf(" Original local buffer (before accumulation): \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_A[i*ld+j]);
printf("\n");
}
printf("\n");
printf(" Get returns: \n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
printf("%d ", local_B[i*ld + j]);
printf("\n");
}
printf("\n");
for(int i=0; i<SIZE; i++)
{
for(int j=0; j<SIZE; j++)
{
if(local_B[i*ld+j]!=(value + (NITERS * value * (output_A[i*ld+j]))))
GA_Error("ERROR", -99);
}
}
}
#if defined(USE_ELEMENTAL)
ElGlobalArraysDestroy_i( eliga, g_A );
#else
GA_Destroy(g_A);
#endif
if(rank == 0)
printf ("OK. Test passed\n");
free (local_A);
free (local_B);
free (output_A);
#if defined(USE_ELEMENTAL)
ElGlobalArraysTerminate_i( eliga );
// call el::global arrays destructor
ElGlobalArraysDestruct_i( eliga );
ElFinalize();
#else
GA_Terminate();
MPI_Finalize();
#endif
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, **local_C=NULL, dims[DIM]={SIZE,SIZE}, val1=5, val2=4, alpha=3, beta=2;
int clo[DIM]={SIZE-SIZE,SIZE-SIZE}, chi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
int **local_tm=NULL;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
local_C=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_C[i]=(int*)malloc(SIZE*sizeof(int));
local_tm=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_tm[i]=(int*)malloc(SIZE*sizeof(int));
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = GA_Duplicate(g_A, "array_B");
g_C = GA_Duplicate(g_A, "array_C");
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
GA_Add(&alpha, g_A, &beta, g_B, g_C);
GA_Sync();
GA_Print(g_A);
GA_Print(g_B);
GA_Print(g_C);
//printf("check 1\n");
NGA_Get(g_C, clo, chi, local_tm, &ld);
//printf("check 2\n");
// GA_Sync();
if(rank==0)
{
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)printf("%d ", local_tm[i][j]);
printf("\n");
}
}
/*
if(rank==0)
{
NGA_Get(g_C, clo, chi, local_C, &ld);
printf("check 1 \n");
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
printf("check 2\n");
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)
if(local_C[i][j]!=(alpha*val1)+(beta*val2)) printf("GA Error : \n");
}
}
*/
//GA_Sync();
if(rank==0)
printf("Test Completed \n");
// GA_Sync();
/*
GA_Destroy(g_A);
GA_Destroy(g_B);
GA_Destroy(g_C);
*/
//*******************************************************************
/* what would be the possible reason for GA_destroy to get failed ..,
* solve this before consolidate the whole
*/
GA_Terminate();
MPI_Finalize();
}
// -------------------------------------------------------------
// AdjacencyList::ready
// -------------------------------------------------------------
void
AdjacencyList::ready(void)
{
#if 1
int grp = this->communicator().getGroup();
int me = GA_Pgroup_nodeid(grp);
int nprocs = GA_Pgroup_nnodes(grp);
p_adjacency.clear();
p_adjacency.resize(p_global_nodes.size());
// Find total number of nodes and edges. Assume no duplicates
int nedges = p_edges.size();
int total_edges = nedges;
char plus[2];
strcpy(plus,"+");
GA_Pgroup_igop(grp,&total_edges, 1, plus);
int nnodes = p_original_nodes.size();
int total_nodes = nnodes;
GA_Pgroup_igop(grp,&total_nodes, 1, plus);
// Create a global array containing original indices of all nodes and indexed
// by the global index of the node
int i, p;
int dist[nprocs];
for (p=0; p<nprocs; p++) {
dist[p] = 0;
}
dist[me] = nnodes;
GA_Pgroup_igop(grp,dist,nprocs,plus);
int *mapc = new int[nprocs+1];
mapc[0] = 0;
for (p=1; p<nprocs; p++) {
mapc[p] = mapc[p-1] + dist[p-1];
}
mapc[nprocs] = total_nodes;
int g_nodes = GA_Create_handle();
int dims = total_nodes;
NGA_Set_data(g_nodes,1,&dims,C_INT);
NGA_Set_pgroup(g_nodes, grp);
if (!GA_Allocate(g_nodes)) {
char buf[256];
sprintf(buf,"AdjacencyList::ready: Unable to allocate distributed array"
" for bus indices\n");
printf(buf);
throw gridpack::Exception(buf);
}
int lo, hi;
lo = mapc[me];
hi = mapc[me+1]-1;
int size = hi - lo + 1;
int o_idx[size], g_idx[size];
for (i=0; i<size; i++) o_idx[i] = p_original_nodes[i];
for (i=0; i<size; i++) g_idx[i] = p_global_nodes[i];
int **indices= new int*[size];
int *iptr = g_idx;
for (i=0; i<size; i++) {
indices[i] = iptr;
iptr++;
}
if (size > 0) NGA_Scatter(g_nodes,o_idx,indices,size);
GA_Pgroup_sync(grp);
delete [] indices;
delete [] mapc;
// Cycle through all nodes and match them up with nodes at end of edges.
for (p=0; p<nprocs; p++) {
int iproc = (me+p)%nprocs;
// Get node data from process iproc
NGA_Distribution(g_nodes,iproc,&lo,&hi);
size = hi - lo + 1;
if (size <= 0) continue;
int *buf = new int[size];
int ld = 1;
NGA_Get(g_nodes,&lo,&hi,buf,&ld);
// Create a map of the nodes from process p
std::map<int,int> nmap;
std::map<int,int>::iterator it;
std::pair<int,int> pr;
for (i=lo; i<=hi; i++){
pr = std::pair<int,int>(buf[i-lo],i);
nmap.insert(pr);
}
delete [] buf;
// scan through the edges looking for matches. If there is a match, set the
// global index
int idx;
for (i=0; i<nedges; i++) {
idx = static_cast<int>(p_edges[i].original_conn.first);
it = nmap.find(idx);
if (it != nmap.end()) {
p_edges[i].global_conn.first = static_cast<Index>(it->second);
}
idx = static_cast<int>(p_edges[i].original_conn.second);
it = nmap.find(idx);
if (it != nmap.end()) {
p_edges[i].global_conn.second = static_cast<Index>(it->second);
}
}
}
GA_Destroy(g_nodes);
// All edges now have global indices assigned to them. Begin constructing
// adjacency list. Start by creating a global array containing all edges
dist[0] = 0;
for (p=1; p<nprocs; p++) {
double max = static_cast<double>(total_edges);
max = (static_cast<double>(p))*(max/(static_cast<double>(nprocs)));
dist[p] = 2*(static_cast<int>(max));
}
int g_edges = GA_Create_handle();
dims = 2*total_edges;
NGA_Set_data(g_edges,1,&dims,C_INT);
NGA_Set_irreg_distr(g_edges,dist,&nprocs);
NGA_Set_pgroup(g_edges, grp);
if (!GA_Allocate(g_edges)) {
char buf[256];
sprintf(buf,"AdjacencyList::ready: Unable to allocate distributed array"
" for branch indices\n");
printf(buf);
throw gridpack::Exception(buf);
}
// Add edge information to global array. Start by figuring out how much data
// is associated with each process
for (p=0; p<nprocs; p++) {
dist[p] = 0;
}
dist[me] = nedges;
GA_Pgroup_igop(grp,dist, nprocs, plus);
int offset[nprocs];
offset[0] = 0;
for (p=1; p<nprocs; p++) {
offset[p] = offset[p-1] + 2*dist[p-1];
}
// Figure out where local data goes in GA and then copy it to GA
lo = offset[me];
hi = lo + 2*nedges - 1;
int edge_ids[2*nedges];
for (i=0; i<nedges; i++) {
edge_ids[2*i] = static_cast<int>(p_edges[i].global_conn.first);
edge_ids[2*i+1] = static_cast<int>(p_edges[i].global_conn.second);
}
if (lo <= hi) {
int ld = 1;
NGA_Put(g_edges,&lo,&hi,edge_ids,&ld);
}
GA_Pgroup_sync(grp);
// Cycle through all edges and find out how many are attached to the nodes on
// your process. Start by creating a map between the global node indices and
// the local node indices
std::map<int,int> gmap;
std::map<int,int>::iterator it;
std::pair<int,int> pr;
for (i=0; i<nnodes; i++){
pr = std::pair<int,int>(static_cast<int>(p_global_nodes[i]),i);
gmap.insert(pr);
}
// Cycle through edge information on each processor
for (p=0; p<nprocs; p++) {
int iproc = (me+p)%nprocs;
NGA_Distribution(g_edges,iproc,&lo,&hi);
int size = hi - lo + 1;
int *buf = new int[size];
int ld = 1;
NGA_Get(g_edges,&lo,&hi,buf,&ld);
BOOST_ASSERT(size%2 == 0);
size = size/2;
int idx1, idx2;
Index idx;
for (i=0; i<size; i++) {
idx1 = buf[2*i];
idx2 = buf[2*i+1];
it = gmap.find(idx1);
if (it != gmap.end()) {
idx = static_cast<Index>(idx2);
p_adjacency[it->second].push_back(idx);
}
it = gmap.find(idx2);
if (it != gmap.end()) {
idx = static_cast<Index>(idx1);
p_adjacency[it->second].push_back(idx);
}
}
delete [] buf;
}
GA_Destroy(g_edges);
GA_Pgroup_sync(grp);
#else
int me(this->processor_rank());
int nproc(this->processor_size());
p_adjacency.clear();
p_adjacency.resize(p_nodes.size());
IndexVector current_indexes;
IndexVector connected_indexes;
for (int p = 0; p < nproc; ++p) {
// broadcast the node indexes owned by process p to all processes,
// all processes work on these at once
current_indexes.clear();
if (me == p) {
std::copy(p_nodes.begin(), p_nodes.end(),
std::back_inserter(current_indexes));
// std::cout << me << ": node indexes: ";
// std::copy(current_indexes.begin(), current_indexes.end(),
// std::ostream_iterator<Index>(std::cout, ","));
// std::cout << std::endl;
}
boost::mpi::broadcast(this->communicator(), current_indexes, p);
// make a copy of the local edges in a list (so it's easier to
// remove those completely accounted for)
std::list<p_Edge> tmpedges;
std::copy(p_edges.begin(), p_edges.end(),
std::back_inserter(tmpedges));
// loop over the process p's node index set
int local_index(0);
for (IndexVector::iterator n = current_indexes.begin();
n != current_indexes.end(); ++n, ++local_index) {
// determine the local edges that refer to the current node index
connected_indexes.clear();
std::list<p_Edge>::iterator e(tmpedges.begin());
// std::cout << me << ": current node index: " << *n
// << ", edges: " << tmpedges.size()
// << std::endl;
while (e != tmpedges.end()) {
if (*n == e->conn.first && e->conn.second != bogus) {
connected_indexes.push_back(e->conn.second);
e->found.first = true;
// std::cout << me << ": found connection: edge " << e->index
// << " (" << e->conn.first << ", " << e->conn.second << ")"
// << std::endl;
}
if (*n == e->conn.second && e->conn.first != bogus) {
connected_indexes.push_back(e->conn.first);
e->found.second = true;
// std::cout << me << ": found connection: edge " << e->index
// << " (" << e->conn.first << ", " << e->conn.second << ")"
// << std::endl;
}
if (e->found.first && e->found.second) {
e = tmpedges.erase(e);
} else if (e->conn.first == bogus ||
e->conn.second == bogus) {
e = tmpedges.erase(e);
} else {
++e;
}
}
// gather all connections for the current node index to the
// node's owner process, we have to gather the vectors because
// processes will have different numbers of connections
if (me == p) {
size_t allsize;
boost::mpi::reduce(this->communicator(),
connected_indexes.size(), allsize, std::plus<size_t>(), p);
std::vector<IndexVector> all_connected_indexes;
boost::mpi::gather(this->communicator(),
connected_indexes, all_connected_indexes, p);
p_adjacency[local_index].clear();
for (std::vector<IndexVector>::iterator k = all_connected_indexes.begin();
k != all_connected_indexes.end(); ++k) {
std::copy(k->begin(), k->end(),
std::back_inserter(p_adjacency[local_index]));
}
} else {
boost::mpi::reduce(this->communicator(),
connected_indexes.size(), std::plus<size_t>(), p);
boost::mpi::gather(this->communicator(), connected_indexes, p);
}
this->communicator().barrier();
}
this->communicator().barrier();
}
#endif
}
void do_work()
{
int ONE=1 ; /* useful constants */
int g_a, g_b;
int n=N, type=MT_F_DBL;
int me=GA_Nodeid(), nproc=GA_Nnodes();
int i, row;
int dims[2]={N,N};
int lo[2], hi[2], ld;
/* Note: on all current platforms DoublePrecision == double */
double buf[N], err, alpha, beta;
if(me==0)printf("Creating matrix A\n");
g_a = NGA_Create(type, 2, dims, "A", NULL);
if(!g_a) GA_Error("create failed: A",n);
if(me==0)printf("OK\n");
if(me==0)printf("Creating matrix B\n");
/* create matrix B so that it has dims and distribution of A*/
g_b = GA_Duplicate(g_a, "B");
if(! g_b) GA_Error("duplicate failed",n);
if(me==0)printf("OK\n");
GA_Zero(g_a); /* zero the matrix */
if(me==0)printf("Initializing matrix A\n");
/* fill in matrix A with random values in range 0.. 1 */
lo[1]=0; hi[1]=n-1;
for(row=me; row<n; row+= nproc){
/* each process works on a different row in MIMD style */
lo[0]=hi[0]=row;
for(i=0; i<n; i++) buf[i]=sin((double)i + 0.1*(row+1));
NGA_Put(g_a, lo, hi, buf, &n);
}
if(me==0)printf("Symmetrizing matrix A\n");
GA_Symmetrize(g_a); /* symmetrize the matrix A = 0.5*(A+A') */
/* check if A is symmetric */
if(me==0)printf("Checking if matrix A is symmetric\n");
GA_Transpose(g_a, g_b); /* B=A' */
alpha=1.; beta=-1.;
GA_Add(&alpha, g_a, &beta, g_b, g_b); /* B= A - B */
err= GA_Ddot(g_b, g_b);
if(me==0)printf("Error=%f\n",(double)err);
if(me==0)printf("\nChecking atomic accumulate \n");
GA_Zero(g_a); /* zero the matrix */
for(i=0; i<n; i++) buf[i]=(double)i;
/* everybody accumulates to the same location/row */
alpha = 1.0;
row = n/2;
lo[0]=hi[0]=row;
lo[1]=0; hi[1]=n-1;
ld = hi[1]-lo[1]+1;
NGA_Acc(g_a, lo, hi, buf, &ld, &alpha );
GA_Sync();
if(me==0){ /* node 0 is checking the result */
NGA_Get(g_a, lo, hi, buf,&ld);
for(i=0; i<n; i++) if(buf[i] != (double)nproc*i)
GA_Error("failed: column=",i);
printf("OK\n\n");
}
GA_Destroy(g_a);
GA_Destroy(g_b);
}
