void verify(int g_a, int g_b, int g_c, int *lo, int *hi, int *ld, int N)
{
double rchk, alpha=1.0, beta=0.0;
int g_chk, me=GA_Nodeid();
g_chk = GA_Duplicate(g_a, "array Check");
if(!g_chk) GA_Error("duplicate failed",NDIMS);
GA_Sync();
GA_Dgemm('n', 'n', N, N, N, 1.0, g_a,
g_b, 0.0, g_chk);
GA_Sync();
alpha=1.0, beta=-1.0;
GA_Add(&alpha, g_c, &beta, g_chk, g_chk);
rchk = GA_Ddot(g_chk, g_chk);
if (me==0) {
printf("Normed difference in matrices: %12.4e\n", rchk);
if(rchk < -TOLERANCE || rchk > TOLERANCE)
GA_Error("Matrix multiply verify failed",0);
else
printf("Matrix Mutiply OK\n");
}
GA_Destroy(g_chk);
}
// -------------------------------------------------------------
// MatDuplicate_DenseGA
// -------------------------------------------------------------
static
PetscErrorCode
MatDuplicate_DenseGA(Mat mat, MatDuplicateOption op, Mat *M)
{
PetscErrorCode ierr = 0;
struct MatGACtx *ctx, *newctx;
ierr = MatShellGetContext(mat, &ctx); CHKERRQ(ierr);
MPI_Comm comm;
ierr = PetscObjectGetComm((PetscObject)mat, &comm); CHKERRQ(ierr);
PetscInt lrows, grows, lcols, gcols;
ierr = MatGetSize(mat, &grows, &gcols); CHKERRQ(ierr);
ierr = MatGetLocalSize(mat, &lrows, &lcols); CHKERRQ(ierr);
ierr = PetscMalloc(sizeof(struct MatGACtx), &newctx); CHKERRQ(ierr);
newctx->gaGroup = ctx->gaGroup;
newctx->ga = GA_Duplicate(ctx->ga, "PETSc Dense Matrix");
ierr = MatCreateShell(comm, lrows, lcols, grows, gcols, newctx, M); CHKERRQ(ierr);
ierr = MatSetOperations_DenseGA(*M);
PetscScalar z(0.0);
switch (op) {
case (MAT_COPY_VALUES):
GA_Copy(ctx->ga, newctx->ga);
break;
default:
GA_Fill(newctx->ga, &z);
break;
}
GA_Pgroup_sync(newctx->gaGroup);
return ierr;
}
void
test(int data_type) {
int me=GA_Nodeid();
int nproc = GA_Nnodes();
int g_a, g_b, g_c;
int ndim = 2;
int dims[2]={N,N};
int lo[2]={0,0};
int hi[2]={N-1,N-1};
int block_size[2]={NB,NB-1};
int proc_grid[2];
int i,j,l,k,m,n, ld;
double alpha_dbl = 1.0, beta_dbl = 0.0;
double dzero = 0.0;
double ddiff;
float alpha_flt = 1.0, beta_flt = 0.0;
float fzero = 0.0;
float fdiff;
float ftmp;
double dtmp;
SingleComplex ctmp;
DoubleComplex ztmp;
DoubleComplex alpha_dcpl = {1.0, 0.0} , beta_dcpl = {0.0, 0.0};
DoubleComplex zzero = {0.0,0.0};
DoubleComplex zdiff;
SingleComplex alpha_scpl = {1.0, 0.0} , beta_scpl = {0.0, 0.0};
SingleComplex czero = {0.0,0.0};
SingleComplex cdiff;
void *alpha=NULL, *beta=NULL;
void *abuf=NULL, *bbuf=NULL, *cbuf=NULL, *c_ptr=NULL;
switch (data_type) {
case C_FLOAT:
alpha = (void *)&alpha_flt;
beta = (void *)&beta_flt;
abuf = (void*)malloc(N*N*sizeof(float));
bbuf = (void*)malloc(N*N*sizeof(float));
cbuf = (void*)malloc(N*N*sizeof(float));
if(me==0) printf("Single Precision: Testing GA_Sgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
case C_DBL:
alpha = (void *)&alpha_dbl;
beta = (void *)&beta_dbl;
abuf = (void*)malloc(N*N*sizeof(double));
bbuf = (void*)malloc(N*N*sizeof(double));
cbuf = (void*)malloc(N*N*sizeof(double));
if(me==0) printf("Double Precision: Testing GA_Dgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
case C_DCPL:
alpha = (void *)&alpha_dcpl;
beta = (void *)&beta_dcpl;
abuf = (void*)malloc(N*N*sizeof(DoubleComplex));
bbuf = (void*)malloc(N*N*sizeof(DoubleComplex));
cbuf = (void*)malloc(N*N*sizeof(DoubleComplex));
if(me==0) printf("Double Complex: Testing GA_Zgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
case C_SCPL:
alpha = (void *)&alpha_scpl;
beta = (void *)&beta_scpl;
abuf = (void*)malloc(N*N*sizeof(SingleComplex));
bbuf = (void*)malloc(N*N*sizeof(SingleComplex));
cbuf = (void*)malloc(N*N*sizeof(SingleComplex));
if(me==0) printf("Single Complex: Testing GA_Cgemm,NGA_Matmul_patch for %d-Dimension", ndim);
break;
default:
GA_Error("wrong data type", data_type);
}
if (me==0) printf("\nCreate A, B, C\n");
#ifdef USE_REGULAR
g_a = NGA_Create(data_type, ndim, dims, "array A", NULL);
#endif
#ifdef USE_SIMPLE_CYCLIC
g_a = NGA_Create_handle();
NGA_Set_data(g_a,ndim,dims,data_type);
NGA_Set_array_name(g_a,"array A");
NGA_Set_block_cyclic(g_a,block_size);
if (!GA_Allocate(g_a)) {
GA_Error("Failed: create: g_a",40);
}
#endif
#ifdef USE_SCALAPACK
g_a = NGA_Create_handle();
NGA_Set_data(g_a,ndim,dims,data_type);
NGA_Set_array_name(g_a,"array A");
grid_factor(nproc,&i,&j);
proc_grid[0] = i;
proc_grid[1] = j;
NGA_Set_block_cyclic_proc_grid(g_a,block_size,proc_grid);
if (!GA_Allocate(g_a)) {
GA_Error("Failed: create: g_a",40);
}
#endif
#ifdef USE_TILED
g_a = NGA_Create_handle();
NGA_Set_data(g_a,ndim,dims,data_type);
NGA_Set_array_name(g_a,"array A");
grid_factor(nproc,&i,&j);
proc_grid[0] = i;
proc_grid[1] = j;
NGA_Set_tiled_proc_grid(g_a,block_size,proc_grid);
if (!GA_Allocate(g_a)) {
GA_Error("Failed: create: g_a",40);
}
#endif
g_b = GA_Duplicate(g_a, "array B");
g_c = GA_Duplicate(g_a, "array C");
if(!g_a || !g_b || !g_c) GA_Error("Create failed: a, b or c",1);
ld = N;
if (me==0) printf("\nInitialize A\n");
/* Set up matrix A */
if (me == 0) {
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
((float*)abuf)[i*N+j] = (float)(i*N+j);
break;
case C_DBL:
((double*)abuf)[i*N+j] = (double)(i*N+j);
break;
case C_DCPL:
((DoubleComplex*)abuf)[i*N+j].real = (double)(i*N+j);
((DoubleComplex*)abuf)[i*N+j].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)abuf)[i*N+j].real = (float)(i*N+j);
((SingleComplex*)abuf)[i*N+j].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_a,lo,hi,abuf,&ld);
}
GA_Sync();
if (me==0) printf("\nInitialize B\n");
/* Set up matrix B */
if (me == 0) {
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
((float*)bbuf)[i*N+j] = (float)(j*N+i);
break;
case C_DBL:
((double*)bbuf)[i*N+j] = (double)(j*N+i);
break;
case C_DCPL:
((DoubleComplex*)bbuf)[i*N+j].real = (double)(j*N+i);
((DoubleComplex*)bbuf)[i*N+j].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)bbuf)[i*N+j].real = (float)(j*N+i);
((SingleComplex*)bbuf)[i*N+j].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_b,lo,hi,bbuf,&ld);
}
GA_Sync();
if (me==0) printf("\nPerform matrix multiply\n");
switch (data_type) {
case C_FLOAT:
NGA_Matmul_patch('N','N',&alpha_flt,&beta_flt,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DBL:
NGA_Matmul_patch('N','N',&alpha_dbl,&beta_dbl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_SCPL:
NGA_Matmul_patch('N','N',&alpha_scpl,&beta_scpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DCPL:
NGA_Matmul_patch('N','N',&alpha_dcpl,&beta_dcpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
default:
GA_Error("wrong data type", data_type);
}
GA_Sync();
#if 0
if (me==0) printf("\nCheck answer\n");
/*
GA_Print(g_a);
if (me == 0) printf("\n\n\n\n");
GA_Print(g_b);
if (me == 0) printf("\n\n\n\n");
GA_Print(g_c);
*/
/* Check answer */
NGA_Get(g_a,lo,hi,abuf,&ld);
NGA_Get(g_b,lo,hi,bbuf,&ld);
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N+j] = fzero;
break;
case C_DBL:
((double*)cbuf)[i*N+j] = dzero;
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N+j] = zzero;
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N+j] = czero;
break;
default:
GA_Error("wrong data type", data_type);
}
for (k=0; k<N; k++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N+j] += ((float*)abuf)[i*N+k]
*((float*)bbuf)[k*N+j];
break;
case C_DBL:
((double*)cbuf)[i*N+j] += ((double*)abuf)[i*N+k]
*((double*)bbuf)[k*N+j];
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N+j].real +=
(((DoubleComplex*)abuf)[i*N+k].real
*((DoubleComplex*)bbuf)[k*N+j].real
-(((DoubleComplex*)abuf)[i*N+k].imag
*((DoubleComplex*)bbuf)[k*N+j].imag));
((DoubleComplex*)cbuf)[i*N+j].imag +=
(((DoubleComplex*)abuf)[i*N+k].real
*((DoubleComplex*)bbuf)[k*N+j].imag
+(((DoubleComplex*)abuf)[i*N+k].imag
*((DoubleComplex*)bbuf)[k*N+j].real));
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N+j].real +=
(((SingleComplex*)abuf)[i*N+k].real
*((SingleComplex*)bbuf)[k*N+j].real
-(((SingleComplex*)abuf)[i*N+k].imag
*((SingleComplex*)bbuf)[k*N+j].imag));
((SingleComplex*)cbuf)[i*N+j].imag +=
(((SingleComplex*)abuf)[i*N+k].real
*((SingleComplex*)bbuf)[k*N+j].imag
+(((SingleComplex*)abuf)[i*N+k].imag
*((SingleComplex*)bbuf)[k*N+j].real));
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
}
GA_Sync();
if (me == 0) {
NGA_Get(g_c,lo,hi,abuf,&ld);
for (i=0; i<N; i++) {
for (j=0; j<N; j++) {
switch (data_type) {
case C_FLOAT:
fdiff = ((float*)abuf)[i*N+j]-((float*)cbuf)[i*N+j];
if (((float*)abuf)[i*N+j] != 0.0) {
fdiff /= ((float*)abuf)[i*N+j];
}
if (fabs(fdiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: %f Expected: %f\n",me,i,j,
((float*)abuf)[i*N+j],((float*)cbuf)[i*N+j]);
}
break;
case C_DBL:
ddiff = ((double*)abuf)[i*N+j]-((double*)cbuf)[i*N+j];
if (((double*)abuf)[i*N+j] != 0.0) {
ddiff /= ((double*)abuf)[i*N+j];
}
if (fabs(ddiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: %f Expected: %f\n",me,i,j,
((double*)abuf)[i*N+j],((double*)cbuf)[i*N+j]);
}
break;
case C_DCPL:
zdiff.real = ((DoubleComplex*)abuf)[i*N+j].real
-((DoubleComplex*)cbuf)[i*N+j].real;
zdiff.imag = ((DoubleComplex*)abuf)[i*N+j].imag
-((DoubleComplex*)cbuf)[i*N+j].imag;
if (((DoubleComplex*)abuf)[i*N+j].real != 0.0 ||
((DoubleComplex*)abuf)[i*N+j].imag != 0.0) {
ztmp = ((DoubleComplex*)abuf)[i*N+j];
ddiff = sqrt((zdiff.real*zdiff.real+zdiff.imag*zdiff.imag)
/(ztmp.real*ztmp.real+ztmp.imag*ztmp.imag));
} else {
ddiff = sqrt(zdiff.real*zdiff.real+zdiff.imag*zdiff.imag);
}
if (fabs(ddiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: (%f,%f) Expected: (%f,%f)\n",me,i,j,
((DoubleComplex*)abuf)[i*N+j].real,
((DoubleComplex*)abuf)[i*N+j].imag,
((DoubleComplex*)cbuf)[i*N+j].real,
((DoubleComplex*)cbuf)[i*N+j].imag);
}
break;
case C_SCPL:
cdiff.real = ((SingleComplex*)abuf)[i*N+j].real
-((SingleComplex*)cbuf)[i*N+j].real;
cdiff.imag = ((SingleComplex*)abuf)[i*N+j].imag
-((SingleComplex*)cbuf)[i*N+j].imag;
if (((SingleComplex*)abuf)[i*N+j].real != 0.0 ||
((SingleComplex*)abuf)[i*N+j].imag != 0.0) {
ctmp = ((SingleComplex*)abuf)[i*N+j];
fdiff = sqrt((cdiff.real*cdiff.real+cdiff.imag*cdiff.imag)
/(ctmp.real*ctmp.real+ctmp.imag*ctmp.imag));
} else {
fdiff = sqrt(cdiff.real*cdiff.real+cdiff.imag*cdiff.imag);
}
if (fabs(fdiff) > TOLERANCE) {
printf("p[%d] [%d,%d] Actual: (%f,%f) Expected: (%f,%f)\n",me,i,j,
((SingleComplex*)abuf)[i*N+j].real,
((SingleComplex*)abuf)[i*N+j].imag,
((SingleComplex*)cbuf)[i*N+j].real,
((SingleComplex*)cbuf)[i*N+j].imag);
}
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
}
GA_Sync();
/* copy cbuf back to g_a */
if (me == 0) {
NGA_Put(g_a,lo,hi,cbuf,&ld);
}
GA_Sync();
/* Get norm of g_a */
switch (data_type) {
case C_FLOAT:
ftmp = GA_Fdot(g_a,g_a);
break;
case C_DBL:
dtmp = GA_Ddot(g_a,g_a);
break;
case C_DCPL:
ztmp = GA_Zdot(g_a,g_a);
break;
case C_SCPL:
ctmp = GA_Cdot(g_a,g_a);
break;
default:
GA_Error("wrong data type", data_type);
}
/* subtract C from A and put the results in B */
beta_flt = -1.0;
beta_dbl = -1.0;
beta_scpl.real = -1.0;
beta_dcpl.real = -1.0;
GA_Zero(g_b);
GA_Add(alpha,g_a,beta,g_c,g_b);
/* evaluate the norm of the difference between the two matrices */
switch (data_type) {
case C_FLOAT:
fdiff = GA_Fdot(g_b, g_b);
if (ftmp != 0.0) {
fdiff /= ftmp;
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(fdiff), TOLERANCE);
GA_Error("GA_Sgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Sgemm OK\n\n");
}
break;
case C_DBL:
ddiff = GA_Ddot(g_b, g_b);
if (dtmp != 0.0) {
ddiff /= dtmp;
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(ddiff), TOLERANCE);
GA_Error("GA_Dgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Dgemm OK\n\n");
}
break;
case C_DCPL:
zdiff = GA_Zdot(g_b, g_b);
if (ztmp.real != 0.0 || ztmp.imag != 0.0) {
ddiff = sqrt((zdiff.real*zdiff.real+zdiff.imag*zdiff.imag)
/(ztmp.real*ztmp.real+ztmp.imag*ztmp.imag));
} else {
ddiff = sqrt(zdiff.real*zdiff.real+zdiff.imag*zdiff.imag);
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(zdiff.real), TOLERANCE);
GA_Error("GA_Zgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Zgemm OK\n\n");
}
break;
case C_SCPL:
cdiff = GA_Cdot(g_b, g_b);
if (ctmp.real != 0.0 || ctmp.imag != 0.0) {
fdiff = sqrt((cdiff.real*cdiff.real+cdiff.imag*cdiff.imag)
/(ctmp.real*ctmp.real+ctmp.imag*ctmp.imag));
} else {
fdiff = sqrt(cdiff.real*cdiff.real+cdiff.imag*cdiff.imag);
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(cdiff.real), TOLERANCE);
GA_Error("GA_Cgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Cgemm OK\n\n");
}
break;
default:
GA_Error("wrong data type", data_type);
}
#endif
free(abuf);
free(bbuf);
free(cbuf);
switch (data_type) {
case C_FLOAT:
abuf = (void*)malloc(N*N*sizeof(float)/4);
bbuf = (void*)malloc(N*N*sizeof(float)/4);
cbuf = (void*)malloc(N*N*sizeof(float)/4);
break;
case C_DBL:
abuf = (void*)malloc(N*N*sizeof(double)/4);
bbuf = (void*)malloc(N*N*sizeof(double)/4);
cbuf = (void*)malloc(N*N*sizeof(double)/4);
break;
case C_DCPL:
abuf = (void*)malloc(N*N*sizeof(DoubleComplex)/4);
bbuf = (void*)malloc(N*N*sizeof(DoubleComplex)/4);
cbuf = (void*)malloc(N*N*sizeof(DoubleComplex)/4);
break;
case C_SCPL:
abuf = (void*)malloc(N*N*sizeof(SingleComplex)/4);
bbuf = (void*)malloc(N*N*sizeof(SingleComplex)/4);
cbuf = (void*)malloc(N*N*sizeof(SingleComplex)/4);
break;
default:
GA_Error("wrong data type", data_type);
}
/* Test multiply on a fraction of matrix. Start by reinitializing
* A and B */
GA_Zero(g_a);
GA_Zero(g_b);
GA_Zero(g_c);
if (me==0) printf("\nTest patch multiply\n");
lo[0] = N/4;
lo[1] = N/4;
hi[0] = 3*N/4-1;
hi[1] = 3*N/4-1;
ld = N/2;
/* Set up matrix A */
if (me==0) printf("\nInitialize A\n");
if (me == 0) {
for (i=N/4; i<3*N/4; i++) {
for (j=N/4; j<3*N/4; j++) {
switch (data_type) {
case C_FLOAT:
((float*)abuf)[(i-N/4)*N/2+(j-N/4)] = (float)(i*N+j);
break;
case C_DBL:
((double*)abuf)[(i-N/4)*N/2+(j-N/4)] = (double)(i*N+j);
break;
case C_DCPL:
((DoubleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].real = (double)(i*N+j);
((DoubleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].real = (float)(i*N+j);
((SingleComplex*)abuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_a,lo,hi,abuf,&ld);
}
GA_Sync();
if (me==0) printf("\nInitialize B\n");
/* Set up matrix B */
if (me == 0) {
for (i=N/4; i<3*N/4; i++) {
for (j=N/4; j<3*N/4; j++) {
switch (data_type) {
case C_FLOAT:
((float*)bbuf)[(i-N/4)*N/2+(j-N/4)] = (float)(j*N+i);
break;
case C_DBL:
((double*)bbuf)[(i-N/4)*N/2+(j-N/4)] = (double)(j*N+i);
break;
case C_DCPL:
((DoubleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].real = (double)(j*N+i);
((DoubleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
case C_SCPL:
((SingleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].real = (float)(j*N+i);
((SingleComplex*)bbuf)[(i-N/4)*N/2+(j-N/4)].imag = 1.0;
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
NGA_Put(g_b,lo,hi,bbuf,&ld);
}
GA_Sync();
beta_flt = 0.0;
beta_dbl = 0.0;
beta_scpl.real = 0.0;
beta_dcpl.real = 0.0;
if (me==0) printf("\nPerform matrix multiply on sub-blocks\n");
switch (data_type) {
case C_FLOAT:
NGA_Matmul_patch('N','N',&alpha_flt,&beta_flt,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DBL:
NGA_Matmul_patch('N','N',&alpha_dbl,&beta_dbl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_SCPL:
NGA_Matmul_patch('N','N',&alpha_scpl,&beta_scpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
case C_DCPL:
NGA_Matmul_patch('N','N',&alpha_dcpl,&beta_dcpl,g_a,lo,hi,
g_b,lo,hi,g_c,lo,hi);
break;
default:
GA_Error("wrong data type", data_type);
}
GA_Sync();
#if 0
if (0) {
/*
if (data_type != C_SCPL && data_type != C_DCPL) {
*/
if (me==0) printf("\nCheck answer\n");
/* Multiply buffers by hand */
if (me == 0) {
for (i=0; i<N/2; i++) {
for (j=0; j<N/2; j++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N/2+j] = fzero;
break;
case C_DBL:
((double*)cbuf)[i*N/2+j] = dzero;
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N/2+j] = zzero;
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N/2+j] = czero;
break;
default:
GA_Error("wrong data type", data_type);
}
for (k=0; k<N/2; k++) {
switch (data_type) {
case C_FLOAT:
((float*)cbuf)[i*N/2+j] += ((float*)abuf)[i*N/2+k]
*((float*)bbuf)[k*N/2+j];
break;
case C_DBL:
((double*)cbuf)[i*N/2+j] += ((double*)abuf)[i*N/2+k]
*((double*)bbuf)[k*N/2+j];
break;
case C_DCPL:
((DoubleComplex*)cbuf)[i*N/2+j].real +=
(((DoubleComplex*)abuf)[i*N/2+k].real
*((DoubleComplex*)bbuf)[k*N/2+j].real
-(((DoubleComplex*)abuf)[i*N/2+k].imag
*((DoubleComplex*)bbuf)[k*N/2+j].imag));
((DoubleComplex*)cbuf)[i*N/2+j].imag +=
(((DoubleComplex*)abuf)[i*N/2+k].real
*((DoubleComplex*)bbuf)[k*N/2+j].imag
+(((DoubleComplex*)abuf)[i*N/2+k].imag
*((DoubleComplex*)bbuf)[k*N/2+j].real));
break;
case C_SCPL:
((SingleComplex*)cbuf)[i*N/2+j].real +=
(((SingleComplex*)abuf)[i*N/2+k].real
*((SingleComplex*)bbuf)[k*N/2+j].real
-(((SingleComplex*)abuf)[i*N/2+k].imag
*((SingleComplex*)bbuf)[k*N/2+j].imag));
((SingleComplex*)cbuf)[i*N/2+j].imag +=
(((SingleComplex*)abuf)[i*N/2+k].real
*((SingleComplex*)bbuf)[k*N/2+j].imag
+(((SingleComplex*)abuf)[i*N/2+k].imag
*((SingleComplex*)bbuf)[k*N/2+j].real));
break;
default:
GA_Error("wrong data type", data_type);
}
}
}
}
NGA_Put(g_a,lo,hi,cbuf,&ld);
}
if (me == 0) printf("\n\n\n\n");
/* Get norm of g_a */
switch (data_type) {
case C_FLOAT:
ftmp = NGA_Fdot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
case C_DBL:
dtmp = NGA_Ddot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
case C_DCPL:
ztmp = NGA_Zdot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
case C_SCPL:
ctmp = NGA_Cdot_patch(g_a,'N',lo,hi,g_a,'N',lo,hi);
break;
default:
GA_Error("wrong data type", data_type);
}
/* subtract C from A and put the results in B */
beta_flt = -1.0;
beta_dbl = -1.0;
beta_scpl.real = -1.0;
beta_dcpl.real = -1.0;
NGA_Zero_patch(g_b,lo,hi);
NGA_Add_patch(alpha,g_a,lo,hi,beta,g_c,lo,hi,g_b,lo,hi);
/* evaluate the norm of the difference between the two matrices */
switch (data_type) {
case C_FLOAT:
fdiff = NGA_Fdot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (ftmp != 0.0) {
fdiff /= ftmp;
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(fdiff), TOLERANCE);
GA_Error("GA_Sgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Sgemm OK\n\n");
}
break;
case C_DBL:
ddiff = NGA_Ddot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (dtmp != 0.0) {
ddiff /= dtmp;
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(ddiff), TOLERANCE);
GA_Error("GA_Dgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Dgemm OK\n\n");
}
break;
case C_DCPL:
zdiff = NGA_Zdot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (ztmp.real != 0.0 || ztmp.imag != 0.0) {
ddiff = sqrt((zdiff.real*zdiff.real+zdiff.imag*zdiff.imag)
/(ztmp.real*ztmp.real+ztmp.imag*ztmp.imag));
} else {
ddiff = sqrt(zdiff.real*zdiff.real+zdiff.imag*zdiff.imag);
}
if(fabs(ddiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(zdiff.real), TOLERANCE);
GA_Error("GA_Zgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Zgemm OK\n\n");
}
break;
case C_SCPL:
cdiff = NGA_Cdot_patch(g_b,'N',lo,hi,g_b,'N',lo,hi);
if (ctmp.real != 0.0 || ctmp.imag != 0.0) {
fdiff = sqrt((cdiff.real*cdiff.real+cdiff.imag*cdiff.imag)
/(ctmp.real*ctmp.real+ctmp.imag*ctmp.imag));
} else {
fdiff = sqrt(cdiff.real*cdiff.real+cdiff.imag*cdiff.imag);
}
if(fabs(fdiff) > TOLERANCE) {
printf("\nabs(result) = %f > %f\n", fabsf(cdiff.real), TOLERANCE);
GA_Error("GA_Cgemm Failed", 1);
} else if (me == 0) {
printf("\nGA_Cgemm OK\n\n");
}
break;
default:
GA_Error("wrong data type", data_type);
}
}
#endif
free(abuf);
free(bbuf);
free(cbuf);
GA_Destroy(g_a);
GA_Destroy(g_b);
GA_Destroy(g_c);
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, local_C[DIM][DIM], dims[DIM]={5,5}, val1=5, val2=4, alpha=3, beta=2, ld=5;
int alo[DIM]={2,2}, ahi[DIM]={3,3}, blo[DIM]={2,2}, bhi[DIM]={3,3}, clo[DIM]={1,1}, chi[DIM]={2,2};
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = GA_Duplicate(g_A, "array_B");
g_C = GA_Duplicate(g_A, "array_C");
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
GA_Zero(g_C);
NGA_Add_patch(&alpha, g_A, clo, chi, &beta, g_B, blo, bhi, g_C, clo, chi);
GA_Sync();
GA_Print(g_A);
GA_Print(g_B);
GA_Print(g_C);
NGA_Get(g_C, clo, chi, local_C, &ld);
//printf("check 1 \n");
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
if(rank == 0)
{
printf("check 2\n");
for(i=0; i<DIM; i++)
{
for(j=0; j<DIM; j++)
if(local_C[i][j]!=(alpha*val1)+(beta*val2)) printf("GA Error : \n");
}
}
if(rank==0)
GA_PRINT_MSG();
GA_Sync();
/*
GA_Destroy(g_A);
GA_Destroy(g_B);
GA_Destroy(g_C);
*/
//*******************************************************************
/* what would be the possible reason for GA_destroy to get failed ..,
* solve this before consolidate the whole
*/
GA_Terminate();
MPI_Finalize();
}
void do_work()
{
int ONE=1 ; /* useful constants */
int g_a, g_b;
int n=N, type=MT_F_DBL;
int me=GA_Nodeid(), nproc=GA_Nnodes();
int i, row;
int dims[2]={N,N};
int lo[2], hi[2], ld;
/* Note: on all current platforms DoublePrecision == double */
double buf[N], err, alpha, beta;
if(me==0)printf("Creating matrix A\n");
g_a = NGA_Create(type, 2, dims, "A", NULL);
if(!g_a) GA_Error("create failed: A",n);
if(me==0)printf("OK\n");
if(me==0)printf("Creating matrix B\n");
/* create matrix B so that it has dims and distribution of A*/
g_b = GA_Duplicate(g_a, "B");
if(! g_b) GA_Error("duplicate failed",n);
if(me==0)printf("OK\n");
GA_Zero(g_a); /* zero the matrix */
if(me==0)printf("Initializing matrix A\n");
/* fill in matrix A with random values in range 0.. 1 */
lo[1]=0; hi[1]=n-1;
for(row=me; row<n; row+= nproc){
/* each process works on a different row in MIMD style */
lo[0]=hi[0]=row;
for(i=0; i<n; i++) buf[i]=sin((double)i + 0.1*(row+1));
NGA_Put(g_a, lo, hi, buf, &n);
}
if(me==0)printf("Symmetrizing matrix A\n");
GA_Symmetrize(g_a); /* symmetrize the matrix A = 0.5*(A+A') */
/* check if A is symmetric */
if(me==0)printf("Checking if matrix A is symmetric\n");
GA_Transpose(g_a, g_b); /* B=A' */
alpha=1.; beta=-1.;
GA_Add(&alpha, g_a, &beta, g_b, g_b); /* B= A - B */
err= GA_Ddot(g_b, g_b);
if(me==0)printf("Error=%f\n",(double)err);
if(me==0)printf("\nChecking atomic accumulate \n");
GA_Zero(g_a); /* zero the matrix */
for(i=0; i<n; i++) buf[i]=(double)i;
/* everybody accumulates to the same location/row */
alpha = 1.0;
row = n/2;
lo[0]=hi[0]=row;
lo[1]=0; hi[1]=n-1;
ld = hi[1]-lo[1]+1;
NGA_Acc(g_a, lo, hi, buf, &ld, &alpha );
GA_Sync();
if(me==0){ /* node 0 is checking the result */
NGA_Get(g_a, lo, hi, buf,&ld);
for(i=0; i<n; i++) if(buf[i] != (double)nproc*i)
GA_Error("failed: column=",i);
printf("OK\n\n");
}
GA_Destroy(g_a);
GA_Destroy(g_b);
}
main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, g_C, **local_C=NULL, dims[DIM]={SIZE,SIZE}, val1=5, val2=4, alpha=3, beta=2;
int clo[DIM]={SIZE-SIZE,SIZE-SIZE}, chi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
int **local_tm=NULL;
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
local_C=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_C[i]=(int*)malloc(SIZE*sizeof(int));
local_tm=(int**)malloc(SIZE*sizeof(int*));
for(i=0; i<SIZE; i++)
local_tm[i]=(int*)malloc(SIZE*sizeof(int));
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
g_B = GA_Duplicate(g_A, "array_B");
g_C = GA_Duplicate(g_A, "array_C");
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
GA_Add(&alpha, g_A, &beta, g_B, g_C);
GA_Sync();
GA_Print(g_A);
GA_Print(g_B);
GA_Print(g_C);
//printf("check 1\n");
NGA_Get(g_C, clo, chi, local_tm, &ld);
//printf("check 2\n");
// GA_Sync();
if(rank==0)
{
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)printf("%d ", local_tm[i][j]);
printf("\n");
}
}
/*
if(rank==0)
{
NGA_Get(g_C, clo, chi, local_C, &ld);
printf("check 1 \n");
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)printf("%d ", local_C[i][j]);
printf("\n");
}
printf("check 2\n");
for(i=0; i<SIZE; i++)
{
for(j=0; j<SIZE; j++)
if(local_C[i][j]!=(alpha*val1)+(beta*val2)) printf("GA Error : \n");
}
}
*/
//GA_Sync();
if(rank==0)
printf("Test Completed \n");
// GA_Sync();
/*
GA_Destroy(g_A);
GA_Destroy(g_B);
GA_Destroy(g_C);
*/
//*******************************************************************
/* what would be the possible reason for GA_destroy to get failed ..,
* solve this before consolidate the whole
*/
GA_Terminate();
MPI_Finalize();
}
/* input is matrix size */
void ga_lu(double *A, int matrix_size)
{
int g_a, g_b, dims[2], type=C_DBL;
int lo[2], hi[2], ld;
int block_size[2], proc_grid[2];
double time, gflops;
/* create a 2-d GA (global matrix) */
dims[0] = matrix_size;
dims[1] = matrix_size;
block_size[0] = BLOCK_SIZE;
block_size[1] = BLOCK_SIZE;
#ifdef USE_SCALAPACK_DISTR
proc_grid[0] = 2;
proc_grid[1] = nprocs/2;
if(nprocs%2) GA_Error("For ScaLAPACK stle distribution, nprocs must be "
" divisible by 2", 0);
#endif
#ifndef BLOCK_CYCLIC
g_a = NGA_Create(type, 2, dims, "A", NULL);
g_b = GA_Duplicate(g_a, "transposed array B");
#else
g_a = GA_Create_handle();
GA_Set_data(g_a, 2, dims, type);
GA_Set_array_name(g_a,"A");
# ifdef USE_SCALAPACK_DISTR
GA_Set_block_cyclic_proc_grid(g_a, block_size, proc_grid);
# else
GA_Set_block_cyclic(g_a, block_size);
# endif
GA_Allocate(g_a);
g_b = GA_Create_handle();
GA_Set_data(g_b, 2, dims, type);
GA_Set_array_name(g_b,"B");
# ifdef USE_SCALAPACK_DISTR
GA_Set_block_cyclic_proc_grid(g_b, block_size, proc_grid);
# else
GA_Set_block_cyclic(g_b, block_size);
# endif
GA_Allocate(g_b);
#endif
/* copy the local matrix into GA */
if(me==0)
{
lo[0] = 0;
hi[0] = matrix_size - 1;
lo[1] = 0;
hi[1] = matrix_size - 1;
ld = matrix_size;
NGA_Put(g_a, lo, hi, A, &ld);
}
GA_Sync();
GA_Transpose(g_a, g_b);
time = CLOCK_();
GA_Lu('n', g_b);
time = CLOCK_() - time;
/* 2/3 N^3 - 1/2 N^2 flops for LU and 2*N^2 for solver */
gflops = ( (((double)matrix_size) * matrix_size)/(time*1.0e+9) *
(2.0/3.0 * (double)matrix_size - 0.5) );
if(me==0) printf("\nGA_Lu: N=%d flops=%2.5e Gflops, time=%2.5e secs\n\n",
matrix_size, gflops, time);
#if DEBUG
GA_Print(g_a);
GA_Print(g_b);
#endif
/* if(me==0) lu(A, matrix_size); */
GA_Destroy(g_a);
GA_Destroy(g_b);
}
/* Square matrix-matrix multiplication */
void matrix_multiply(int M, int N, int K,
int blockX_len, int blockY_len)
{
/* Local buffers and Global arrays declaration */
double *a=NULL, *b=NULL, *c=NULL;
int dims[NDIMS], ld[NDIMS], chunks[NDIMS];
int lo[NDIMS], hi[NDIMS], cdims[NDIMS]; /* dim of blocks */
int g_a, g_b, g_c, g_cnt, g_cnt2;
int offset;
double alpha = 1.0, beta=0.0;
int count_p = 0, next_p = 0;
int count_gac = 0, next_gac = 0;
double t1,t2,seconds;
ga_nbhdl_t nbh;
int count_acc = 0;
/* Find local processor ID and the number of processes */
int proc=GA_Nodeid(), nprocs=GA_Nnodes();
if ((M % blockX_len) != 0 || (M % blockY_len) != 0 || (N % blockX_len) != 0 || (N % blockY_len) != 0
|| (K % blockX_len) != 0 || (K % blockY_len) != 0)
GA_Error("Dimension size M/N/K is not divisible by X/Y block sizes", 101);
/* Allocate/Set process local buffers */
a = malloc (blockX_len * blockY_len * sizeof(double));
b = malloc (blockX_len * blockY_len * sizeof(double));
c = malloc (blockX_len * blockY_len * sizeof(double));
cdims[0] = blockX_len;
cdims[1] = blockY_len;
/* Configure array dimensions */
for(int i = 0; i < NDIMS; i++) {
dims[i] = N;
chunks[i] = -1;
ld[i] = cdims[i]; /* leading dimension/stride of the local buffer */
}
/* create a global array g_a and duplicate it to get g_b and g_c*/
g_a = NGA_Create(C_DBL, NDIMS, dims, "array A", chunks);
if (!g_a)
GA_Error("NGA_Create failed: A", NDIMS);
#if DEBUG>1
if (proc == 0)
printf(" Created Array A\n");
#endif
/* Ditto for C and B */
g_b = GA_Duplicate(g_a, "array B");
g_c = GA_Duplicate(g_a, "array C");
if (!g_b || !g_c)
GA_Error("GA_Duplicate failed",NDIMS);
if (proc == 0)
printf("Created Arrays B and C\n");
/* Subscript array for read-incr, which is nothing but proc */
int * rdcnt = malloc (nprocs * sizeof(int));
memset (rdcnt, 0, nprocs * sizeof(int));
int * rdcnt2 = malloc (nprocs * sizeof(int));
memset (rdcnt2, 0, nprocs * sizeof(int));
/* Create global array of nprocs elements for nxtval */
int counter_dim[1];
counter_dim[0] = nprocs;
g_cnt = NGA_Create(C_INT, 1, counter_dim, "Shared counter", NULL);
if (!g_cnt)
GA_Error("Shared counter failed",1);
g_cnt2 = GA_Duplicate(g_cnt, "another shared counter");
if (!g_cnt2)
GA_Error("Another shared counter failed",1);
GA_Zero(g_cnt);
GA_Zero(g_cnt2);
#if DEBUG>1
/* initialize data in matrices a and b */
if(proc == 0)
printf("Initializing local buffers - a and b\n");
#endif
int w = 0;
int l = 7;
for(int i = 0; i < cdims[0]; i++) {
for(int j = 0; j < cdims[1]; j++) {
a[i*cdims[1] + j] = (double)(++w%29);
b[i*cdims[1] + j] = (double)(++l%37);
}
}
/* Copy data to global arrays g_a and g_b from local buffers */
next_p = NGA_Read_inc(g_cnt2,&rdcnt[proc],(long)1);
for (int i = 0; i < N; i+=cdims[0])
{
if (next_p == count_p) {
for (int j = 0; j < N; j+=cdims[1])
{
/* Indices of patch */
lo[0] = i;
lo[1] = j;
hi[0] = lo[0] + cdims[0];
hi[1] = lo[1] + cdims[1];
hi[0] = hi[0]-1;
hi[1] = hi[1]-1;
#if DEBUG>1
printf ("%d: PUT_GA_A_B: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_Put(g_a, lo, hi, a, ld);
NGA_Put(g_b, lo, hi, b, ld);
}
next_p = NGA_Read_inc(g_cnt2,&rdcnt[proc],(long)1);
}
count_p++;
}
#if DEBUG>1
printf ("After NGA_PUT to global - A and B arrays\n");
#endif
/* Synchronize all processors to make sure puts from
nprocs has finished before proceeding with dgemm */
GA_Sync();
t1 = GA_Wtime();
next_gac = NGA_Read_inc(g_cnt,&rdcnt2[proc],(long)1);
for (int m = 0; m < N; m+=cdims[0])
{
for (int k = 0; k < N; k+=cdims[0])
{
if (next_gac == count_gac)
{
/* A = m x k */
lo[0] = m; lo[1] = k;
hi[0] = cdims[0] + lo[0]; hi[1] = cdims[1] + lo[1];
hi[0] = hi[0]-1; hi[1] = hi[1]-1;
#if DEBUG>3
printf ("%d: GET GA_A: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_Get(g_a, lo, hi, a, ld);
for (int n = 0; n < N; n+=cdims[1])
{
memset (c, 0, sizeof(double) * cdims[0] * cdims[1]);
/* B = k x n */
lo[0] = k; lo[1] = n;
hi[0] = cdims[0] + lo[0]; hi[1] = cdims[1] + lo[1];
hi[0] = hi[0]-1; hi[1] = hi[1]-1;
#if DEBUG>3
printf ("%d: GET_GA_B: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_Get(g_b, lo, hi, b, ld);
//_my_dgemm_ (a, local_N, b, local_N, c, local_N, local_N, local_N, local_N, alpha, beta=1.0);
/* TODO I am assuming square matrix blocks, further testing/work
required for rectangular matrices */
cblas_dgemm ( CblasRowMajor, CblasNoTrans, /* TransA */CblasNoTrans, /* TransB */
cdims[0] /* M */, cdims[1] /* N */, cdims[0] /* K */, alpha,
a, cdims[0], /* lda */ b, cdims[1], /* ldb */
beta=1.0, c, cdims[0] /* ldc */);
NGA_NbWait(&nbh);
/* C = m x n */
lo[0] = m; lo[1] = n;
hi[0] = cdims[0] + lo[0]; hi[1] = cdims[1] + lo[1];
hi[0] = hi[0]-1; hi[1] = hi[1]-1;
#if DEBUG>3
printf ("%d: ACC_GA_C: lo[0,1] = %d,%d and hi[0,1] = %d,%d\n",proc,lo[0],lo[1],hi[0],hi[1]);
#endif
NGA_NbAcc(g_c, lo, hi, c, ld, &alpha, &nbh);
count_acc += 1;
} /* END LOOP N */
next_gac = NGA_Read_inc(g_cnt,&rdcnt2[proc],(long)1);
} /* ENDIF if count == next */
count_gac++;
} /* END LOOP K */
} /* END LOOP M */
GA_Sync();
t2 = GA_Wtime();
seconds = t2 - t1;
if (proc == 0)
printf("Time taken for MM (secs):%lf \n", seconds);
printf("Number of ACC: %d\n", count_acc);
/* Correctness test - modify data again before this function */
for (int i = 0; i < NDIMS; i++) {
lo[i] = 0;
hi[i] = dims[i]-1;
ld[i] = dims[i];
}
verify(g_a, g_b, g_c, lo, hi, ld, N);
/* Clear local buffers */
free(a);
free(b);
free(c);
free(rdcnt);
free(rdcnt2);
GA_Sync();
/* Deallocate arrays */
GA_Destroy(g_a);
GA_Destroy(g_b);
GA_Destroy(g_c);
GA_Destroy(g_cnt);
GA_Destroy(g_cnt2);
}
void matrix_multiply() {
int dims[NDIM], chunk[NDIM], ld[NDIM];
int lo[NDIM], hi[NDIM], lo1[NDIM], hi1[NDIM];
int lo2[NDIM], hi2[NDIM], lo3[NDIM], hi3[NDIM];
int g_a, g_b, g_c, i, j, k, l;
int me, nprocs;
/* Find local processor ID and the number of processors */
/* ### assign processor ID to the int variable "me" and the total number
* ### of processors to the int variable "nprocs" */
me = GA_Nodeid();
nprocs = GA_Nnodes();
/* Configure array dimensions. Force an unequal data distribution */
for(i=0; i<NDIM; i++) {
dims[i] = TOTALELEMS;
ld[i] = dims[i];
chunk[i] = TOTALELEMS/nprocs-1; /*minimum block size on each process*/
}
/* create a global array g_a and duplicate it to get g_b and g_c*/
/* ### create GA of doubles with dimension "NDIM" and size "dims" with
* ### minimum block size "chunk" and assign the handle to the
* ### integer variable "g_a". Then create remaining global arrays,
* ### assigned to the integer handle "g_b" and "g_c" by duplicating
* ### g_a. Assign the names "Array A", "Array B" and "Array C" to
* ### "g_a", "g_b", and "g_c". */
g_a=NGA_Create(C_DBL, NDIM, dims, "array A", chunk);
if (!g_a) GA_Error("create failed: A", NDIM);
if (me==0) printf(" Created Array A\n");
g_b=GA_Duplicate(g_a,"array B");
g_c=GA_Duplicate(g_a,"array C");
if (!g_b || !g_c) GA_Error("duplicate failed",NDIM);
if (me==0) printf(" Created Arrays B and C\n");
/* initialize data in matrices a and b */
if(me==0)printf(" Initializing matrix A and B\n");
k = 0; l = 7;
for(i=0; i<dims[0]; i++) {
for(j=0; j<dims[1]; j++) {
a[i][j] = (double)(++k%29);
b[i][j] = (double)(++l%37);
}
}
/* Copy data to global arrays g_a and g_b */
lo1[0] = 0;
lo1[1] = 0;
hi1[0] = dims[0]-1;
hi1[1] = dims[1]-1;
if (me==0) {
/* ### copy the contents of array "a" into the portion of global array
* ### "g_a" described by "lo1" and "hi1". Similarly, copy the contents
* ### of the array "b" into corresponding portion of global array "g_b".
* ### Use the array of strides "ld" to describe the physical layout of
* ### arrays "a" and "b". */
NGA_Put(g_a,lo1,hi1,a,ld);
NGA_Put(g_b,lo1,hi1,b,ld);
}
/* Synchronize all processors to make sure everyone has data */
/* ### synchronize all processors */
GA_Sync();
/* Determine which block of data is locally owned. Note that
the same block is locally owned for all GAs. */
/* ### find out which block of data my node ("me") owns for the global
* ### array "g_c" and store the contents in the integer arrays "lo" and
* ### "hi". */
NGA_Distribution(g_c,me,lo,hi);
/* Get the blocks from g_a and g_b needed to compute this block in
g_c and copy them into the local buffers a and b. */
lo2[0] = lo[0];
lo2[1] = 0;
hi2[0] = hi[0];
hi2[1] = dims[0]-1;
/* ### copy the block of data described by the arrays "lo2" and "hi2" from
* ### the global array "g_a" into the local array "a". Use the array of
* ### strides "ld" to describe the physical layout of "a". */
NGA_Get(g_a,lo2,hi2,a,ld);
lo3[0] = 0;
lo3[1] = lo[1];
hi3[0] = dims[1]-1;
hi3[1] = hi[1];
/* ### copy the block of data described by the arrays "lo3" and "hi3" from
* ### the global array "g_b" into the local array "b". Use the array of
* ### strides "ld" to describe the physical layout of "b". */
NGA_Get(g_b,lo3,hi3,b,ld);
/* Do local matrix multiplication and store the result in local
buffer c. Start by evaluating the transpose of b. */
for(i=0; i < hi3[0]-lo3[0]+1; i++)
for(j=0; j < hi3[1]-lo3[1]+1; j++)
btrns[j][i] = b[i][j];
/* Multiply a and b to get c */
for(i=0; i < hi[0] - lo[0] + 1; i++) {
for(j=0; j < hi[1] - lo[1] + 1; j++) {
c[i][j] = 0.0;
for(k=0; k<dims[0]; k++)
c[i][j] = c[i][j] + a[i][k]*btrns[j][k];
}
}
/* Copy c back to g_c */
/* ### copy data from the local array "c" into the block of the global
* ### array "g_c" described by the integer arrays "lo" and "hi". Use
* ### the array of strides "ld" to describe the physical layout of "c". */
NGA_Put(g_c,lo,hi,c,ld);
verify(g_a, g_b, g_c, lo1, hi1, ld);
/* Deallocate arrays */
/* ### destroy the global arrays "g_a", "g_b", "g_c" */
GA_Destroy(g_a);
GA_Destroy(g_b);
GA_Destroy(g_c);
}
void TRANSPOSE1D() {
int dims[MAXDIM], chunk[MAXDIM], ld[MAXDIM], lo[MAXDIM], hi[MAXDIM];
int lo1[MAXDIM], hi1[MAXDIM], lo2[MAXDIM], hi2[MAXDIM];
int g_a, g_b, a[MAXPROC*TOTALELEMS],b[MAXPROC*TOTALELEMS];
int nelem, i;
int me, nprocs;
/* Find local processor ID and number of processors */
/* ### assign the local processor ID to the int variable "me"
* ### and the total number of processors to the int variable
* ### "nprocs" */
me = GA_Nodeid();
nprocs = GA_Nnodes();
/* Configure array dimensions. Force an unequal data distribution */
dims[0] = nprocs*TOTALELEMS + nprocs/2;
ld[0] = dims[0];
chunk[0] = TOTALELEMS; /* minimum data on each process */
/* create a global array g_a and duplicate it to get g_b */
/* ### create GA of integers with dimension "NDIM" and size "dims" with
* ### minimum block size "chunk" and assign the handle to the
* ### integer variable "g_a". Then create a second global array
* ### assigned to the integer handle "g_b" by duplicating "g_a".
* ### Assign the names "Array A" and "Array B" to "g_a" and "g_b". */
g_a=NGA_Create(C_INT, 1, dims, "array A", chunk);
g_b=GA_Duplicate(g_a,"array B");
if (!g_a) GA_Error("create failed: A", NDIM);
if (me==0) printf(" Created Array A\n");
if (! g_b) GA_Error("duplicate failed",NDIM);
if (me==0) printf(" Created Array B\n");
/* initialize data in g_a */
if (me==0) {
printf(" Initializing matrix A\n");
for(i=0; i<dims[0]; i++) a[i] = i;
lo[0] = 0;
hi[0] = dims[0]-1;
/* ### copy the contents of array "a" into the portion of global array
* ### "g_a" described by "lo" and "hi". Use the array of strides
* ### "ld" to describe the physical layout of array "a". */
NGA_Put(g_a,lo,hi,a,ld);
}
/* Synchronize all processors to guarantee that everyone has data
before proceeding to the next step. */
/* ### synchronize all processors */
GA_Sync();
/* Start initial phase of inversion by inverting the data held locally on
each processor. Start by finding out which data each processor owns. */
/* ### find out which block of data my node ("me") owns for the global
* ### array "g_a" and store the contents in the integer arrays "lo1" and
* ### "hi1". */
NGA_Distribution(g_a,me,lo1,hi1);
/* Get locally held data and copy it into local buffer a */
/* ### use the arrays "lo1" and "hi1" to copy the locally held block of data
* ### from the global array "g_a" into the local array "a". Use the array
* ### of strides "ld" to describe the physical layout of "a". */
NGA_Get(g_a,lo1,hi1,a,ld);
/* Invert data locally */
nelem = hi1[0] - lo1[0] + 1;
for (i=0; i<nelem; i++) b[i] = a[nelem-1-i];
/* Invert data globally by copying locally inverted blocks into
* their inverted positions in the GA */
lo2[0] = dims[0] - hi1[0] -1;
hi2[0] = dims[0] - lo1[0] -1;
/* ### copy data from the local array "b" into the block of the global
* ### array "g_b" described by the integer arrays "lo2" and "hi2". Use
* ### the array of strides "ld" to describe the physical layout of "b". */
NGA_Put(g_b,lo2,hi2,b,ld);
/* Synchronize all processors to make sure inversion is complete */
/* ### synchronize all processors */
GA_Sync();
/* Check to see if inversion is correct */
if(me == 0) verify(g_a, g_b);
/* Deallocate arrays */
/* ### destroy global arrays "g_a" and "g_b" */
GA_Destroy(g_a);
GA_Destroy(g_b);
}
void TRANSPOSE1D() {
int ndim, dims[1], chunk[1], ld[1], lo[1], hi[1];
int lo1[1], hi1[1], lo2[1], hi2[1];
int g_a, g_b, a[MAXPROC*TOTALELEMS],b[MAXPROC*TOTALELEMS];
int nelem, i, q;
int me, nprocs;
/* Find local processor ID and number of processors */
me = GA_Nodeid();
nprocs = GA_Nnodes();
/* Configure array dimensions. Force an unequal data distribution */
ndim = 1; /* 1-d transpose */
dims[0] = nprocs*TOTALELEMS + nprocs/2;
ld[0] = dims[0];
chunk[0] = TOTALELEMS; /* minimum data on each process */
/* create a global array g_a and duplicate it to get g_b */
g_a = NGA_Create(C_INT, 1, dims, "array A", chunk);
if (!g_a) GA_Error("create failed: A", 0);
g_b = GA_Duplicate(g_a, "array B");
if (! g_b) GA_Error("duplicate failed", 0);
/* initialize data in g_a */
if (me==0) {
for(i=0; i<dims[0]; i++) a[i] = i;
lo[0] = 0;
hi[0] = dims[0]-1;
NGA_Put(g_a, lo, hi, a, ld);
}
/* Synchronize all processors to guarantee that everyone has data
before proceeding to the next step. */
GA_Sync();
/* Start initial phase of inversion by inverting the data held locally on
each processor. Start by finding out which data each processor owns. */
NGA_Distribution(g_a, me, lo1, hi1);
/* Get locally held data and copy it into local buffer a */
NGA_Get(g_a, lo1, hi1, a, ld);
/* Invert data locally */
nelem = hi1[0] - lo1[0] + 1;
for (i=0; i<nelem; i++) b[i] = a[nelem-1-i];
/* Invert data globally by copying locally inverted blocks into
* their inverted positions in the GA */
lo2[0] = dims[0] - hi1[0] -1;
hi2[0] = dims[0] - lo1[0] -1;
NGA_Put(g_b,lo2,hi2,b,ld);
/* Synchronize all processors to make sure inversion is complete */
GA_Sync();
/* Deallocate arrays */
GA_Destroy(g_a);
GA_Destroy(g_b);
}
void do_work()
{
int g_a, g_b;
int me=GA_Nodeid(), nproc=GA_Nnodes(), proc, loop;
int dims[NDIM], lo[NDIM], hi[NDIM], block[NDIM], ld[NDIM-1];
int i,d,*proclist, offset;
int adims[NDIM], ndim,type;
typedef struct {
int lo[NDIM];
int hi[NDIM];
} patch_t;
patch_t *regions;
int *map;
double *buf;
/***** create array A with default distribution *****/
if(me==0){printf("Creating array A\n"); fflush(stdout);}
for(i = 0; i<NDIM; i++)dims[i] = N*(i+1);
#ifdef NEW_API
g_a = GA_Create_handle();
GA_Set_data(g_a,NDIM,dims,MT_F_DBL);
GA_Set_array_name(g_a,"array A");
(void)GA_Allocate(g_a);
#else
g_a = NGA_Create(MT_F_DBL, NDIM, dims, "array A", NULL);
#endif
if(!g_a) GA_Error("create failed: A",0);
if(me==0)printf("OK\n\n");
/* print info about array we got */
NGA_Inquire(g_a, &type, &ndim, adims);
GA_Print_distribution(g_a);
GA_Sync();
/* duplicate array A with ga_create irreg rather than ga_duplicate
* -- want to show distribution control
* -- with ga_duplicate it would be g_b=GA_Duplicate(g_a,name)
*/
if(me==0)printf("\nReconstructing distribution description for A\n");
/* get memory for arrays describing distribution */
proclist = (int*)malloc(nproc*sizeof(int));
if(!proclist)GA_Error("malloc failed for proclist",0);
regions = (patch_t*)malloc(nproc*sizeof(patch_t));
if(!regions)GA_Error("malloc failed for regions",0);
map = (int*)malloc((nproc+ndim)*sizeof(int)); /* ubound= nproc+mdim */
if(!map)GA_Error("malloc failed for map",0);
/* first find out how array g_a is distributed */
for(i=0;i<ndim;i++)lo[i]=BASE;
for(i=0;i<ndim;i++)hi[i]=adims[i] -1 + BASE;
proc = NGA_Locate_region(g_a, lo, hi, (int*)regions, proclist);
if(proc<1) GA_Error("error in NGA_Locate_region",proc);
/* determine blocking for each dimension */
for(i=0;i<ndim;i++)block[i]=0;
for(i=0;i<ndim;i++)adims[i]=0;
offset =0;
for(d=0; d<ndim; d++)
for(i=0;i<proc;i++)
if( regions[i].hi[d]>adims[d] ){
map[offset] = regions[i].lo[d];
offset++;
block[d]++;
adims[d]= regions[i].hi[d];
}
if(me==0){
printf("Distribution map contains %d elements\n",offset);
print_subscript("number of blocks for each dimension",ndim,block,"\n");
print_subscript("distribution map",offset,map,"\n\n");
fflush(stdout);
}
if(me==0)printf("Creating array B applying distribution of A\n");
# ifdef USE_DUPLICATE
g_b = GA_Duplicate(g_a,"array B");
# else
g_b = NGA_Create_irreg(MT_F_DBL, NDIM, dims, "array B", block,map);
# endif
if(!g_b) GA_Error("create failed: B",0);
if(me==0)printf("OK\n\n");
free(proclist); free(regions); free(map);
GA_Print_distribution(g_b);
GA_Sync();
if(me==0){
printf("\nCompare distributions of A and B\n");
if(GA_Compare_distr(g_a,g_b))
printf("Failure: distributions NOT identical\n");
else
printf("Success: distributions identical\n");
fflush(stdout);
}
if(me==0){
printf("\nAccessing local elements of A: set them to the owner process id\n");
fflush(stdout);
}
GA_Sync();
NGA_Distribution(g_a,me,lo,hi);
if(hi[0]>=0){/* -1 means no elements stored on this processor */
double *ptr;
int locdim[NDIM];
NGA_Access(g_a, lo,hi, &ptr, ld);
for(i=0;i<ndim;i++)locdim[i]=hi[i]-lo[i]+1;
fill_patch(ptr, locdim, ld, ndim,(double)me);
}
for(i=0;i<nproc; i++){
if(me==i && hi[0]>=0){
char msg[100];
sprintf(msg,"%d: leading dimensions",me);
print_subscript(msg,ndim-1,ld,"\n");
fflush(stdout);
}
GA_Sync();
}
GA_Sync();
if(me==0)printf("\nRandomly checking the update using ga_get on array sections\n");
GA_Sync();
/* show ga_get working and verify array updates
* every process does N random gets
* for simplicity get only a single row at a time
*/
srand(me); /* different seed for every process */
hi[ndim-1]=adims[ndim-1] -1 + BASE;
for(i=1;i<ndim-1; i++)ld[i]=1; ld[ndim-2]=adims[ndim-1] -1 + BASE;
/* get buffer memory */
buf = (double*)malloc(adims[ndim-1]*sizeof(double));
if(!buf)GA_Error("malloc failed for buf",0);
/* half of the processes check the result */
if(me<=nproc/2)
for(loop = 0; loop< N; loop++){ /* task parallel loop */
lo[ndim-1]=BASE;
for (i= 0; i < ndim -1; i ++){
lo[i] = hi[i] = rand()%adims[i]+BASE;
}
/* print_subscript("getting",ndim,lo,"\n");*/
NGA_Get(g_a,lo,hi,buf,ld);
/* check values */
for(i=0;i<adims[ndim-1]; i++){
int p = NGA_Locate(g_a, lo);
if((double)p != buf[i]) {
char msg[100];
sprintf(msg,"%d: wrong value: %d != %lf a",me, p, buf[i]);
print_subscript(msg,ndim,lo,"\n");
GA_Error("Error - bye",i);
}
lo[ndim-1]++;
}
}
free(buf);
GA_Sync();
if(me==0)printf("OK\n");
GA_Destroy(g_a);
GA_Destroy(g_b);
}
int main(int argc, char **argv)
{
int rank, nprocs, i, j;
int g_A, g_B, dims[DIM]={SIZE,SIZE}, val1=5, val2=4;
int lo[DIM]={SIZE-SIZE,SIZE-SIZE}, hi[DIM]={SIZE-1,SIZE-1}, ld=SIZE;
#if defined(USE_ELEMENTAL)
// initialize Elemental (which will initialize MPI)
ElInitialize( &argc, &argv );
ElMPICommRank( MPI_COMM_WORLD, &rank );
ElMPICommSize( MPI_COMM_WORLD, &nprocs );
// instantiate el::global array
ElGlobalArraysConstruct_i( &eliga );
// initialize global arrays
ElGlobalArraysInitialize_i( eliga );
#else
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
MA_init(C_INT, 1000, 1000);
GA_Initialize();
#endif
// create global arrays
#if defined(USE_ELEMENTAL)
ElGlobalArraysCreate_i( eliga, DIM, dims, "array_A", NULL, &g_A );
#else
g_A = NGA_Create(C_INT, DIM, dims, "array_A", NULL);
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysDuplicate_i( eliga, g_A, "array_B", &g_B );
#else
g_B = GA_Duplicate(g_A, "array_B");
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysFill_i( eliga, g_A, &val1 );
ElGlobalArraysFill_i( eliga, g_B, &val2 );
#else
GA_Fill(g_A, &val1);
GA_Fill(g_B, &val2);
#endif
int dot_AB = -99;
#if defined(USE_ELEMENTAL)
ElGlobalArraysDot_i( eliga, g_A, g_B, &dot_AB );
#else
dot_AB = GA_Idot(g_A, g_B);
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
#else
GA_Sync();
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysPrint_i( eliga, g_A );
ElGlobalArraysPrint_i( eliga, g_B );
#else
GA_Print(g_A);
GA_Print(g_B);
#endif
// Check
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
#else
GA_Sync();
#endif
if(rank==0)
printf ("Integer dot product of g_A and g_B: %d\n", dot_AB);
#if defined(USE_ELEMENTAL)
ElGlobalArraysSync_i( eliga );
#else
GA_Sync();
#endif
if(rank==0)
printf("Test Completed \n");
#if defined(USE_ELEMENTAL)
ElGlobalArraysDestroy_i( eliga, g_A );
ElGlobalArraysDestroy_i( eliga, g_B );
#else
GA_Destroy(g_A);
GA_Destroy(g_B);
#endif
#if defined(USE_ELEMENTAL)
ElGlobalArraysTerminate_i( eliga );
// call el::global arrays destructor
ElGlobalArraysDestruct_i( eliga );
ElFinalize();
#else
GA_Terminate();
MPI_Finalize();
#endif
return 0;
}
void matrix_multiply() {
int dims[NDIM], chunk[NDIM], ld[NDIM];
int lo[NDIM], hi[NDIM], lo1[NDIM], hi1[NDIM];
int lo2[NDIM], hi2[NDIM], lo3[NDIM], hi3[NDIM];
int g_a, g_b, g_c;
int i, j, k;
double start, end;
/* Find local processor ID and the number of processors */
int me=GA_Nodeid(), nprocs=GA_Nnodes();
/* Configure array dimensions */
for(i=0; i<NDIM; i++) {
dims[i] = TOTALELEMS;
ld[i] = dims[i];
chunk[i] = TOTALELEMS/nprocs-1; /*minimum block size on each process*/
}
/* Create a global array g_a and duplicate it to get g_b and g_c*/
g_a = NGA_Create(C_DBL, NDIM, dims, "array A", chunk);
if (!g_a) GA_Error("create failed: A", NDIM);
g_b = GA_Duplicate(g_a, "array B");
g_c = GA_Duplicate(g_a, "array C");
if (!g_b || !g_c) GA_Error("duplicate failed",NDIM);
/* Initialize data in matrices a and b */
for(i=0; i<dims[0]; i++) {
for(j=0; j<dims[1]; j++) {
a[i][j] = i+j;
b[i][j] = i*j;
}
}
/* Copy data to global arrays g_a and g_b */
lo1[0] = 0;
lo1[1] = 0;
hi1[0] = dims[0]-1;
hi1[1] = dims[1]-1;
if (me==0) {
NGA_Put(g_a, lo1, hi1, a, ld);
NGA_Put(g_b, lo1, hi1, b, ld);
}
/* Synchronize all processors to make sure everyone has data */
GA_Sync();
/* Determine which block of data is locally owned. Note that
the same block is locally owned for all GAs. */
NGA_Distribution(g_c, me, lo, hi);
/* Get the blocks from g_a and g_b needed to compute this block in
g_c and copy them into the local buffers a and b. */
lo2[0] = lo[0];
lo2[1] = 0;
hi2[0] = hi[0];
hi2[1] = dims[0]-1;
NGA_Get(g_a, lo2, hi2, a, ld);
lo3[0] = 0;
lo3[1] = lo[1];
hi3[0] = dims[1]-1;
hi3[1] = hi[1];
NGA_Get(g_b, lo3, hi3, b, ld);
/* Do local matrix multiplication and store the result in local
buffer c. Start by evaluating the transpose of b. */
for(i=0; i < hi3[0]-lo3[0]+1; i++)
for(j=0; j < hi3[1]-lo3[1]+1; j++)
btrns[j][i] = b[i][j];
/* Multiply a and b to get c */
for(i=0; i < hi[0] - lo[0] + 1; i++) {
for(j=0; j < hi[1] - lo[1] + 1; j++) {
c[i][j] = 0.0;
for(k=0; k<dims[0]; k++)
c[i][j] = c[i][j] + a[i][k]*btrns[j][k];
}
}
/* Copy c back to g_c */
NGA_Put(g_c, lo, hi, c, ld);
/* Synchronize all processors to make sure inversion is complete */
GA_Sync();
/* Deallocate arrays */
GA_Destroy(g_a);
GA_Destroy(g_b);
GA_Destroy(g_c);
}
