void FATR ga_antisymmetrize_(Integer *g_a) { DoublePrecision alpha = 0.5; int i, me = GA_Nodeid(); extern void * FATR ga_malloc(Integer nelem, int type, char *name); extern void FATR ga_free(void *ptr); void FATR gai_subtr(int *lo, int *hi, void *a, void *b, DoublePrecision alpha, int type, Integer nelem, int ndim); int alo[GA_MAX_DIM], ahi[GA_MAX_DIM], lda[GA_MAX_DIM]; int blo[GA_MAX_DIM], bhi[GA_MAX_DIM], ldb[GA_MAX_DIM]; int ndim, dims[GA_MAX_DIM], type; Integer nelem=1; Logical have_data; void *a_ptr, *b_ptr; GA_Sync(); NGA_Inquire((int)(*g_a), &type, &ndim, dims); if (dims[0] != dims[1]) GA_Error("ga_sym: can only sym square matrix", 0L); /* Find the local distribution */ NGA_Distribution((int)(*g_a), me, alo, ahi); have_data = ahi[0]>=0; for(i=1; i<ndim; i++) have_data = have_data && ahi[i]>=0; if(have_data) { NGA_Access((int)(*g_a), alo, ahi, &a_ptr, lda); for(i=0; i<ndim; i++) nelem *= ahi[i]-alo[i] +1; b_ptr = (void *) ga_malloc(nelem, MT_C_DBL, "v"); for(i=2; i<ndim; i++) {bhi[i]=ahi[i]; blo[i]=alo[i]; } /* switch rows and cols */ blo[1]=alo[0]; bhi[1]=ahi[0]; blo[0]=alo[1]; bhi[0]=ahi[1]; for (i=0; i < ndim-1; i++) ldb[i] = bhi[i+1] - blo[i+1] + 1; NGA_Get((int)(*g_a), blo, bhi, b_ptr, ldb); } GA_Sync(); if(have_data) { gai_subtr(alo, ahi, a_ptr, b_ptr, alpha, type, nelem, ndim); NGA_Release_update((int)(*g_a), alo, ahi); ga_free(b_ptr); } GA_Sync(); }
PetscErrorCode testCreate2D() { int ga; DA da; DALocalInfo info; Vec vec; PetscErrorCode ierr; PetscFunctionBegin; int d1 = 1453, d2 = 1451; ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR, d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr); ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr); ierr = DACreateGlobalArray( da, &ga, &vec); CHKERRQ(ierr); PetscReal **v; ierr = DAVecGetArray(da,vec,&v); CHKERRQ(ierr); int xe = info.xs+info.xm, ye = info.ys+info.ym; for (int j = info.ys; j < ye; ++j) { for (int i = info.xs; i < xe; ++i) { v[j][i] = 1.*i + d1 * j; } } ierr = DAVecRestoreArray(da,vec,&v); CHKERRQ(ierr); PetscPrintf(PETSC_COMM_WORLD,"Updated local portion with DAVec\n"); PetscBarrier(0); { double *da_ptr; VecGetArray(vec, &da_ptr); double *ptr; int low[2],hi[2],ld; NGA_Distribution(ga,GA_Nodeid(),low,hi); NGA_Access(ga,low,hi,&ptr,&ld); printf("[%d] ga:%p\tda:%p\tdiff:%p\n", GA_Nodeid(), ptr, da_ptr, (ptr-da_ptr) ); NGA_Release_update(ga,low,hi); } int lo[2],ld; double val; for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { lo[0] = j; lo[1] = i; NGA_Get(ga,lo,lo,&val,&ld); if( PetscAbs( i + d1*j - val) > .1 ) printf("."); // printf("[%d] (%3.0f,%3.0f)\n", GA_Nodeid(), 1.*i + d1*j, val); } } GA_Print_stats(); ierr = VecDestroy(vec); CHKERRQ(ierr); GA_Destroy(ga); PetscFunctionReturn(0); }
PetscErrorCode vizGA2DA() { PetscErrorCode ierr; int rank; MPI_Comm_rank(PETSC_COMM_WORLD,&rank); int d1 = 40, d2 = 50; DA da; Vec vec; const PetscInt *lx, *ly, *lz; PetscInt m,n,p; DALocalInfo info; ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR, d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr); ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr); ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr); ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr); ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr); /**/ ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr); for (int i = 0; i < m; ++i) { PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]); } for (int i = 0; i < n; ++i) { PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]); } /**/ int ga = GA_Create_handle(); int ndim = 2; int dims[2] = {d2,d1}; GA_Set_data(ga,2,dims,MT_DBL); int *map; PetscMalloc( sizeof(int)*(m+n), &map); map[0] = 0; for( int i = 1; i < n; i++ ) { map[i] = ly[i-1] + map[i-1]; } map[n] = 0; for( int i = n+1; i < m+n; i++ ) { map[i] = lx[i-n-1] + map[i-1]; } /* correct ordering, but nodeid's dont line up with mpi rank for petsc's da * DA: +---+---+ GA: +---+---+ * +-2-+-3-+ +-1-+-3-+ * +---+---+ +---+---+ * +-0-+-1-+ +-0-+-2-+ * +---+---+ +---+---+ int *map; PetscMalloc( sizeof(int)*(m+n), &map); map[0] = 0; for( int i = 1; i < m; i++ ) { map[i] = lx[i] + map[i-1]; } map[m] = 0; for( int i = m+1; i < m+n; i++ ) { map[i] = ly[i-m] + map[i-1]; } */ int block[2] = {n,m}; GA_Set_irreg_distr(ga,map,block); ierr = GA_Allocate( ga ); if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr); if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr); if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1); GA_Print_distribution(ga); int lo[2], hi[2]; NGA_Distribution(ga,rank,lo,hi); if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 || lo[0] != info.ys || hi[0] != info.ys+info.ym-1 ) { PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n", rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1); } PetscBarrier(0); PetscSynchronizedFlush(PETSC_COMM_WORLD); AO ao; DAGetAO(da,&ao); if( rank == 0 ) { int *idx, len = d1*d2; PetscReal *val; PetscMalloc(sizeof(PetscReal)*len, &val); PetscMalloc(sizeof(int)*len, &idx); for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { idx[i + d1*j] = i + d1*j; val[i + d1*j] = i + d1*j; } } AOApplicationToPetsc(ao,len,idx); VecSetValues(vec,len,idx,val,INSERT_VALUES); int a[2], b[2],ld[1]={0}; double c = 0; for (int j = 0; j < d2; ++j) { for (int i = 0; i < d1; ++i) { a[0] = j; a[1] = i; // printf("%5.0f ",c); NGA_Put(ga,a,a,&c,ld); c++; } } } // GA_Print(ga); VecAssemblyBegin(vec); VecAssemblyEnd(vec); int ld; double *ptr; NGA_Access(ga,lo,hi,&ptr,&ld); PetscReal **d; int c=0; ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr); for (int j = info.ys; j < info.ys+info.ym; ++j) { for (int i = info.xs; i < info.xs+info.xm; ++i) { if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] ) GA_Error("DA array is not equal to GA array",1); // printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]); c++; } } ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr); c=0; PetscReal *v; int start, end; VecGetOwnershipRange(vec, &start, &end); VecGetArray( vec, &v ); for( int i = start; i < end; i++) { // printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") ); } VecRestoreArray( vec, &v ); NGA_Release_update(ga,lo,hi); Vec gada; VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada); VecView(gada,PETSC_VIEWER_STDOUT_SELF); GA_Destroy(ga); ierr = VecDestroy(vec); CHKERRQ(ierr); ierr = DADestroy(da); CHKERRQ(ierr); PetscFunctionReturn(0); }