// -------------------------------------------------------------
// CreateMatGA
// -------------------------------------------------------------
static
PetscErrorCode
CreateMatGA(int pgroup, int lrows, int lcols, int grows, int gcols, int *ga)
{
PetscErrorCode ierr = 0;
/* Try to honor local ownership request (of rows). */
int nprocs = GA_Pgroup_nnodes(pgroup);
int me = GA_Pgroup_nodeid(pgroup);
int tmapc[nprocs+1];
int mapc[nprocs+1];
int i;
for (i = 0; i < nprocs+1; i++) tmapc[i] = 0;
tmapc[me] = lrows;
GA_Pgroup_igop(pgroup, tmapc, nprocs+1, "+");
mapc[0] = 0;
for (i = 1; i < nprocs; i++) mapc[i] = mapc[i-1]+tmapc[i-1];
mapc[nprocs] = 0;
int dims[2] = {grows, gcols};
int blocks[2] = { nprocs, 1 };
*ga = GA_Create_handle();
GA_Set_data(*ga, 2, dims, MT_PETSC_SCALAR);
GA_Set_irreg_distr(*ga, mapc, blocks);
GA_Set_pgroup(*ga, pgroup);
if (!GA_Allocate(*ga)) {
ierr = 1;
}
PetscScalar z(0.0);
GA_Fill(*ga, &z);
return ierr;
}
PetscErrorCode vizGA2DA()
{
PetscErrorCode ierr;
int rank;
MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
int d1 = 40, d2 = 50;
DA da;
Vec vec;
const PetscInt *lx, *ly, *lz;
PetscInt m,n,p;
DALocalInfo info;
ierr = DACreate2d(PETSC_COMM_WORLD,DA_NONPERIODIC,DA_STENCIL_STAR,
d1,d2,PETSC_DECIDE,PETSC_DECIDE,1,1,0,0, &da); CHKERRQ(ierr);
ierr = DACreateGlobalVector(da, &vec); CHKERRQ(ierr);
ierr = DAGetOwnershipRanges(da, &lx, &ly, &lz); CHKERRQ(ierr);
ierr = DAGetLocalInfo(da,&info); CHKERRQ(ierr);
ierr = DAGetInfo(da,0,0,0,0,&m,&n,&p,0,0,0,0); CHKERRQ(ierr);
/**/
ierr = DAView(da, PETSC_VIEWER_STDOUT_WORLD); CHKERRQ(ierr);
for (int i = 0; i < m; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tlx: %d\n",i,lx[i]);
}
for (int i = 0; i < n; ++i) {
PetscPrintf(PETSC_COMM_WORLD,"%d\tly: %d\n",i,ly[i]);
}
/**/
int ga = GA_Create_handle();
int ndim = 2;
int dims[2] = {d2,d1};
GA_Set_data(ga,2,dims,MT_DBL);
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < n; i++ )
{
map[i] = ly[i-1] + map[i-1];
}
map[n] = 0;
for( int i = n+1; i < m+n; i++ )
{
map[i] = lx[i-n-1] + map[i-1];
}
/* correct ordering, but nodeid's dont line up with mpi rank for petsc's da
* DA: +---+---+ GA: +---+---+
* +-2-+-3-+ +-1-+-3-+
* +---+---+ +---+---+
* +-0-+-1-+ +-0-+-2-+
* +---+---+ +---+---+
int *map;
PetscMalloc( sizeof(int)*(m+n), &map);
map[0] = 0;
for( int i = 1; i < m; i++ )
{
map[i] = lx[i] + map[i-1];
}
map[m] = 0;
for( int i = m+1; i < m+n; i++ )
{
map[i] = ly[i-m] + map[i-1];
}
*/
int block[2] = {n,m};
GA_Set_irreg_distr(ga,map,block);
ierr = GA_Allocate( ga );
if( !ierr ) GA_Error("\n\n\nga allocaltion failed\n\n",ierr);
if( !ga ) GA_Error("\n\n\n ga null \n\n",ierr);
if( rank != GA_Nodeid() ) GA_Error("MPI rank does not match GA_Nodeid()",1);
GA_Print_distribution(ga);
int lo[2], hi[2];
NGA_Distribution(ga,rank,lo,hi);
if( lo[1] != info.xs || hi[1] != info.xs+info.xm-1 ||
lo[0] != info.ys || hi[0] != info.ys+info.ym-1 )
{
PetscSynchronizedPrintf(PETSC_COMM_SELF,"[%d] lo:(%2d,%2d) hi:(%2d,%2d) \t DA: (%2d,%2d), (%2d, %2d)\n",
rank, lo[1], lo[0], hi[1], hi[0], info.xs, info.ys, info.xs+info.xm-1, info.ys+info.ym-1);
}
PetscBarrier(0);
PetscSynchronizedFlush(PETSC_COMM_WORLD);
AO ao;
DAGetAO(da,&ao);
if( rank == 0 )
{
int *idx, len = d1*d2;
PetscReal *val;
PetscMalloc(sizeof(PetscReal)*len, &val);
PetscMalloc(sizeof(int)*len, &idx);
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
idx[i + d1*j] = i + d1*j;
val[i + d1*j] = i + d1*j;
}
}
AOApplicationToPetsc(ao,len,idx);
VecSetValues(vec,len,idx,val,INSERT_VALUES);
int a[2], b[2],ld[1]={0};
double c = 0;
for (int j = 0; j < d2; ++j)
{
for (int i = 0; i < d1; ++i)
{
a[0] = j;
a[1] = i;
// printf("%5.0f ",c);
NGA_Put(ga,a,a,&c,ld);
c++;
}
}
}
// GA_Print(ga);
VecAssemblyBegin(vec);
VecAssemblyEnd(vec);
int ld;
double *ptr;
NGA_Access(ga,lo,hi,&ptr,&ld);
PetscReal **d;
int c=0;
ierr = DAVecGetArray(da,vec,&d); CHKERRQ(ierr);
for (int j = info.ys; j < info.ys+info.ym; ++j)
{
for (int i = info.xs; i < info.xs+info.xm; ++i)
{
if( d[j][i] != ptr[(i-info.xs)+ld*(j-info.ys)] )
GA_Error("DA array is not equal to GA array",1);
// printf("%d (%d,%d):\t%3.0f\t%3.0f\n", c, i, j, d[j][i], ptr[(i-info.xs)+ld*(j-info.ys)]);
c++;
}
}
ierr = DAVecRestoreArray(da,vec,&d); CHKERRQ(ierr);
c=0;
PetscReal *v;
int start, end;
VecGetOwnershipRange(vec, &start, &end);
VecGetArray( vec, &v );
for( int i = start; i < end; i++)
{
// printf("%d:\t%3.0f\t%3.0f\t%s\n", start, v[i-start], ptr[i-start], (v[i-start]-ptr[i-start]==0?"":"NO") );
}
VecRestoreArray( vec, &v );
NGA_Release_update(ga,lo,hi);
Vec gada;
VecCreateMPIWithArray(((PetscObject)da)->comm,da->Nlocal,PETSC_DETERMINE,ptr,&gada);
VecView(gada,PETSC_VIEWER_STDOUT_SELF);
GA_Destroy(ga);
ierr = VecDestroy(vec); CHKERRQ(ierr);
ierr = DADestroy(da); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
/**
* Evaluate offsets for each network component
*/
void setOffsets(void)
{
// Interleave contributions from buses and branches to match matrices
int i,j,jdx,jdx1,jdx2;
int *i_bus_offsets = new int[p_nBuses];
int *i_branch_offsets = new int[p_nBranches];
for (i=0; i<p_nBuses; i++) {
i_bus_offsets[i] = 0;
}
for (i=0; i<p_nBranches; i++) {
i_branch_offsets[i] = 0;
}
int icnt = 0;
int nsize;
// Evaluate offsets for individual network components
for (i=0; i<p_nBuses; i++) {
if (p_network->getActiveBus(i)) {
i_bus_offsets[i] = icnt;
icnt += p_network->getBus(i)->vectorNumElements();
std::vector<int> nghbrs = p_network->getConnectedBranches(i);
nsize = nghbrs.size();
for (j=0; j<nsize; j++) {
// Need to avoid double counting of branches when evaluating offsets.
// If branch is non-local and it is active, then include it in offsets.
// Otherwise, if branch is local and bus i is equal to the "from" bus,
// then include it in the offsets.
jdx = nghbrs[j];
if (isLocalBranch(jdx)) {
p_network->getBranchEndpoints(jdx,&jdx1,&jdx2);
if (jdx1 == i) {
i_branch_offsets[jdx] = icnt;
icnt += p_network->getBranch(jdx)->vectorNumElements();
}
} else {
if (p_network->getActiveBranch(jdx)) {
i_branch_offsets[jdx] = icnt;
icnt += p_network->getBranch(jdx)->vectorNumElements();
}
}
}
}
}
// Total number of rows and columns from this processor have been evaluated,
// now create buffers that can scatter individual offsets to global arrays
int **i_bus_index = new int*[p_nBuses];
int **i_branch_index = new int*[p_nBranches];
int *i_bus_index_buf = new int[p_nBuses];
int *i_branch_index_buf = new int[p_nBranches];
int *i_bus_value_buf = new int[p_nBuses];
int *i_branch_value_buf = new int[p_nBranches];
int i_bus_cnt = 0;
int i_branch_cnt = 0;
int row_offset = p_Offsets[p_me];
int nbus = 0;
int nbranch = 0;
for (i=0; i<p_nBuses; i++) {
if (p_network->getActiveBus(i)) {
nbus++;
i_bus_value_buf[i_bus_cnt] = i_bus_offsets[i]+row_offset;
i_bus_index_buf[i_bus_cnt] = p_network->getGlobalBusIndex(i);
i_bus_index[i_bus_cnt] = &i_bus_index_buf[i_bus_cnt];
i_bus_cnt++;
}
}
for (i=0; i<p_nBranches; i++) {
if (p_network->getActiveBranch(i)) {
nbranch++;
i_branch_value_buf[i_branch_cnt] = i_branch_offsets[i]+row_offset;
i_branch_index_buf[i_branch_cnt] = p_network->getGlobalBranchIndex(i);
i_branch_index[i_branch_cnt] = &i_branch_index_buf[i_branch_cnt];
i_branch_cnt++;
}
}
delete [] i_bus_offsets;
delete [] i_branch_offsets;
// Create global arrays that hold column and row offsets for all buses and
// branches in the network. First create map array for global arrays
int *t_busMap = new int[p_nNodes];
int *t_branchMap = new int[p_nNodes];
for (i=0; i<p_nNodes; i++) {
t_busMap[i] = 0;
t_branchMap[i] = 0;
}
t_busMap[p_me] = nbus;
t_branchMap[p_me] = nbranch;
char plus[2];
strcpy(plus,"+");
GA_Pgroup_igop(p_GAgrp, t_busMap, p_nNodes, plus);
GA_Pgroup_igop(p_GAgrp, t_branchMap, p_nNodes, plus);
int *busMap = new int[p_nNodes];
int *branchMap = new int[p_nNodes];
busMap[0] = 0;
branchMap[0] = 0;
int total_buses = t_busMap[0];
int total_branches = t_branchMap[0];
for (i=1; i<p_nNodes; i++) {
busMap[i] = busMap[i-1] + t_busMap[i-1];
total_buses += t_busMap[i];
branchMap[i] = branchMap[i-1] + t_branchMap[i-1];
total_branches += t_branchMap[i];
}
delete [] t_busMap;
delete [] t_branchMap;
int one = 1;
g_bus_offsets = GA_Create_handle();
GA_Set_data(g_bus_offsets, one, &total_buses, C_INT);
GA_Set_irreg_distr(g_bus_offsets, busMap, &p_nNodes);
GA_Set_pgroup(g_bus_offsets, p_GAgrp);
if (!GA_Allocate(g_bus_offsets)) {
char buf[256];
sprintf(buf,"GenVectorMap::setOffsets: Unable to allocate distributed array for bus offsets\n");
printf("%s",buf);
throw gridpack::Exception(buf);
}
GA_Zero(g_bus_offsets);
g_branch_offsets = GA_Create_handle();
GA_Set_data(g_branch_offsets, one, &total_branches, C_INT);
GA_Set_irreg_distr(g_branch_offsets, branchMap, &p_nNodes);
GA_Set_pgroup(g_branch_offsets, p_GAgrp);
if (!GA_Allocate(g_branch_offsets)) {
char buf[256];
sprintf(buf,"GenVectorMap::setOffsets: Unable to allocate distributed array for branch offsets\n");
printf("%s",buf);
throw gridpack::Exception(buf);
}
GA_Zero(g_branch_offsets);
delete [] busMap;
delete [] branchMap;
// Scatter offsets to global arrays
NGA_Scatter(g_bus_offsets, i_bus_value_buf, i_bus_index, i_bus_cnt);
NGA_Scatter(g_branch_offsets, i_branch_value_buf, i_branch_index, i_branch_cnt);
NGA_Pgroup_sync(p_GAgrp);
delete [] i_bus_index;
delete [] i_branch_index;
delete [] i_bus_index_buf;
delete [] i_branch_index_buf;
delete [] i_bus_value_buf;
delete [] i_branch_value_buf;
}
