// -------------------------------------------------------------
// CreateMatGA
// -------------------------------------------------------------
static
PetscErrorCode
CreateMatGA(int pgroup, int lrows, int lcols, int grows, int gcols, int *ga)
{
PetscErrorCode ierr = 0;
/* Try to honor local ownership request (of rows). */
int nprocs = GA_Pgroup_nnodes(pgroup);
int me = GA_Pgroup_nodeid(pgroup);
int tmapc[nprocs+1];
int mapc[nprocs+1];
int i;
for (i = 0; i < nprocs+1; i++) tmapc[i] = 0;
tmapc[me] = lrows;
GA_Pgroup_igop(pgroup, tmapc, nprocs+1, "+");
mapc[0] = 0;
for (i = 1; i < nprocs; i++) mapc[i] = mapc[i-1]+tmapc[i-1];
mapc[nprocs] = 0;
int dims[2] = {grows, gcols};
int blocks[2] = { nprocs, 1 };
*ga = GA_Create_handle();
GA_Set_data(*ga, 2, dims, MT_PETSC_SCALAR);
GA_Set_irreg_distr(*ga, mapc, blocks);
GA_Set_pgroup(*ga, pgroup);
if (!GA_Allocate(*ga)) {
ierr = 1;
}
PetscScalar z(0.0);
GA_Fill(*ga, &z);
return ierr;
}
int main(int argc, char **argv)
{
int me;
int nproc;
int status;
int g_a;
int dims[NDIM];
int chunk[NDIM];
int pg_world;
size_t num = 10;
double *p1 = NULL;
double *p2 = NULL;
size_t i;
int num_mutex;
int lo[1];
int hi[1];
int ld[1]={1};
MPI_Comm comm;
MP_INIT(argc,argv);
GA_INIT(argc,argv);
me = GA_Nodeid();
nproc = GA_Nnodes();
comm = GA_MPI_Comm_pgroup_default();
printf("%d: Hello world!\n",me);
if (me==0) printf("%d: GA_Initialize\n",me);
/*if (me==0) printf("%d: ARMCI_Init\n",me);*/
/*ARMCI_Init();*/
/*if (me==0) printf("%d: MA_Init\n",me);*/
/*MA_init(MT_DBL, 8*1024*1024, 2*1024*1024);*/
if (me==0) printf("%d: GA_Create_handle\n",me);
g_a = GA_Create_handle();
if (me==0) printf("%d: GA_Set_array_name\n",me);
GA_Set_array_name(g_a,"test array A");
dims[0] = 30;
if (me==0) printf("%d: GA_Set_data\n",me);
GA_Set_data(g_a,NDIM,dims,MT_DBL);
chunk[0] = -1;
if (me==0) printf("%d: GA_Set_chunk\n",me);
GA_Set_chunk(g_a,chunk);
if (me==0) printf("%d: GA_Pgroup_get_world\n",me);
pg_world = GA_Pgroup_get_world();
if (me==0) printf("%d: GA_Set_pgroup\n",me);
GA_Set_pgroup(g_a,pg_world);
if (me==0) printf("%d: GA_Allocate\n",me);
status = GA_Allocate(g_a);
if(0 == status) MPI_Abort(comm,100);
if (me==0) printf("%d: GA_Zero\n",me);
GA_Zero(g_a);
if (me==0) printf("%d: GA_Sync\n",me);
GA_Sync();
num = 10;
p1 = malloc(num*sizeof(double));
/*double* p1 = ARMCI_Malloc_local(num*sizeof(double));*/
if (p1==NULL) MPI_Abort(comm,1000);
p2 = malloc(num*sizeof(double));
/*double* p2 = ARMCI_Malloc_local(num*sizeof(double));*/
if (p2==NULL) MPI_Abort(comm,2000);
for ( i=0 ; i<num ; i++ ) p1[i] = 7.0;
for ( i=0 ; i<num ; i++ ) p2[i] = 3.0;
num_mutex = 17;
status = GA_Create_mutexes(num_mutex);
if (me==0) printf("%d: GA_Create_mutexes = %d\n",me,status);
/***************************************************************/
if (me==0) {
printf("%d: before GA_Lock\n",me);
GA_Lock(0);
lo[0] = 0;
hi[0] = num-1;
GA_Init_fence();
NGA_Put(g_a,lo,hi,p1,ld);
GA_Fence();
GA_Unlock(0);
printf("%d: after GA_Unlock\n",me);
}
GA_Print(g_a);
if (me==1) {
printf("%d: before GA_Lock\n",me);
GA_Lock(0);
lo[0] = 0;
hi[0] = num-1;
GA_Init_fence();
NGA_Get(g_a,lo,hi,p2,ld);
GA_Fence();
GA_Unlock(0);
printf("%d: after GA_Unlock\n",me);
for ( i=0 ; i<num ; i++ ) printf("p2[%2lu] = %20.10f\n",
(long unsigned)i,p2[i]);
}
/***************************************************************/
status = GA_Destroy_mutexes();
if (me==0) printf("%d: GA_Destroy_mutexes = %d\n",me,status);
/*ARMCI_Free(p2);*/
/*ARMCI_Free(p1);*/
free(p2);
free(p1);
if (me==0) printf("%d: GA_Destroy\n",me);
GA_Destroy(g_a);
/*if (me==0) printf("%d: ARMCI_Finalize\n",me);*/
/*ARMCI_Finalize();*/
if (me==0) printf("%d: GA_Terminate\n",me);
GA_Terminate();
if (me==0) printf("%d: MPI_Finalize\n",me);
MPI_Finalize();
return(0);
}
/**
* Evaluate offsets for each network component
*/
void setOffsets(void)
{
// Interleave contributions from buses and branches to match matrices
int i,j,jdx,jdx1,jdx2;
int *i_bus_offsets = new int[p_nBuses];
int *i_branch_offsets = new int[p_nBranches];
for (i=0; i<p_nBuses; i++) {
i_bus_offsets[i] = 0;
}
for (i=0; i<p_nBranches; i++) {
i_branch_offsets[i] = 0;
}
int icnt = 0;
int nsize;
// Evaluate offsets for individual network components
for (i=0; i<p_nBuses; i++) {
if (p_network->getActiveBus(i)) {
i_bus_offsets[i] = icnt;
icnt += p_network->getBus(i)->vectorNumElements();
std::vector<int> nghbrs = p_network->getConnectedBranches(i);
nsize = nghbrs.size();
for (j=0; j<nsize; j++) {
// Need to avoid double counting of branches when evaluating offsets.
// If branch is non-local and it is active, then include it in offsets.
// Otherwise, if branch is local and bus i is equal to the "from" bus,
// then include it in the offsets.
jdx = nghbrs[j];
if (isLocalBranch(jdx)) {
p_network->getBranchEndpoints(jdx,&jdx1,&jdx2);
if (jdx1 == i) {
i_branch_offsets[jdx] = icnt;
icnt += p_network->getBranch(jdx)->vectorNumElements();
}
} else {
if (p_network->getActiveBranch(jdx)) {
i_branch_offsets[jdx] = icnt;
icnt += p_network->getBranch(jdx)->vectorNumElements();
}
}
}
}
}
// Total number of rows and columns from this processor have been evaluated,
// now create buffers that can scatter individual offsets to global arrays
int **i_bus_index = new int*[p_nBuses];
int **i_branch_index = new int*[p_nBranches];
int *i_bus_index_buf = new int[p_nBuses];
int *i_branch_index_buf = new int[p_nBranches];
int *i_bus_value_buf = new int[p_nBuses];
int *i_branch_value_buf = new int[p_nBranches];
int i_bus_cnt = 0;
int i_branch_cnt = 0;
int row_offset = p_Offsets[p_me];
int nbus = 0;
int nbranch = 0;
for (i=0; i<p_nBuses; i++) {
if (p_network->getActiveBus(i)) {
nbus++;
i_bus_value_buf[i_bus_cnt] = i_bus_offsets[i]+row_offset;
i_bus_index_buf[i_bus_cnt] = p_network->getGlobalBusIndex(i);
i_bus_index[i_bus_cnt] = &i_bus_index_buf[i_bus_cnt];
i_bus_cnt++;
}
}
for (i=0; i<p_nBranches; i++) {
if (p_network->getActiveBranch(i)) {
nbranch++;
i_branch_value_buf[i_branch_cnt] = i_branch_offsets[i]+row_offset;
i_branch_index_buf[i_branch_cnt] = p_network->getGlobalBranchIndex(i);
i_branch_index[i_branch_cnt] = &i_branch_index_buf[i_branch_cnt];
i_branch_cnt++;
}
}
delete [] i_bus_offsets;
delete [] i_branch_offsets;
// Create global arrays that hold column and row offsets for all buses and
// branches in the network. First create map array for global arrays
int *t_busMap = new int[p_nNodes];
int *t_branchMap = new int[p_nNodes];
for (i=0; i<p_nNodes; i++) {
t_busMap[i] = 0;
t_branchMap[i] = 0;
}
t_busMap[p_me] = nbus;
t_branchMap[p_me] = nbranch;
char plus[2];
strcpy(plus,"+");
GA_Pgroup_igop(p_GAgrp, t_busMap, p_nNodes, plus);
GA_Pgroup_igop(p_GAgrp, t_branchMap, p_nNodes, plus);
int *busMap = new int[p_nNodes];
int *branchMap = new int[p_nNodes];
busMap[0] = 0;
branchMap[0] = 0;
int total_buses = t_busMap[0];
int total_branches = t_branchMap[0];
for (i=1; i<p_nNodes; i++) {
busMap[i] = busMap[i-1] + t_busMap[i-1];
total_buses += t_busMap[i];
branchMap[i] = branchMap[i-1] + t_branchMap[i-1];
total_branches += t_branchMap[i];
}
delete [] t_busMap;
delete [] t_branchMap;
int one = 1;
g_bus_offsets = GA_Create_handle();
GA_Set_data(g_bus_offsets, one, &total_buses, C_INT);
GA_Set_irreg_distr(g_bus_offsets, busMap, &p_nNodes);
GA_Set_pgroup(g_bus_offsets, p_GAgrp);
if (!GA_Allocate(g_bus_offsets)) {
char buf[256];
sprintf(buf,"GenVectorMap::setOffsets: Unable to allocate distributed array for bus offsets\n");
printf("%s",buf);
throw gridpack::Exception(buf);
}
GA_Zero(g_bus_offsets);
g_branch_offsets = GA_Create_handle();
GA_Set_data(g_branch_offsets, one, &total_branches, C_INT);
GA_Set_irreg_distr(g_branch_offsets, branchMap, &p_nNodes);
GA_Set_pgroup(g_branch_offsets, p_GAgrp);
if (!GA_Allocate(g_branch_offsets)) {
char buf[256];
sprintf(buf,"GenVectorMap::setOffsets: Unable to allocate distributed array for branch offsets\n");
printf("%s",buf);
throw gridpack::Exception(buf);
}
GA_Zero(g_branch_offsets);
delete [] busMap;
delete [] branchMap;
// Scatter offsets to global arrays
NGA_Scatter(g_bus_offsets, i_bus_value_buf, i_bus_index, i_bus_cnt);
NGA_Scatter(g_branch_offsets, i_branch_value_buf, i_branch_index, i_branch_cnt);
NGA_Pgroup_sync(p_GAgrp);
delete [] i_bus_index;
delete [] i_branch_index;
delete [] i_bus_index_buf;
delete [] i_branch_index_buf;
delete [] i_bus_value_buf;
delete [] i_branch_value_buf;
}
