ParMetisGraph::ParMetisGraph(ParMetisMesh* parMesh,
MPI::Intracomm* comm,
int ncommonnodes)
: parMetisMesh(parMesh)
{
FUNCNAME("ParMetisGraph::ParMetisGraph()");
TEST_EXIT(parMesh)("No ParMetisMesh defined!\n");
TEST_EXIT(comm)("No MPI communicator defined!\n");
int numflag = 0;
if (ncommonnodes == -1)
ncommonnodes = parMetisMesh->getDim();
MPI_Comm tmpComm = MPI_Comm(*comm);
ParMETIS_V3_Mesh2Dual(parMetisMesh->getElementDist(),
parMetisMesh->getElementPtr(),
parMetisMesh->getElementInd(),
&numflag,
&ncommonnodes,
&xadj,
&adjncy,
&tmpComm);
}
/*@
MatMeshToCellGraph - Uses the ParMETIS package to convert a Mat that represents a mesh to a Mat the represents the graph of the coupling
between cells (the "dual" graph) and is suitable for partitioning with the MatPartitioning object. Use this to partition
cells of a mesh.
Collective on Mat
Input Parameter:
+ mesh - the graph that represents the mesh
- ncommonnodes - mesh elements that share this number of common nodes are considered neighbors, use 2 for triangules and
quadralaterials, 3 for tetrahedrals and 4 for hexahedrals
Output Parameter:
. dual - the dual graph
Notes:
Currently requires ParMetis to be installed and uses ParMETIS_V3_Mesh2Dual()
The columns of each row of the Mat mesh are the global vertex numbers of the vertices of that rows cell. The number of rows in mesh is
number of cells, the number of columns is the number of vertices.
Level: advanced
.seealso: MatMeshToVertexGraph(), MatCreateMPIAdj(), MatPartitioningCreate()
@*/
PetscErrorCode MatMeshToCellGraph(Mat mesh,PetscInt ncommonnodes,Mat *dual)
{
PetscErrorCode ierr;
PetscInt *newxadj,*newadjncy;
PetscInt numflag=0;
Mat_MPIAdj *adj = (Mat_MPIAdj *)mesh->data,*newadj;
PetscBool flg;
int status;
PetscFunctionBegin;
ierr = PetscObjectTypeCompare((PetscObject)mesh,MATMPIADJ,&flg);CHKERRQ(ierr);
if (!flg) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SUP,"Must use MPIAdj matrix type");
CHKMEMQ;
status = ParMETIS_V3_Mesh2Dual(mesh->rmap->range,adj->i,adj->j,&numflag,&ncommonnodes,&newxadj,&newadjncy,&((PetscObject)mesh)->comm);CHKERRQPARMETIS(status);
CHKMEMQ;
ierr = MatCreateMPIAdj(((PetscObject)mesh)->comm,mesh->rmap->n,mesh->rmap->N,newxadj,newadjncy,PETSC_NULL,dual);CHKERRQ(ierr);
newadj = (Mat_MPIAdj *)(*dual)->data;
newadj->freeaijwithfree = PETSC_TRUE; /* signal the matrix should be freed with system free since space was allocated by ParMETIS */
PetscFunctionReturn(0);
}
void
ParmetisLoadBalancer :: calculateLoadTransfer()
{
idx_t *eind, *eptr, *xadj, *adjncy, *vwgt, *vsize;
idx_t *part;
int i, nlocalelems, eind_size, nelem = domain->giveNumberOfElements();
int ndofman, idofman, numflag, ncommonnodes, options [ 4 ], ie, nproc;
int edgecut, wgtflag, ncon;
real_t ubvec [ 1 ], itr;
Element *ielem;
MPI_Comm communicator = MPI_COMM_WORLD;
LoadBalancerMonitor *lbm = domain->giveEngngModel()->giveLoadBalancerMonitor();
nproc = domain->giveEngngModel()->giveNumberOfProcesses();
// init parmetis element numbering
this->initGlobalParmetisElementNumbering();
// prepare data structures for ParMETIS_V3_Mesh2Dual
// count the size of eind array
eind_size = 0;
nlocalelems = 0;
for ( i = 1; i <= nelem; i++ ) {
ielem = domain->giveElement(i);
if ( ielem->giveParallelMode() == Element_local ) {
nlocalelems++;
eind_size += ielem->giveNumberOfDofManagers();
}
}
// allocate eind and eptr arrays
eind = new idx_t [ eind_size ];
eptr = new idx_t [ nlocalelems + 1 ];
if ( ( eind == NULL ) || ( eptr == NULL ) ) {
OOFEM_ERROR("failed to allocate eind and eptr arrays");
}
// fill in the eind and eptr (mesh graph)
int eind_pos = 0, eptr_pos = 0;
for ( i = 1; i <= nelem; i++ ) {
ielem = domain->giveElement(i);
if ( ielem->giveParallelMode() == Element_local ) {
eptr [ eptr_pos ] = eind_pos;
ndofman = ielem->giveNumberOfDofManagers();
for ( idofman = 1; idofman <= ndofman; idofman++ ) {
eind [ eind_pos++ ] = ielem->giveDofManager(idofman)->giveGlobalNumber() - 1;
}
eptr_pos++;
}
}
// last rec
eptr [ nlocalelems ] = eind_pos;
// call ParMETIS_V3_Mesh2Dual to construct dual graph (in parallel)
// dual graph: elements are vertices; element edges are graph edges
// this is necessary, since cut runs through graph edges
numflag = 0;
ncommonnodes = 2;
ParMETIS_V3_Mesh2Dual(elmdist, eptr, eind, & numflag, & ncommonnodes, & xadj, & adjncy, & communicator);
#ifdef ParmetisLoadBalancer_DEBUG_PRINT
int myrank = domain->giveEngngModel()->giveRank();
// DEBUG PRINT
fprintf(stderr, "[%d] xadj:", myrank);
for ( i = 0; i <= nlocalelems; i++ ) {
fprintf(stderr, " %d", xadj [ i ]);
}
fprintf(stderr, "\n[%d] adjncy:", myrank);
for ( i = 0; i < xadj [ nlocalelems ]; i++ ) {
fprintf(stderr, " %d", adjncy [ i ]);
}
fprintf(stderr, "\n");
#endif
// setup imbalance tolerance for each vertex weight - ubvec param
ubvec [ 0 ] = 1.05;
// setup options array
options [ 0 ] = 1; // set to zero for default
options [ 1 ] = 1; // get timings
options [ 2 ] = 15; // random seed
options [ 3 ] = 1; // sub-domains and processors are coupled
// set ratio of inter-proc communication compared to data redistribution time
itr = 1000.0;
// set partition weights by quering load balance monitor
const FloatArray &_procweights = lbm->giveProcessorWeights();
if ( tpwgts == NULL ) {
if ( ( tpwgts = new real_t [ nproc ] ) == NULL ) {
OOFEM_ERROR("failed to allocate tpwgts");
}
}
for ( i = 0; i < nproc; i++ ) {
tpwgts [ i ] = _procweights(i);
}
/*
* // log processor weights
* OOFEM_LOG_RELEVANT ("[%d] ParmetisLoadBalancer: proc weights: ", myrank);
* for (i=0; i<nproc; i++) OOFEM_LOG_RELEVANT ("%4.3f ",tpwgts[i]);
* OOFEM_LOG_RELEVANT ("\n");
*/
// obtain vertices weights (element weights) representing relative computational cost
if ( ( vwgt = new idx_t [ nlocalelems ] ) == NULL ) {
OOFEM_ERROR("failed to allocate vwgt");
}
if ( ( vsize = new idx_t [ nlocalelems ] ) == NULL ) {
OOFEM_ERROR("failed to allocate vsize");
}
for ( ie = 0, i = 0; i < nelem; i++ ) {
ielem = domain->giveElement(i + 1);
if ( ielem->giveParallelMode() == Element_local ) {
vwgt [ ie ] = ( int ) ( ielem->predictRelativeComputationalCost() * 100.0 );
vsize [ ie++ ] = 1; //ielem->predictRelativeRedistributionCost();
}
}
wgtflag = 2;
numflag = 0;
ncon = 1;
if ( ( part = new idx_t [ nlocalelems ] ) == NULL ) {
OOFEM_ERROR("failed to allocate part");
}
// call ParMETIS balancing routineParMETIS_V3_AdaptiveRepart
ParMETIS_V3_AdaptiveRepart(elmdist, xadj, adjncy, vwgt, vsize, NULL, & wgtflag, & numflag, & ncon, & nproc,
tpwgts, ubvec, & itr, options, & edgecut, part, & communicator);
// part contains partition vector for local elements on receiver
// we need to map it to domain elements (this is not the same, since
// domain may contain not only its local elements but remote elements as well)
int loc_num = 0;
this->elementPart.resize(nelem);
for ( i = 1; i <= nelem; i++ ) {
ielem = domain->giveElement(i);
if ( ielem->giveParallelMode() == Element_local ) {
this->elementPart.at(i) = part [ loc_num++ ];
} else {
// we can not say anything about remote elements; this information is available on partition
// that has its local counterpart
this->elementPart.at(i) = -1;
}
}
if ( part ) {
delete[] part;
}
#ifdef ParmetisLoadBalancer_DEBUG_PRINT
// debug
fprintf(stderr, "[%d] edgecut: %d elementPart:", myrank, edgecut);
for ( i = 1; i <= nelem; i++ ) {
fprintf( stderr, " %d", elementPart.at(i) );
}
fprintf(stderr, "\n");
#endif
// delete allocated xadj, adjncy arrays by ParMETIS
delete[] eind;
delete[] eptr;
delete[] vwgt;
delete[] vsize;
free(xadj);
free(adjncy);
this->labelDofManagers();
}
