void AlgZoltan<Adapter>::partition(
const RCP<PartitioningSolution<Adapter> > &solution
)
{
HELLO;
char paramstr[128];
size_t numGlobalParts = solution->getTargetGlobalNumberOfParts();
sprintf(paramstr, "%lu", numGlobalParts);
zz->Set_Param("NUM_GLOBAL_PARTS", paramstr);
int wdim = adapter->getNumWeightsPerID();
sprintf(paramstr, "%d", wdim);
zz->Set_Param("OBJ_WEIGHT_DIM", paramstr);
const Teuchos::ParameterList &pl = env->getParameters();
double tolerance;
const Teuchos::ParameterEntry *pe = pl.getEntryPtr("imbalance_tolerance");
if (pe){
char str[30];
tolerance = pe->getValue<double>(&tolerance);
sprintf(str, "%f", tolerance);
zz->Set_Param("IMBALANCE_TOL", str);
}
pe = pl.getEntryPtr("partitioning_approach");
if (pe){
std::string approach;
approach = pe->getValue<std::string>(&approach);
if (approach == "partition")
zz->Set_Param("LB_APPROACH", "PARTITION");
else
zz->Set_Param("LB_APPROACH", "REPARTITION");
}
// Look for zoltan_parameters sublist; pass all zoltan parameters to Zoltan
try {
const Teuchos::ParameterList &zpl = pl.sublist("zoltan_parameters");
for (ParameterList::ConstIterator iter = zpl.begin();
iter != zpl.end(); iter++) {
const std::string &zname = pl.name(iter);
// Convert the value to a string to pass to Zoltan
std::string zval = pl.entry(iter).getValue(&zval);
zz->Set_Param(zname.c_str(), zval.c_str());
}
}
catch (std::exception &e) {
// No zoltan_parameters sublist found; no Zoltan parameters to register
}
// Get target part sizes
int pdim = (wdim > 1 ? wdim : 1);
for (int d = 0; d < pdim; d++) {
if (!solution->criteriaHasUniformPartSizes(d)) {
float *partsizes = new float[numGlobalParts];
int *partidx = new int[numGlobalParts];
int *wgtidx = new int[numGlobalParts];
for (size_t i=0; i<numGlobalParts; i++) partidx[i] = i;
for (size_t i=0; i<numGlobalParts; i++) wgtidx[i] = d;
for (size_t i=0; i<numGlobalParts; i++)
partsizes[i] = solution->getCriteriaPartSize(0, i);
zz->LB_Set_Part_Sizes(1, numGlobalParts, partidx, wgtidx, partsizes);
delete [] partsizes;
delete [] partidx;
delete [] wgtidx;
}
}
// Make the call to LB_Partition
int changed = 0;
int nGidEnt = 1, nLidEnt = 1;
int nDummy = -1; // Dummy vars to satisfy arglist
ZOLTAN_ID_PTR dGids = NULL, dLids = NULL;
int *dProcs = NULL, *dParts = NULL;
int nObj = -1; // Output vars with new part info
ZOLTAN_ID_PTR oGids = NULL, oLids = NULL;
int *oProcs = NULL, *oParts = NULL;
zz->Set_Param("RETURN_LISTS", "PARTS"); // required format for Zoltan2;
// results in last five arguments
int ierr = zz->LB_Partition(changed, nGidEnt, nLidEnt,
nDummy, dGids, dLids, dProcs, dParts,
nObj, oGids, oLids, oProcs, oParts);
env->globalInputAssertion(__FILE__, __LINE__, "Zoltan LB_Partition",
(ierr==ZOLTAN_OK || ierr==ZOLTAN_WARN), BASIC_ASSERTION, problemComm);
int numObjects=nObj;
// The number of objects may be larger than zoltan knows due to copies that
// were removed by the hypergraph model
if (model!=RCP<const Model<Adapter> >() &&
dynamic_cast<const HyperGraphModel<Adapter>* >(&(*model)) &&
!(dynamic_cast<const HyperGraphModel<Adapter>* >(&(*model))->areVertexIDsUnique())) {
numObjects=model->getLocalNumObjects();
}
// Load answer into the solution.
ArrayRCP<part_t> partList(new part_t[numObjects], 0, numObjects, true);
for (int i = 0; i < nObj; i++) partList[oLids[i]] = oParts[i];
if (model!=RCP<const Model<Adapter> >() &&
dynamic_cast<const HyperGraphModel<Adapter>* >(&(*model)) &&
!(dynamic_cast<const HyperGraphModel<Adapter>* >(&(*model))->areVertexIDsUnique())) {
// Setup the part ids for copied entities removed by ownership in
// hypergraph model.
const HyperGraphModel<Adapter>* mdl =
static_cast<const HyperGraphModel<Adapter>* >(&(*model));
typedef typename HyperGraphModel<Adapter>::map_t map_t;
Teuchos::RCP<const map_t> mapWithCopies;
Teuchos::RCP<const map_t> oneToOneMap;
mdl->getVertexMaps(mapWithCopies,oneToOneMap);
typedef Tpetra::Vector<scalar_t, lno_t, gno_t> vector_t;
vector_t vecWithCopies(mapWithCopies);
vector_t oneToOneVec(oneToOneMap);
// Set values in oneToOneVec: each entry == rank
assert(nObj == lno_t(oneToOneMap->getNodeNumElements()));
for (lno_t i = 0; i < nObj; i++)
oneToOneVec.replaceLocalValue(i, oParts[i]);
// Now import oneToOneVec's values back to vecWithCopies
Teuchos::RCP<const Tpetra::Import<lno_t, gno_t> > importer =
Tpetra::createImport<lno_t, gno_t>(oneToOneMap, mapWithCopies);
vecWithCopies.doImport(oneToOneVec, *importer, Tpetra::REPLACE);
// Should see copied vector values when print VEC WITH COPIES
lno_t nlocal = lno_t(mapWithCopies->getNodeNumElements());
for (lno_t i = 0; i < nlocal; i++)
partList[i] = vecWithCopies.getData()[i];
}
solution->setParts(partList);
// Clean up
zz->LB_Free_Part(&oGids, &oLids, &oProcs, &oParts);
}
int main(int narg, char **arg)
{
typedef Tpetra::Map<> map_t;
typedef map_t::local_ordinal_type lno_t;
typedef map_t::global_ordinal_type gno_t;
typedef int scalar_t;
// Usual Teuchos MPI stuff
Teuchos::GlobalMPISession mpiSession(&narg,&arg);
Teuchos::RCP<const Teuchos::Comm<int> >
comm = Teuchos::DefaultComm<int>::getComm();
int me = comm->getRank();
// Create a map with duplicated entries (mapWithCopies)
// Each rank has 15 IDs, the last five of which overlap with the next rank.
lno_t numLocalCoords = 15;
lno_t offset = me * 10;
Teuchos::Array<gno_t> gids(numLocalCoords);
for (lno_t i = 0 ; i < numLocalCoords; i++)
gids[i] = static_cast<gno_t> (offset + i);
Tpetra::global_size_t numGlobalCoords =
Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid();
Teuchos::RCP<const map_t> mapWithCopies =
rcp(new map_t(numGlobalCoords, gids(), 0, comm));
// Create a new map with IDs uniquely assigned to ranks (oneToOneMap)
Teuchos::RCP<const map_t> oneToOneMap =
Tpetra::createOneToOne<lno_t, gno_t>(mapWithCopies);
// Create vectors with each map
typedef Tpetra::Vector<scalar_t, lno_t, gno_t> vector_t;
vector_t vecWithCopies(mapWithCopies);
vector_t oneToOneVec(oneToOneMap);
// Set values in oneToOneVec: each entry == rank
for (lno_t i = 0; i < lno_t(oneToOneMap->getNodeNumElements()); i++)
oneToOneVec.replaceLocalValue(i, me);
// Now import oneToOneVec's values back to vecWithCopies
Teuchos::RCP<const Tpetra::Import<lno_t, gno_t> > importer =
Tpetra::createImport<lno_t, gno_t>(oneToOneMap, mapWithCopies);
vecWithCopies.doImport(oneToOneVec, *importer, Tpetra::REPLACE);
// Print the entries of each vector
std::cout << me << " ONE TO ONE VEC ("
<< oneToOneMap->getGlobalNumElements() << "): ";
lno_t nlocal = lno_t(oneToOneMap->getNodeNumElements());
for (lno_t i = 0; i < nlocal; i++)
std::cout << "[" << oneToOneMap->getGlobalElement(i) << " "
<< oneToOneVec.getData()[i] << "] ";
std::cout << std::endl;
// Should see copied vector values when print VEC WITH COPIES
std::cout << me << " VEC WITH COPIES ("
<< mapWithCopies->getGlobalNumElements() << "): ";
nlocal = lno_t(mapWithCopies->getNodeNumElements());
for (lno_t i = 0; i < nlocal; i++)
std::cout << "[" << mapWithCopies->getGlobalElement(i) << " "
<< vecWithCopies.getData()[i] << "] ";
std::cout << std::endl;
return 0;
}
