/*! \brief Print all the metrics
*/
void printGraphMetrics(std::ostream &os) const {
Zoltan2::printMetrics<scalar_t, part_t>(os,
targetGlobalParts_, numGlobalParts_,
graphMetrics_.view(0, graphMetrics_.size()));
}
/*! \brief Create a mesh of approximately the desired size.
*
* We want 3 dimensions close to equal in length.
*/
const RCP<tMVector_t> getMeshCoordinates(
const RCP<const Teuchos::Comm<int> > & comm,
zgno_t numGlobalCoords)
{
int rank = comm->getRank();
int nprocs = comm->getSize();
double k = log(numGlobalCoords) / 3;
double xdimf = exp(k) + 0.5;
ssize_t xdim = static_cast<ssize_t>(floor(xdimf));
ssize_t ydim = xdim;
ssize_t zdim = numGlobalCoords / (xdim*ydim);
ssize_t num=xdim*ydim*zdim;
ssize_t diff = numGlobalCoords - num;
ssize_t newdiff = 0;
while (diff > 0){
if (zdim > xdim && zdim > ydim){
zdim++;
newdiff = diff - (xdim*ydim);
if (newdiff < 0)
if (diff < -newdiff)
zdim--;
}
else if (ydim > xdim && ydim > zdim){
ydim++;
newdiff = diff - (xdim*zdim);
if (newdiff < 0)
if (diff < -newdiff)
ydim--;
}
else{
xdim++;
newdiff = diff - (ydim*zdim);
if (newdiff < 0)
if (diff < -newdiff)
xdim--;
}
diff = newdiff;
}
num=xdim*ydim*zdim;
diff = numGlobalCoords - num;
if (diff < 0)
diff /= -numGlobalCoords;
else
diff /= numGlobalCoords;
if (rank == 0){
if (diff > .01)
cout << "Warning: Difference " << diff*100 << " percent" << endl;
cout << "Mesh size: " << xdim << "x" << ydim << "x" <<
zdim << ", " << num << " vertices." << endl;
}
// Divide coordinates.
ssize_t numLocalCoords = num / nprocs;
ssize_t leftOver = num % nprocs;
ssize_t gid0 = 0;
if (rank <= leftOver)
gid0 = zgno_t(rank) * (numLocalCoords+1);
else
gid0 = (leftOver * (numLocalCoords+1)) +
((zgno_t(rank) - leftOver) * numLocalCoords);
if (rank < leftOver)
numLocalCoords++;
ssize_t gid1 = gid0 + numLocalCoords;
zgno_t *ids = new zgno_t [numLocalCoords];
if (!ids)
throw bad_alloc();
ArrayRCP<zgno_t> idArray(ids, 0, numLocalCoords, true);
for (ssize_t i=gid0; i < gid1; i++)
*ids++ = zgno_t(i);
RCP<const tMap_t> idMap = rcp(
new tMap_t(num, idArray.view(0, numLocalCoords), 0, comm));
// Create a Tpetra::MultiVector of coordinates.
zscalar_t *x = new zscalar_t [numLocalCoords*3];
if (!x)
throw bad_alloc();
ArrayRCP<zscalar_t> coordArray(x, 0, numLocalCoords*3, true);
zscalar_t *y = x + numLocalCoords;
zscalar_t *z = y + numLocalCoords;
zgno_t xStart = 0;
zgno_t yStart = 0;
zgno_t xyPlane = xdim*ydim;
zgno_t zStart = gid0 / xyPlane;
zgno_t rem = gid0 % xyPlane;
if (rem > 0){
yStart = rem / xdim;
xStart = rem % xdim;
}
zlno_t next = 0;
for (zscalar_t zval=zStart; next < numLocalCoords && zval < zdim; zval++){
for (zscalar_t yval=yStart; next < numLocalCoords && yval < ydim; yval++){
for (zscalar_t xval=xStart; next < numLocalCoords && xval < xdim; xval++){
x[next] = xval;
y[next] = yval;
z[next] = zval;
next++;
}
xStart = 0;
}
yStart = 0;
}
ArrayView<const zscalar_t> xArray(x, numLocalCoords);
ArrayView<const zscalar_t> yArray(y, numLocalCoords);
ArrayView<const zscalar_t> zArray(z, numLocalCoords);
ArrayRCP<ArrayView<const zscalar_t> > coordinates =
arcp(new ArrayView<const zscalar_t> [3], 0, 3);
coordinates[0] = xArray;
coordinates[1] = yArray;
coordinates[2] = zArray;
ArrayRCP<const ArrayView<const zscalar_t> > constCoords =
coordinates.getConst();
RCP<tMVector_t> meshCoords = rcp(new tMVector_t(
idMap, constCoords.view(0,3), 3));
return meshCoords;
}
int main(int argc, char *argv[])
{
Teuchos::GlobalMPISession session(&argc, &argv);
RCP<const Comm<int> > comm = Teuchos::DefaultComm<int>::getComm();
int nprocs = comm->getSize();
int rank = comm->getRank();
int fail=0, gfail=0;
double epsilon = 10e-6;
////////////////
// Arrays to hold part Ids and part Sizes for each weight
int numIdsPerProc = 10;
int maxNumWeights = 3;
int maxNumPartSizes = nprocs;
int *lengths = new int [maxNumWeights];
part_t **idLists = new part_t * [maxNumWeights];
scalar_t **sizeLists = new scalar_t * [maxNumWeights];
for (int w=0; w < maxNumWeights; w++){
idLists[w] = new part_t [maxNumPartSizes];
sizeLists[w] = new scalar_t [maxNumPartSizes];
}
/////////////
// A default environment
RCP<const Zoltan2::Environment> env = rcp(new Zoltan2::Environment);
/////////////
// A simple identifier map.
gno_t *myGids = new gno_t [numIdsPerProc];
for (int i=0, x=rank*numIdsPerProc; i < numIdsPerProc; i++){
myGids[i] = x++;
}
ArrayRCP<const gno_t> gidArray(myGids, 0, numIdsPerProc, true);
RCP<const Zoltan2::IdentifierMap<user_t> > idMap =
rcp(new Zoltan2::IdentifierMap<user_t>(env, comm, gidArray));
/////////////
// TEST:
// One weight, one part per proc.
// Some part sizes are 2 and some are 1.
int numGlobalParts = nprocs;
int nWeights = 1;
ArrayRCP<ArrayRCP<part_t> > ids;
ArrayRCP<ArrayRCP<scalar_t> > sizes;
memset(lengths, 0, sizeof(int) * maxNumWeights);
lengths[0] = 1; // We give a size for 1 part.
idLists[0][0] = rank; // The part is my part.
sizeLists[0][0] = rank%2 + 1.0; // The size is 1.0 or 2.0
makeArrays(1, lengths, idLists, sizeLists, ids, sizes);
// Normalized part size for every part, for checking later on
scalar_t *normalizedPartSizes = new scalar_t [numGlobalParts];
scalar_t sumSizes=0;
for (int i=0; i < numGlobalParts; i++){
normalizedPartSizes[i] = 1.0;
if (i % 2) normalizedPartSizes[i] = 2.0;
sumSizes += normalizedPartSizes[i];
}
for (int i=0; i < numGlobalParts; i++)
normalizedPartSizes[i] /= sumSizes;
/////////////
// Create a solution object with part size information, and check it.
RCP<Zoltan2::PartitioningSolution<idInput_t> > solution;
try{
solution = rcp(new Zoltan2::PartitioningSolution<idInput_t>(
env, // application environment info
comm, // problem communicator
idMap, // problem identifiers (global Ids, local Ids)
nWeights, // number of weights
ids.view(0,nWeights), // part ids
sizes.view(0,nWeights))); // part sizes
}
catch (std::exception &e){
fail=1;
}
TEST_FAIL_AND_EXIT(*comm, fail==0, "constructor call 1", 1);
// Test the Solution queries that are used by algorithms
if (solution->getTargetGlobalNumberOfParts() != size_t(numGlobalParts))
fail=2;
if (!fail && solution->getLocalNumberOfParts() != 1)
fail=3;
if (!fail && !solution->oneToOnePartDistribution())
fail=4;
if (!fail && solution->getPartDistribution() != NULL)
fail=5;
if (!fail && solution->getProcDistribution() != NULL)
fail=6;
if (!fail &&
((nprocs>1 && solution->criteriaHasUniformPartSizes(0)) ||
(nprocs==1 && !solution->criteriaHasUniformPartSizes(0))) )
fail=8;
if (!fail){
for (int partId=0; !fail && partId < numGlobalParts; partId++){
scalar_t psize = solution->getCriteriaPartSize(0, partId);
if ( psize < normalizedPartSizes[partId] - epsilon ||
psize > normalizedPartSizes[partId] + epsilon )
fail=9;
}
}
delete [] normalizedPartSizes;
gfail = globalFail(comm, fail);
if (gfail){
printFailureCode(comm, fail); // exits after printing "FAIL"
}
// Test the Solution set method that is called by algorithms
part_t *partAssignments = new part_t [numIdsPerProc];
for (int i=0; i < numIdsPerProc; i++){
partAssignments[i] = myGids[i] % numGlobalParts; // round robin
}
ArrayRCP<part_t> partList = arcp(partAssignments, 0, numIdsPerProc);
try{
solution->setParts(gidArray, partList, true);
}
catch (std::exception &e){
fail=10;
}
gfail = globalFail(comm, fail);
if (gfail){
printFailureCode(comm, fail); // exits after printing "FAIL"
}
// Test the Solution get methods that may be called by users
// or migration functions.
if (solution->getLocalNumberOfIds() != size_t(numIdsPerProc))
fail = 11;
if (!fail){
const gno_t *gids = solution->getIdList();
for (int i=0; !fail && i < numIdsPerProc; i++){
if (gids[i] != myGids[i])
fail = 12;
}
}
if (!fail){
const part_t *parts = solution->getPartList();
for (int i=0; !fail && i < numIdsPerProc; i++){
if (parts[i] != myGids[i] % numGlobalParts)
fail = 13;
}
}
gfail = globalFail(comm, fail);
if (gfail){
printFailureCode(comm, fail); // exits after printing "FAIL"
}
if (rank==0)
std::cout << "PASS" << std::endl;
///////////////////////////////////////////////////////////////////
// TODO:
/////////////
// Create a solution object without part size information, and check it.
/////////////
// Test multiple weights.
/////////////
// Test multiple parts per process.
/////////////
// Specify a list of parts of size 0. (The rest should be uniform.)
delete [] lengths;
for (int w=0; w < maxNumWeights; w++){
delete [] idLists[w];
delete [] sizeLists[w];
}
delete [] idLists;
delete [] sizeLists;
}
/*! \brief Print all the metrics
*/
void printMetrics(ostream &os) const {
Zoltan2::printMetrics<scalar_t>(os,
targetGlobalParts_, numGlobalParts_, numNonEmpty_,
metrics_.view(0, metrics_.size()));
}
size_t computeLocalEdgeList(
const RCP<const Environment> &env, const RCP<const Comm<int> > &comm,
size_t numLocalEdges, // local edges
size_t numLocalGraphEdges, // edges in "local" graph
RCP<const IdentifierMap<User> > &idMap,
ArrayRCP<const typename InputTraits<User>::zgid_t> &allEdgeIds, // in
ArrayRCP<const typename InputTraits<User>::gno_t> &allEdgeGnos, // in
ArrayRCP<int> &allProcs, // in
ArrayRCP<const typename InputTraits<User>::lno_t> &allOffs, // in
ArrayRCP<StridedData<typename InputTraits<User>::lno_t,
typename InputTraits<User>::scalar_t> > &allWeights,// in
ArrayRCP<const typename InputTraits<User>::lno_t> &edgeLocalIds, //
ArrayRCP<const typename InputTraits<User>::lno_t> &offsets, // out
ArrayRCP<StridedData<typename InputTraits<User>::lno_t,
typename InputTraits<User>::scalar_t> > &eWeights) // out
{
typedef typename InputTraits<User>::zgid_t zgid_t;
typedef typename InputTraits<User>::gno_t gno_t;
typedef typename InputTraits<User>::scalar_t scalar_t;
typedef typename InputTraits<User>::lno_t lno_t;
typedef StridedData<lno_t, scalar_t> input_t;
int rank = comm->getRank();
bool gnosAreGids = idMap->gnosAreGids();
edgeLocalIds = ArrayRCP<const lno_t>(Teuchos::null);
eWeights = ArrayRCP<input_t>(Teuchos::null);
offsets = ArrayRCP<const lno_t>(Teuchos::null);
if (numLocalGraphEdges == 0) {
// Set the offsets array and return
size_t allOffsSize = allOffs.size();
lno_t *offs = new lno_t [allOffsSize];
env->localMemoryAssertion(__FILE__, __LINE__, allOffsSize, offs);
for (size_t i = 0; i < allOffsSize; i++) offs[i] = 0;
offsets = arcp(offs, 0, allOffsSize, true);
return 0;
}
if (numLocalGraphEdges == numLocalEdges){
// Entire graph is local.
lno_t *lnos = new lno_t [numLocalGraphEdges];
env->localMemoryAssertion(__FILE__, __LINE__, numLocalGraphEdges, lnos);
if (comm->getSize() == 1) {
// With one rank, Can use gnos as local index.
if (gnosAreGids)
for (size_t i=0; i < numLocalEdges; i++) lnos[i] = allEdgeIds[i];
else
for (size_t i=0; i < numLocalEdges; i++) lnos[i] = allEdgeGnos[i];
}
else {
ArrayRCP<gno_t> gnoArray;
if (gnosAreGids){
ArrayRCP<const gno_t> gnosConst =
arcp_reinterpret_cast<const gno_t>(allEdgeIds);
gnoArray = arcp_const_cast<gno_t>(gnosConst);
}
else {
gnoArray = arcp_const_cast<gno_t>(allEdgeGnos);
}
// Need to translate to gnos to local indexing
ArrayView<lno_t> lnoView(lnos, numLocalGraphEdges);
try {
idMap->lnoTranslate(lnoView,
gnoArray.view(0,numLocalGraphEdges),
TRANSLATE_LIB_TO_APP);
}
Z2_FORWARD_EXCEPTIONS;
}
edgeLocalIds = arcp(lnos, 0, numLocalGraphEdges, true);
offsets = allOffs;
eWeights = allWeights;
}
void
timeTpetra (const TPETRA_GO numGlobalCoords,
const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
const bool doMemory)
{
using Teuchos::arcp;
using Teuchos::ArrayRCP;
using Teuchos::ArrayView;
using Teuchos::Comm;
using Teuchos::RCP;
using Teuchos::rcp;
using std::cout;
using std::endl;
typedef Tpetra::Map<TPETRA_LO, TPETRA_GO> map_type;
typedef Tpetra::MultiVector<TPETRA_SCALAR, TPETRA_LO, TPETRA_GO> MV;
typedef ArrayView<const TPETRA_SCALAR> coordList_t;
const int nprocs = comm->getSize ();
const int rank = comm->getRank ();
///////////// Step 1 //////////////////////////////////
// Create a MV with contiguous global IDs
const TPETRA_LO numLocalCoords = numSequentialGlobalIds(numGlobalCoords, nprocs, rank);
RCP<const map_type> tmap;
RCP<MV> mvector;
TPETRA_SCALAR* coords = NULL;
{
Teuchos::TimeMonitor timeMon (*tmvBuild);
tmap = rcp (new map_type (numGlobalCoords, numLocalCoords, 0, comm));
coords = new TPETRA_SCALAR [COORDDIM * numLocalCoords];
memset (coords, 0, sizeof(TPETRA_SCALAR) * numLocalCoords * COORDDIM);
coordList_t *avList = new coordList_t [COORDDIM];
TPETRA_LO offset = 0;
for (int dim = 0; dim < COORDDIM; ++dim) {
avList[dim] = coordList_t(coords + offset, numLocalCoords);
offset += numLocalCoords;
}
ArrayRCP<const coordList_t> vectors = arcp (avList, 0, COORDDIM);
mvector = rcp (new MV (tmap, vectors.view (0, COORDDIM), COORDDIM));
}
if (rank == 0 && doMemory) {
const long nkb = Zoltan2::getProcessKilobytes ();
cout << "Create mvector 1: " << nkb << endl;
}
///////////// Step 2 //////////////////////////////////
// Migrate the MV.
ArrayRCP<const TPETRA_GO> newGidArray;
{
TPETRA_GO *newGids = NULL;
roundRobinGlobalIds<TPETRA_GO> (numGlobalCoords, nprocs, rank, newGids);
newGidArray = arcp<const TPETRA_GO> (newGids, 0, numLocalCoords, true);
}
RCP<const map_type> newTmap;
RCP<Tpetra::Import<TPETRA_LO, TPETRA_GO> > importer;
RCP<MV> newMvector;
{
Teuchos::TimeMonitor timeMon (*tmvMigrate);
newTmap = rcp (new map_type (numGlobalCoords, newGidArray.view(0, numLocalCoords), 0, comm));
importer = rcp (new Tpetra::Import<TPETRA_LO, TPETRA_GO> (tmap, newTmap));
newMvector = rcp (new MV (newTmap, COORDDIM, true));
newMvector->doImport (*mvector, *importer, Tpetra::INSERT);
mvector = newMvector;
}
delete [] coords;
if (rank == 0 && doMemory) {
const long nkb = Zoltan2::getProcessKilobytes ();
cout << "Create mvector 2: " << nkb << endl;
}
///////////// Step 3 //////////////////////////////////
// Divide processes into two halves.
RCP<Comm<int> > subComm;
{
int groupSize = 0;
int leftHalfNumProcs = nprocs / 2;
int *myHalfProcs = NULL;
if (rank < leftHalfNumProcs){
groupSize = leftHalfNumProcs;
myHalfProcs = new int [groupSize];
for (int i=0; i < groupSize; i++)
myHalfProcs[i] = i;
}
else {
groupSize = nprocs - leftHalfNumProcs;
myHalfProcs = new int [groupSize];
int firstNum = leftHalfNumProcs;
for (int i=0; i < groupSize; i++)
myHalfProcs[i] = firstNum++;
}
ArrayView<const int> idView(myHalfProcs, groupSize);
subComm = comm->createSubcommunicator (idView);
delete [] myHalfProcs;
}
// Divide the multivector into two. Each process group is creating
// a multivector with non-contiguous global ids. For one group,
// base gid is not 0.
size_t globalSize = Teuchos::OrdinalTraits<size_t>::invalid ();
RCP<map_type> subMap;
RCP<MV> subMvector;
{
Teuchos::TimeMonitor timeMon (*tmvBuildN);
ArrayView<const TPETRA_GO> gidList = mvector->getMap ()->getNodeElementList ();
subMap = rcp (new map_type (globalSize, gidList, 0, subComm));
globalSize = subMap->getGlobalNumElements ();
// Get a view of the block of rows to copy.
RCP<MV> tmp = mvector->offsetViewNonConst (subMap, 0);
// Create a new multivector to hold the group's rows.
subMvector = rcp (new MV (subMap, mvector->getNumVectors ()));
// Copy the view into the new multivector.
Tpetra::deep_copy (*subMvector, *tmp);
}
///////////// Step 4 //////////////////////////////////
// Each subgroup migrates the sub-multivector so the
// global Ids are increasing with process rank.
TPETRA_GO *increasingGids = NULL;
subGroupGloballyIncreasingIds<TPETRA_GO> (numGlobalCoords, nprocs,
rank, increasingGids);
ArrayRCP<const TPETRA_GO> incrGidArray (increasingGids, 0, numLocalCoords, true);
RCP<const map_type> newSubMap;
RCP<Tpetra::Import<TPETRA_LO, TPETRA_GO> > subImporter;
RCP<MV> newSubMvector;
{
Teuchos::TimeMonitor timeMon (*tmvMigrateN);
newSubMap =
rcp (new map_type (globalSize, incrGidArray.view (0, numLocalCoords),
0, subComm));
subImporter = rcp (new Tpetra::Import<TPETRA_LO, TPETRA_GO> (subMap, newSubMap));
newSubMvector = rcp (new MV (newSubMap, COORDDIM, true));
newSubMvector->doImport (*subMvector, *subImporter, Tpetra::INSERT);
mvector = newSubMvector;
}
}
