/*! \brief Print all the metrics */ void printGraphMetrics(std::ostream &os) const { Zoltan2::printMetrics<scalar_t, part_t>(os, targetGlobalParts_, numGlobalParts_, graphMetrics_.view(0, graphMetrics_.size())); }
/*! \brief Create a mesh of approximately the desired size. * * We want 3 dimensions close to equal in length. */ const RCP<tMVector_t> getMeshCoordinates( const RCP<const Teuchos::Comm<int> > & comm, zgno_t numGlobalCoords) { int rank = comm->getRank(); int nprocs = comm->getSize(); double k = log(numGlobalCoords) / 3; double xdimf = exp(k) + 0.5; ssize_t xdim = static_cast<ssize_t>(floor(xdimf)); ssize_t ydim = xdim; ssize_t zdim = numGlobalCoords / (xdim*ydim); ssize_t num=xdim*ydim*zdim; ssize_t diff = numGlobalCoords - num; ssize_t newdiff = 0; while (diff > 0){ if (zdim > xdim && zdim > ydim){ zdim++; newdiff = diff - (xdim*ydim); if (newdiff < 0) if (diff < -newdiff) zdim--; } else if (ydim > xdim && ydim > zdim){ ydim++; newdiff = diff - (xdim*zdim); if (newdiff < 0) if (diff < -newdiff) ydim--; } else{ xdim++; newdiff = diff - (ydim*zdim); if (newdiff < 0) if (diff < -newdiff) xdim--; } diff = newdiff; } num=xdim*ydim*zdim; diff = numGlobalCoords - num; if (diff < 0) diff /= -numGlobalCoords; else diff /= numGlobalCoords; if (rank == 0){ if (diff > .01) cout << "Warning: Difference " << diff*100 << " percent" << endl; cout << "Mesh size: " << xdim << "x" << ydim << "x" << zdim << ", " << num << " vertices." << endl; } // Divide coordinates. ssize_t numLocalCoords = num / nprocs; ssize_t leftOver = num % nprocs; ssize_t gid0 = 0; if (rank <= leftOver) gid0 = zgno_t(rank) * (numLocalCoords+1); else gid0 = (leftOver * (numLocalCoords+1)) + ((zgno_t(rank) - leftOver) * numLocalCoords); if (rank < leftOver) numLocalCoords++; ssize_t gid1 = gid0 + numLocalCoords; zgno_t *ids = new zgno_t [numLocalCoords]; if (!ids) throw bad_alloc(); ArrayRCP<zgno_t> idArray(ids, 0, numLocalCoords, true); for (ssize_t i=gid0; i < gid1; i++) *ids++ = zgno_t(i); RCP<const tMap_t> idMap = rcp( new tMap_t(num, idArray.view(0, numLocalCoords), 0, comm)); // Create a Tpetra::MultiVector of coordinates. zscalar_t *x = new zscalar_t [numLocalCoords*3]; if (!x) throw bad_alloc(); ArrayRCP<zscalar_t> coordArray(x, 0, numLocalCoords*3, true); zscalar_t *y = x + numLocalCoords; zscalar_t *z = y + numLocalCoords; zgno_t xStart = 0; zgno_t yStart = 0; zgno_t xyPlane = xdim*ydim; zgno_t zStart = gid0 / xyPlane; zgno_t rem = gid0 % xyPlane; if (rem > 0){ yStart = rem / xdim; xStart = rem % xdim; } zlno_t next = 0; for (zscalar_t zval=zStart; next < numLocalCoords && zval < zdim; zval++){ for (zscalar_t yval=yStart; next < numLocalCoords && yval < ydim; yval++){ for (zscalar_t xval=xStart; next < numLocalCoords && xval < xdim; xval++){ x[next] = xval; y[next] = yval; z[next] = zval; next++; } xStart = 0; } yStart = 0; } ArrayView<const zscalar_t> xArray(x, numLocalCoords); ArrayView<const zscalar_t> yArray(y, numLocalCoords); ArrayView<const zscalar_t> zArray(z, numLocalCoords); ArrayRCP<ArrayView<const zscalar_t> > coordinates = arcp(new ArrayView<const zscalar_t> [3], 0, 3); coordinates[0] = xArray; coordinates[1] = yArray; coordinates[2] = zArray; ArrayRCP<const ArrayView<const zscalar_t> > constCoords = coordinates.getConst(); RCP<tMVector_t> meshCoords = rcp(new tMVector_t( idMap, constCoords.view(0,3), 3)); return meshCoords; }
int main(int argc, char *argv[]) { Teuchos::GlobalMPISession session(&argc, &argv); RCP<const Comm<int> > comm = Teuchos::DefaultComm<int>::getComm(); int nprocs = comm->getSize(); int rank = comm->getRank(); int fail=0, gfail=0; double epsilon = 10e-6; //////////////// // Arrays to hold part Ids and part Sizes for each weight int numIdsPerProc = 10; int maxNumWeights = 3; int maxNumPartSizes = nprocs; int *lengths = new int [maxNumWeights]; part_t **idLists = new part_t * [maxNumWeights]; scalar_t **sizeLists = new scalar_t * [maxNumWeights]; for (int w=0; w < maxNumWeights; w++){ idLists[w] = new part_t [maxNumPartSizes]; sizeLists[w] = new scalar_t [maxNumPartSizes]; } ///////////// // A default environment RCP<const Zoltan2::Environment> env = rcp(new Zoltan2::Environment); ///////////// // A simple identifier map. gno_t *myGids = new gno_t [numIdsPerProc]; for (int i=0, x=rank*numIdsPerProc; i < numIdsPerProc; i++){ myGids[i] = x++; } ArrayRCP<const gno_t> gidArray(myGids, 0, numIdsPerProc, true); RCP<const Zoltan2::IdentifierMap<user_t> > idMap = rcp(new Zoltan2::IdentifierMap<user_t>(env, comm, gidArray)); ///////////// // TEST: // One weight, one part per proc. // Some part sizes are 2 and some are 1. int numGlobalParts = nprocs; int nWeights = 1; ArrayRCP<ArrayRCP<part_t> > ids; ArrayRCP<ArrayRCP<scalar_t> > sizes; memset(lengths, 0, sizeof(int) * maxNumWeights); lengths[0] = 1; // We give a size for 1 part. idLists[0][0] = rank; // The part is my part. sizeLists[0][0] = rank%2 + 1.0; // The size is 1.0 or 2.0 makeArrays(1, lengths, idLists, sizeLists, ids, sizes); // Normalized part size for every part, for checking later on scalar_t *normalizedPartSizes = new scalar_t [numGlobalParts]; scalar_t sumSizes=0; for (int i=0; i < numGlobalParts; i++){ normalizedPartSizes[i] = 1.0; if (i % 2) normalizedPartSizes[i] = 2.0; sumSizes += normalizedPartSizes[i]; } for (int i=0; i < numGlobalParts; i++) normalizedPartSizes[i] /= sumSizes; ///////////// // Create a solution object with part size information, and check it. RCP<Zoltan2::PartitioningSolution<idInput_t> > solution; try{ solution = rcp(new Zoltan2::PartitioningSolution<idInput_t>( env, // application environment info comm, // problem communicator idMap, // problem identifiers (global Ids, local Ids) nWeights, // number of weights ids.view(0,nWeights), // part ids sizes.view(0,nWeights))); // part sizes } catch (std::exception &e){ fail=1; } TEST_FAIL_AND_EXIT(*comm, fail==0, "constructor call 1", 1); // Test the Solution queries that are used by algorithms if (solution->getTargetGlobalNumberOfParts() != size_t(numGlobalParts)) fail=2; if (!fail && solution->getLocalNumberOfParts() != 1) fail=3; if (!fail && !solution->oneToOnePartDistribution()) fail=4; if (!fail && solution->getPartDistribution() != NULL) fail=5; if (!fail && solution->getProcDistribution() != NULL) fail=6; if (!fail && ((nprocs>1 && solution->criteriaHasUniformPartSizes(0)) || (nprocs==1 && !solution->criteriaHasUniformPartSizes(0))) ) fail=8; if (!fail){ for (int partId=0; !fail && partId < numGlobalParts; partId++){ scalar_t psize = solution->getCriteriaPartSize(0, partId); if ( psize < normalizedPartSizes[partId] - epsilon || psize > normalizedPartSizes[partId] + epsilon ) fail=9; } } delete [] normalizedPartSizes; gfail = globalFail(comm, fail); if (gfail){ printFailureCode(comm, fail); // exits after printing "FAIL" } // Test the Solution set method that is called by algorithms part_t *partAssignments = new part_t [numIdsPerProc]; for (int i=0; i < numIdsPerProc; i++){ partAssignments[i] = myGids[i] % numGlobalParts; // round robin } ArrayRCP<part_t> partList = arcp(partAssignments, 0, numIdsPerProc); try{ solution->setParts(gidArray, partList, true); } catch (std::exception &e){ fail=10; } gfail = globalFail(comm, fail); if (gfail){ printFailureCode(comm, fail); // exits after printing "FAIL" } // Test the Solution get methods that may be called by users // or migration functions. if (solution->getLocalNumberOfIds() != size_t(numIdsPerProc)) fail = 11; if (!fail){ const gno_t *gids = solution->getIdList(); for (int i=0; !fail && i < numIdsPerProc; i++){ if (gids[i] != myGids[i]) fail = 12; } } if (!fail){ const part_t *parts = solution->getPartList(); for (int i=0; !fail && i < numIdsPerProc; i++){ if (parts[i] != myGids[i] % numGlobalParts) fail = 13; } } gfail = globalFail(comm, fail); if (gfail){ printFailureCode(comm, fail); // exits after printing "FAIL" } if (rank==0) std::cout << "PASS" << std::endl; /////////////////////////////////////////////////////////////////// // TODO: ///////////// // Create a solution object without part size information, and check it. ///////////// // Test multiple weights. ///////////// // Test multiple parts per process. ///////////// // Specify a list of parts of size 0. (The rest should be uniform.) delete [] lengths; for (int w=0; w < maxNumWeights; w++){ delete [] idLists[w]; delete [] sizeLists[w]; } delete [] idLists; delete [] sizeLists; }
/*! \brief Print all the metrics */ void printMetrics(ostream &os) const { Zoltan2::printMetrics<scalar_t>(os, targetGlobalParts_, numGlobalParts_, numNonEmpty_, metrics_.view(0, metrics_.size())); }
size_t computeLocalEdgeList( const RCP<const Environment> &env, const RCP<const Comm<int> > &comm, size_t numLocalEdges, // local edges size_t numLocalGraphEdges, // edges in "local" graph RCP<const IdentifierMap<User> > &idMap, ArrayRCP<const typename InputTraits<User>::zgid_t> &allEdgeIds, // in ArrayRCP<const typename InputTraits<User>::gno_t> &allEdgeGnos, // in ArrayRCP<int> &allProcs, // in ArrayRCP<const typename InputTraits<User>::lno_t> &allOffs, // in ArrayRCP<StridedData<typename InputTraits<User>::lno_t, typename InputTraits<User>::scalar_t> > &allWeights,// in ArrayRCP<const typename InputTraits<User>::lno_t> &edgeLocalIds, // ArrayRCP<const typename InputTraits<User>::lno_t> &offsets, // out ArrayRCP<StridedData<typename InputTraits<User>::lno_t, typename InputTraits<User>::scalar_t> > &eWeights) // out { typedef typename InputTraits<User>::zgid_t zgid_t; typedef typename InputTraits<User>::gno_t gno_t; typedef typename InputTraits<User>::scalar_t scalar_t; typedef typename InputTraits<User>::lno_t lno_t; typedef StridedData<lno_t, scalar_t> input_t; int rank = comm->getRank(); bool gnosAreGids = idMap->gnosAreGids(); edgeLocalIds = ArrayRCP<const lno_t>(Teuchos::null); eWeights = ArrayRCP<input_t>(Teuchos::null); offsets = ArrayRCP<const lno_t>(Teuchos::null); if (numLocalGraphEdges == 0) { // Set the offsets array and return size_t allOffsSize = allOffs.size(); lno_t *offs = new lno_t [allOffsSize]; env->localMemoryAssertion(__FILE__, __LINE__, allOffsSize, offs); for (size_t i = 0; i < allOffsSize; i++) offs[i] = 0; offsets = arcp(offs, 0, allOffsSize, true); return 0; } if (numLocalGraphEdges == numLocalEdges){ // Entire graph is local. lno_t *lnos = new lno_t [numLocalGraphEdges]; env->localMemoryAssertion(__FILE__, __LINE__, numLocalGraphEdges, lnos); if (comm->getSize() == 1) { // With one rank, Can use gnos as local index. if (gnosAreGids) for (size_t i=0; i < numLocalEdges; i++) lnos[i] = allEdgeIds[i]; else for (size_t i=0; i < numLocalEdges; i++) lnos[i] = allEdgeGnos[i]; } else { ArrayRCP<gno_t> gnoArray; if (gnosAreGids){ ArrayRCP<const gno_t> gnosConst = arcp_reinterpret_cast<const gno_t>(allEdgeIds); gnoArray = arcp_const_cast<gno_t>(gnosConst); } else { gnoArray = arcp_const_cast<gno_t>(allEdgeGnos); } // Need to translate to gnos to local indexing ArrayView<lno_t> lnoView(lnos, numLocalGraphEdges); try { idMap->lnoTranslate(lnoView, gnoArray.view(0,numLocalGraphEdges), TRANSLATE_LIB_TO_APP); } Z2_FORWARD_EXCEPTIONS; } edgeLocalIds = arcp(lnos, 0, numLocalGraphEdges, true); offsets = allOffs; eWeights = allWeights; }
void timeTpetra (const TPETRA_GO numGlobalCoords, const Teuchos::RCP<const Teuchos::Comm<int> >& comm, const bool doMemory) { using Teuchos::arcp; using Teuchos::ArrayRCP; using Teuchos::ArrayView; using Teuchos::Comm; using Teuchos::RCP; using Teuchos::rcp; using std::cout; using std::endl; typedef Tpetra::Map<TPETRA_LO, TPETRA_GO> map_type; typedef Tpetra::MultiVector<TPETRA_SCALAR, TPETRA_LO, TPETRA_GO> MV; typedef ArrayView<const TPETRA_SCALAR> coordList_t; const int nprocs = comm->getSize (); const int rank = comm->getRank (); ///////////// Step 1 ////////////////////////////////// // Create a MV with contiguous global IDs const TPETRA_LO numLocalCoords = numSequentialGlobalIds(numGlobalCoords, nprocs, rank); RCP<const map_type> tmap; RCP<MV> mvector; TPETRA_SCALAR* coords = NULL; { Teuchos::TimeMonitor timeMon (*tmvBuild); tmap = rcp (new map_type (numGlobalCoords, numLocalCoords, 0, comm)); coords = new TPETRA_SCALAR [COORDDIM * numLocalCoords]; memset (coords, 0, sizeof(TPETRA_SCALAR) * numLocalCoords * COORDDIM); coordList_t *avList = new coordList_t [COORDDIM]; TPETRA_LO offset = 0; for (int dim = 0; dim < COORDDIM; ++dim) { avList[dim] = coordList_t(coords + offset, numLocalCoords); offset += numLocalCoords; } ArrayRCP<const coordList_t> vectors = arcp (avList, 0, COORDDIM); mvector = rcp (new MV (tmap, vectors.view (0, COORDDIM), COORDDIM)); } if (rank == 0 && doMemory) { const long nkb = Zoltan2::getProcessKilobytes (); cout << "Create mvector 1: " << nkb << endl; } ///////////// Step 2 ////////////////////////////////// // Migrate the MV. ArrayRCP<const TPETRA_GO> newGidArray; { TPETRA_GO *newGids = NULL; roundRobinGlobalIds<TPETRA_GO> (numGlobalCoords, nprocs, rank, newGids); newGidArray = arcp<const TPETRA_GO> (newGids, 0, numLocalCoords, true); } RCP<const map_type> newTmap; RCP<Tpetra::Import<TPETRA_LO, TPETRA_GO> > importer; RCP<MV> newMvector; { Teuchos::TimeMonitor timeMon (*tmvMigrate); newTmap = rcp (new map_type (numGlobalCoords, newGidArray.view(0, numLocalCoords), 0, comm)); importer = rcp (new Tpetra::Import<TPETRA_LO, TPETRA_GO> (tmap, newTmap)); newMvector = rcp (new MV (newTmap, COORDDIM, true)); newMvector->doImport (*mvector, *importer, Tpetra::INSERT); mvector = newMvector; } delete [] coords; if (rank == 0 && doMemory) { const long nkb = Zoltan2::getProcessKilobytes (); cout << "Create mvector 2: " << nkb << endl; } ///////////// Step 3 ////////////////////////////////// // Divide processes into two halves. RCP<Comm<int> > subComm; { int groupSize = 0; int leftHalfNumProcs = nprocs / 2; int *myHalfProcs = NULL; if (rank < leftHalfNumProcs){ groupSize = leftHalfNumProcs; myHalfProcs = new int [groupSize]; for (int i=0; i < groupSize; i++) myHalfProcs[i] = i; } else { groupSize = nprocs - leftHalfNumProcs; myHalfProcs = new int [groupSize]; int firstNum = leftHalfNumProcs; for (int i=0; i < groupSize; i++) myHalfProcs[i] = firstNum++; } ArrayView<const int> idView(myHalfProcs, groupSize); subComm = comm->createSubcommunicator (idView); delete [] myHalfProcs; } // Divide the multivector into two. Each process group is creating // a multivector with non-contiguous global ids. For one group, // base gid is not 0. size_t globalSize = Teuchos::OrdinalTraits<size_t>::invalid (); RCP<map_type> subMap; RCP<MV> subMvector; { Teuchos::TimeMonitor timeMon (*tmvBuildN); ArrayView<const TPETRA_GO> gidList = mvector->getMap ()->getNodeElementList (); subMap = rcp (new map_type (globalSize, gidList, 0, subComm)); globalSize = subMap->getGlobalNumElements (); // Get a view of the block of rows to copy. RCP<MV> tmp = mvector->offsetViewNonConst (subMap, 0); // Create a new multivector to hold the group's rows. subMvector = rcp (new MV (subMap, mvector->getNumVectors ())); // Copy the view into the new multivector. Tpetra::deep_copy (*subMvector, *tmp); } ///////////// Step 4 ////////////////////////////////// // Each subgroup migrates the sub-multivector so the // global Ids are increasing with process rank. TPETRA_GO *increasingGids = NULL; subGroupGloballyIncreasingIds<TPETRA_GO> (numGlobalCoords, nprocs, rank, increasingGids); ArrayRCP<const TPETRA_GO> incrGidArray (increasingGids, 0, numLocalCoords, true); RCP<const map_type> newSubMap; RCP<Tpetra::Import<TPETRA_LO, TPETRA_GO> > subImporter; RCP<MV> newSubMvector; { Teuchos::TimeMonitor timeMon (*tmvMigrateN); newSubMap = rcp (new map_type (globalSize, incrGidArray.view (0, numLocalCoords), 0, subComm)); subImporter = rcp (new Tpetra::Import<TPETRA_LO, TPETRA_GO> (subMap, newSubMap)); newSubMvector = rcp (new MV (newSubMap, COORDDIM, true)); newSubMvector->doImport (*subMvector, *subImporter, Tpetra::INSERT); mvector = newSubMvector; } }