bool Tpetra::Utils::parseRfmt(Teuchos::ArrayRCP<char> fmt, int &perline, int &width, int &prec, char &valformat) { TEUCHOS_TEST_FOR_EXCEPT(fmt.size() != 0 && fmt[fmt.size()-1] != '\0'); std::transform(fmt.begin(), fmt.end(), fmt, static_cast < int(*)(int) > (std::toupper)); // find the first left paren '(' and the last right paren ')' Teuchos::ArrayRCP<char>::iterator firstLeftParen = std::find( fmt.begin(), fmt.end(), '('); Teuchos::ArrayRCP<char>::iterator lastRightParen = std::find(std::reverse_iterator<Teuchos::ArrayRCP<char>::iterator>(fmt.end()), std::reverse_iterator<Teuchos::ArrayRCP<char>::iterator>(fmt.begin()), ')').base()-1; // select the substring between the parens, including them // if neither was found, set the string to empty if (firstLeftParen == fmt.end() || lastRightParen == fmt.begin()) { fmt.resize(0 + 1); fmt[0] = '\0'; } else { fmt += (firstLeftParen - fmt.begin()); size_t newLen = lastRightParen - firstLeftParen + 1; fmt.resize(newLen + 1); fmt[newLen] = '\0'; } if (std::find(fmt.begin(),fmt.end(),'P') != fmt.end()) { // not supported return true; } bool error = true; if (std::sscanf(fmt.getRawPtr(),"(%d%c%d.%d)",&perline,&valformat,&width,&prec) == 4) { if (valformat == 'E' || valformat == 'D' || valformat == 'F') { error = false; } } return error; }
void dump(const MV& v, const std::string& name) { std::cout << name << std::endl; Teuchos::ArrayRCP<const Scalar> view = v.get1dView(); for (Teuchos::ArrayRCP<const Scalar>::iterator it = view.begin(); it != view.end(); ++it) { std::cout << *it << std::endl; } std::cout << std::endl; }
void Tpetra::Utils::readHBHeader(std::ifstream &fin, Teuchos::ArrayRCP<char> &Title, Teuchos::ArrayRCP<char> &Key, Teuchos::ArrayRCP<char> &Type, int &Nrow, int &Ncol, int &Nnzero, int &Nrhs, Teuchos::ArrayRCP<char> &Ptrfmt, Teuchos::ArrayRCP<char> &Indfmt, Teuchos::ArrayRCP<char> &Valfmt, Teuchos::ArrayRCP<char> &Rhsfmt, int &Ptrcrd, int &Indcrd, int &Valcrd, int &Rhscrd, Teuchos::ArrayRCP<char> &Rhstype) { int Totcrd, Neltvl, Nrhsix; const int MAXLINE = 81; char line[MAXLINE]; // Title.resize(72 + 1); std::fill(Title.begin(), Title.end(), '\0'); Key.resize(8 + 1); std::fill(Key.begin(), Key.end(), '\0'); Type.resize(3 + 1); std::fill(Type.begin(), Type.end(), '\0'); Ptrfmt.resize(16 + 1); std::fill(Ptrfmt.begin(), Ptrfmt.end(), '\0'); Indfmt.resize(16 + 1); std::fill(Indfmt.begin(), Indfmt.end(), '\0'); Valfmt.resize(20 + 1); std::fill(Valfmt.begin(), Valfmt.end(), '\0'); Rhsfmt.resize(20 + 1); std::fill(Rhsfmt.begin(), Rhsfmt.end(), '\0'); // const std::string errStr("Tpetra::Utils::readHBHeader(): Improperly formatted H/B file: "); /* First line: (A72,A8) */ fin.getline(line,MAXLINE); TEUCHOS_TEST_FOR_EXCEPTION( std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line."); (void)std::sscanf(line, "%72c%8[^\n]", Title.getRawPtr(), Key.getRawPtr()); /* Second line: (5I14) or (4I14) */ fin.getline(line,MAXLINE); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line."); if ( std::sscanf(line,"%14d%14d%14d%14d%14d",&Totcrd,&Ptrcrd,&Indcrd,&Valcrd,&Rhscrd) != 5 ) { Rhscrd = 0; TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%14d%14d%14d%14d",&Totcrd,&Ptrcrd,&Indcrd,&Valcrd) != 4, std::runtime_error, errStr << "error reading pointers (line 2)"); } /* Third line: (A3, 11X, 4I14) */ fin.getline(line,MAXLINE); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line."); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line, "%3c%14i%14i%14i%14i", Type.getRawPtr(),&Nrow,&Ncol,&Nnzero,&Neltvl) != 5 , std::runtime_error, errStr << "error reading matrix meta-data (line 3)"); std::transform(Type.begin(), Type.end(), Type.begin(), static_cast < int(*)(int) > (std::toupper)); /* Fourth line: */ fin.getline(line,MAXLINE); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line."); if (Rhscrd != 0) { TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%16c%16c%20c%20c",Ptrfmt.getRawPtr(),Indfmt.getRawPtr(),Valfmt.getRawPtr(),Rhsfmt.getRawPtr()) != 4, std::runtime_error, errStr << "error reading formats (line 4)"); } else { TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%16c%16c%20c",Ptrfmt.getRawPtr(),Indfmt.getRawPtr(),Valfmt.getRawPtr()) != 3, std::runtime_error, errStr << "error reading formats (line 4)"); } /* (Optional) Fifth line: */ if (Rhscrd != 0 ) { Rhstype.resize(3 + 1,'\0'); fin.getline(line,MAXLINE); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line."); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%3c%14d%14d", Rhstype.getRawPtr(), &Nrhs, &Nrhsix) != 3, std::runtime_error, errStr << "error reading right-hand-side meta-data (line 5)"); } }
//----------------------------------------------------------------------------- double TpetraVector::sum() const { dolfin_assert(!_x.is_null()); Teuchos::ArrayRCP<const double> arr = _x->getData(0); const double _sum = std::accumulate(arr.begin(), arr.end(), 0.0); return MPI::sum(mpi_comm(), _sum); }
bool Tpetra::Utils::parseIfmt(Teuchos::ArrayRCP<char> fmt, int &perline, int &width) { TEUCHOS_TEST_FOR_EXCEPT(fmt.size() != 0 && fmt[fmt.size()-1] != '\0'); // parses integers n and d out of (nId) bool error = true; std::transform(fmt.begin(), fmt.end(), fmt, static_cast < int(*)(int) > (std::toupper)); if (std::sscanf(fmt.getRawPtr(),"(%dI%d)",&perline,&width) == 2) { error = false; } return error; }
void Filtered_UniqueGlobalIndexer<LocalOrdinalT,GlobalOrdinalT>:: getOwnedAndGhostedNotFilteredIndicator(std::vector<int> & indicator) const { using Teuchos::RCP; typedef GlobalOrdinalT GO; typedef LocalOrdinalT LO; typedef panzer::TpetraNodeType Node; typedef Tpetra::Map<LO, GO, Node> Map; typedef Tpetra::Vector<GO,LO,GO,Node> Vector; typedef Tpetra::Import<LO,GO,Node> Import; std::vector<GlobalOrdinalT> ownedIndices; std::vector<GlobalOrdinalT> ghostedIndices; // build owned and ghosted maps getOwnedIndices(ownedIndices); getOwnedAndGhostedIndices(ghostedIndices); RCP<const Map> ownedMap = Tpetra::createNonContigMap<LO,GO>(ownedIndices,getComm()); RCP<const Map> ghostedMap = Tpetra::createNonContigMap<LO,GO>(ghostedIndices,getComm()); // allocate the owned vector, mark those GIDs as unfiltered // (they are by definition) Vector ownedActive(ownedMap); ownedActive.putScalar(1); // Initialize all indices to zero Vector ghostedActive(ghostedMap); ghostedActive.putScalar(0); // do communication, marking unfiltered indices as 1 (filtered // indices locally are marked as zero) Import importer(ownedMap,ghostedMap); ghostedActive.doImport(ownedActive,importer,Tpetra::INSERT); Teuchos::ArrayRCP<const GO> data = ghostedActive.getData(); // copy communicated data (clear it out first) indicator.clear(); indicator.insert(indicator.end(),data.begin(),data.end()); }
//---------------------------------------------------------------------------// TEUCHOS_UNIT_TEST( FieldTools, tensor_field_test ) { using namespace DataTransferKit; // Setup random numbers for the test. int num_rand = 4; double rand_max = 10.0; Teuchos::Array<double> random_numbers( num_rand ); std::srand( 1 ); for ( int i = 0; i < num_rand; ++i ) { random_numbers[i] = rand_max * (double) std::rand() / RAND_MAX; } // Setup communication. Teuchos::RCP< const Teuchos::Comm<int> > comm = getDefaultComm<int>(); int my_rank = comm->getRank(); int my_size = comm->getSize(); // Setup a field manager. int field_dim = 9; typedef FieldTraits<ArrayField> FT; Teuchos::RCP<ArrayField> array_field = Teuchos::rcp( new ArrayField( field_dim*(my_rank+1), field_dim ) ); FieldManager<ArrayField> field_manager( array_field, comm ); // Test the field tools. typedef FieldTools<ArrayField> Tools; // Dimension iterators. FT::iterator dim_begin, dim_end; for ( int d = 0; d < field_dim; ++d ) { dim_begin = Tools::dimBegin( *field_manager.field(), d ); dim_end = Tools::dimEnd( *field_manager.field(), d ); TEST_ASSERT( std::distance( dim_begin, dim_end ) == my_rank+1 ); } FT::const_iterator const_dim_begin, const_dim_end; for ( int d = 0; d < field_dim; ++d ) { const_dim_begin = Tools::dimBegin( *field_manager.field(), d ); const_dim_end = Tools::dimEnd( *field_manager.field(), d ); TEST_ASSERT( std::distance( const_dim_begin, const_dim_end ) == my_rank+1 ); } // Length. int global_size = 0; for ( int i = 0; i < my_size; ++i ) { global_size += field_dim*(i+1); } TEST_ASSERT( Tools::globalSize( *field_manager.field(), field_manager.comm() ) == global_size ); // Scalar filling. Tools::putScalar( *field_manager.field(), random_numbers[0] ); // Views. double local_val; Teuchos::ArrayRCP<const double> const_view = Tools::view( *field_manager.field() ); Teuchos::ArrayRCP<const double>::const_iterator const_view_iterator; for ( const_view_iterator = const_view.begin(); const_view_iterator != const_view.end(); ++const_view_iterator ) { local_val = random_numbers[0]; TEST_ASSERT( *const_view_iterator == local_val ); } Teuchos::ArrayRCP<double> view = Tools::nonConstView( *field_manager.field() ); Teuchos::ArrayRCP<double>::iterator view_iterator; for ( view_iterator = view.begin(); view_iterator != view.end(); ++view_iterator ) { local_val = random_numbers[0]; TEST_ASSERT( *view_iterator == local_val ); } Teuchos::ArrayRCP<double> copy = Tools::copy( *field_manager.field() ); Teuchos::ArrayRCP<double>::const_iterator copy_iterator; for ( copy_iterator = copy.begin(); copy_iterator != copy.end(); ++copy_iterator ) { local_val = random_numbers[0]; TEST_ASSERT( *copy_iterator == local_val ); } // Multiple scalar filling. Teuchos::Array<double> fillers( field_dim ); for ( int d = 0; d < field_dim; ++d ) { fillers[d] = (d+1)*random_numbers[1]; } Tools::putScalar( *field_manager.field(), fillers() ); // Dim iterators. FT::const_iterator const_dim_iterator; for ( int d = 0; d < field_dim; ++d ) { for ( const_dim_iterator = Tools::dimBegin( *field_manager.field(), d ); const_dim_iterator != Tools::dimEnd( *field_manager.field(), d ); ++const_dim_iterator ) { local_val = (d+1) * random_numbers[1]; TEST_ASSERT( *const_dim_iterator == local_val ); } } FT::iterator dim_iterator; for ( int d = 0; d < field_dim; ++d ) { for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d ); dim_iterator != Tools::dimEnd( *field_manager.field(), d ); ++dim_iterator ) { local_val = (d+1) * random_numbers[1]; TEST_ASSERT( *dim_iterator == local_val ); } } // Scaling. Tools::scale( *field_manager.field(), random_numbers[2] ); for ( int d = 0; d < field_dim; ++d ) { for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d ); dim_iterator != Tools::dimEnd( *field_manager.field(), d ); ++dim_iterator ) { local_val = (d+1) * random_numbers[1] * random_numbers[2]; TEST_ASSERT( *dim_iterator == local_val ); } } // Multiple scaling. Teuchos::Array<double> multipliers( field_dim ); for ( int d = 0; d < field_dim; ++d ) { multipliers[d] = -(d+1) - random_numbers[3]; } Tools::scale( *field_manager.field(), multipliers() ); for ( int d = 0; d < field_dim; ++d ) { for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d ); dim_iterator != Tools::dimEnd( *field_manager.field(), d ); ++dim_iterator ) { local_val = (d+1) * random_numbers[1] * random_numbers[2] * (-(d+1) - random_numbers[3]); TEST_ASSERT( *dim_iterator == local_val ); } } // Norms. Teuchos::Array<double> inf_norms; Tools::normInf( *field_manager.field(), field_manager.comm(), inf_norms ); for ( int d = 0; d < field_dim; ++d ) { local_val = (d+1) * random_numbers[1] * random_numbers[2] * (-(d+1) - random_numbers[3]); TEST_ASSERT( softEquivalence( inf_norms[d], std::abs( local_val ) ) ); } Teuchos::Array<double> one_norms; Tools::norm1( *field_manager.field(), field_manager.comm(), one_norms ); for ( int d = 0; d < field_dim; ++d ) { local_val = ( global_size / FT::dim( *field_manager.field() ) ) * (d+1) * random_numbers[1] * random_numbers[2] * (-(d+1) - random_numbers[3]); TEST_ASSERT( softEquivalence( one_norms[d], std::abs( local_val ) ) ); } Teuchos::Array<double> two_norms; Tools::norm2( *field_manager.field(), field_manager.comm(), two_norms ); for ( int d = 0; d < field_dim; ++d ) { local_val = ( global_size / FT::dim( *field_manager.field() ) ) * std::pow( std::abs( (d+1) * random_numbers[1] * random_numbers[2] * (-(d+1) - random_numbers[3]) ), 2.0 ); local_val = std::pow( local_val, 1.0/2.0 ); TEST_ASSERT( softEquivalence( two_norms[d], local_val ) ); } Teuchos::Array<double> q_norms; int q = std::floor( (10 * std::rand()) / RAND_MAX ) + 1; Tools::normQ( *field_manager.field(), field_manager.comm(), q, q_norms ); for ( int d = 0; d < field_dim; ++d ) { local_val = ( global_size / FT::dim( *field_manager.field() ) ) * std::pow( std::abs( (d+1) * random_numbers[1] * random_numbers[2] * (-(d+1) - random_numbers[3]) ), q ); local_val = std::pow( local_val, 1.0/q ); TEST_ASSERT( softEquivalence( q_norms[d], local_val ) ); } // Average. Teuchos::Array<double> averages; Tools::average( *field_manager.field(), field_manager.comm(), averages ); for ( int d = 0; d < field_dim; ++d ) { local_val = (d+1) * random_numbers[1] * random_numbers[2] * (-(d+1) - random_numbers[3]); TEST_ASSERT( softEquivalence( averages[d], local_val ) ); } }
void Tpetra::Utils::readHBMatDouble(const std::string &filename, int &numRows, int &numCols, int &numNZ, std::string &type, Teuchos::ArrayRCP<int> &colPtrs, Teuchos::ArrayRCP<int> &rowInds, Teuchos::ArrayRCP<double> &vals) { // NOTE: if complex, allocate enough space for 2*NNZ and interleave real and imaginary parts (real,imag) // if pattern, don't touch parameter vals; do not allocate space space for values try { std::ifstream fin; int ptrCrd, indCrd, valCrd; Teuchos::ArrayRCP<char> Title, Key, Ptrfmt, Indfmt, Valfmt; const int MAXSIZE = 81; char lineBuf[MAXSIZE]; // nitty gritty int ptrsPerLine, ptrWidth, indsPerLine, indWidth, valsPerLine, valWidth, valPrec; char valFlag; // fin.open(filename.c_str(),std::ifstream::in); { // we don't care about RHS-related stuff, so declare those vars in an expiring scope int Nrhs, rhsCrd; Teuchos::ArrayRCP<char> Rhstype, Rhsfmt; Teuchos::ArrayRCP<char> TypeArray; Tpetra::Utils::readHBHeader(fin, Title, Key, TypeArray, numRows, numCols, numNZ, Nrhs, Ptrfmt, Indfmt, Valfmt, Rhsfmt, ptrCrd, indCrd, valCrd, rhsCrd, Rhstype); if (TypeArray.size() > 0) { type.resize(TypeArray.size()-1); std::copy(TypeArray.begin(), TypeArray.end(), type.begin()); } } const std::string errStr("Tpetra::Utils::readHBHeader(): Improperly formatted H/B file."); const bool readPatternOnly = (type[0] == 'P' || type[0] == 'p'); const bool readComplex = (type[0] == 'C' || type[0] == 'c'); /* Parse the array input formats from Line 3 of HB file */ TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseIfmt(Ptrfmt,ptrsPerLine,ptrWidth) == true, std::runtime_error, "Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format."); TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseIfmt(Indfmt,indsPerLine,indWidth) == true, std::runtime_error, "Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format."); if (readPatternOnly == false) { TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseRfmt(Valfmt,valsPerLine,valWidth,valPrec,valFlag) == true, std::runtime_error, "Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format."); } // special case this: the reason is that the number of colPtrs read is numCols+1, which is non-zero even if numCols == 0 // furthermore, if numCols == 0, there is nothing of interest to read if (numCols == 0) return; // allocate space for column pointers, row indices, and values // if the file is empty, do not touch these ARCPs colPtrs = Teuchos::arcp<int>(numCols+1); if (numNZ > 0) { rowInds = Teuchos::arcp<int>(numNZ); if (readPatternOnly == false) { if (readComplex) { vals = Teuchos::arcp<double>(2*numNZ); } else { vals = Teuchos::arcp<double>(numNZ); } } } /* Read column pointer array: Specifically, read ptrCrd number of lines, and on each line, read ptrsPerLine number of integers, each of width ptrWidth Store these in colPtrs */ { int colPtrsRead = 0; char NullSub = '\0'; for (int lno=0; lno < ptrCrd; ++lno) { fin.getline(lineBuf, MAXSIZE); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr); char *linePtr = lineBuf; for (int ptr=0; ptr < ptrsPerLine; ++ptr) { if (colPtrsRead == numCols + 1) break; int cptr; // terminate the string at the end of the current ptr block, saving the character in that location std::swap(NullSub,linePtr[ptrWidth]); // read the ptr std::sscanf(linePtr, "%d", &cptr); // put the saved character back, and put the '\0' back into NullSub for use again std::swap(NullSub,linePtr[ptrWidth]); linePtr += ptrWidth; colPtrs[colPtrsRead++] = cptr; } } TEUCHOS_TEST_FOR_EXCEPT(colPtrsRead != numCols + 1); } /* Read row index array: Specifically, read indCrd number of lines, and on each line, read indsPerLine number of integers, each of width indWidth Store these in rowInds */ { char NullSub = '\0'; int indicesRead = 0; for (int lno=0; lno < indCrd; ++lno) { fin.getline(lineBuf, MAXSIZE); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr); char *linePtr = lineBuf; for (int indcntr=0; indcntr < indsPerLine; ++indcntr) { if (indicesRead == numNZ) break; int ind; // terminate the string at the end of the current ind block, saving the character in that location std::swap(NullSub,linePtr[indWidth]); // read the ind std::sscanf(linePtr, "%d", &ind); // put the saved character back, and put the '\0' back into NullSub for use again std::swap(NullSub,linePtr[indWidth]); linePtr += indWidth; rowInds[indicesRead++] = ind; } } TEUCHOS_TEST_FOR_EXCEPT(indicesRead != numNZ); } /* Read array of values: Specifically, read valCrd number of lines, and on each line, read valsPerLine number of real values, each of width/precision valWidth/valPrec Store these in vals If readComplex, then read twice as many non-zeros, and interleave real,imag into vals */ if (readPatternOnly == false) { int totalNumVals; if (readComplex) { totalNumVals = 2*numNZ; } else { totalNumVals = numNZ; } char NullSub = '\0'; int valsRead = 0; for (int lno=0; lno < valCrd; ++lno) { fin.getline(lineBuf, MAXSIZE); TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr); // if valFlag == 'D', then we need to convert [dD] in fp vals into [eE] that scanf can parse if (valFlag == 'D') std::replace_if(lineBuf, lineBuf+MAXSIZE, std::bind2nd(std::equal_to<char>(), 'D'), 'E'); char *linePtr = lineBuf; for (int valcntr=0; valcntr < valsPerLine; ++valcntr) { if (valsRead == totalNumVals) break; double val; // terminate the string at the end of the current val block, saving the character in that location std::swap(NullSub,linePtr[valWidth]); // read the val std::sscanf(linePtr, "%le", &val); // put the saved character back, and put the '\0' back into NullSub for use again std::swap(NullSub,linePtr[valWidth]); linePtr += valWidth; vals[valsRead++] = val; } } TEUCHOS_TEST_FOR_EXCEPT(valsRead != totalNumVals); } fin.close(); } catch (std::exception &e) { TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, "Tpetra::Utils::readHBInfo() of filename \"" << filename << "\" caught exception: " << std::endl << e.what() << std::endl); } }
void Tpetra::Utils::readHBMatrix(const std::string &filename, const Teuchos::RCP<const Teuchos::Comm<int> > &comm, const Teuchos::RCP<Node> &node, Teuchos::RCP< Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> > &A, Teuchos::RCP< const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > rowMap, const Teuchos::RCP<ParameterList> ¶ms) { const int myRank = comm->getRank(); int numRows,numCols,numNZ; Teuchos::ArrayRCP<Scalar> svals; Teuchos::ArrayRCP<GlobalOrdinal> colinds; Teuchos::ArrayRCP<int> rowptrs; Teuchos::ArrayRCP<size_t> nnzPerRow; int fail = 0; if (myRank == 0) { bool isSymmetric=false; Teuchos::ArrayRCP<double> dvals; Teuchos::ArrayRCP<int> colptrs, rowinds; std::string type; Tpetra::Utils::readHBMatDouble(filename,numRows,numCols,numNZ,type,colptrs,rowinds,dvals); TEUCHOS_TEST_FOR_EXCEPT(type.size() != 3); if (type[0] != 'R' && type[0] != 'r') { // only real matrices right now fail = 1; } if (fail == 0 && numNZ > 0) { if (type[1] == 'S' || type[1] == 's') { isSymmetric = true; } else { isSymmetric = false; } } if (fail == 0 && numNZ > 0) { // find num non-zero per row nnzPerRow = Teuchos::arcp<size_t>(numRows); std::fill(nnzPerRow.begin(), nnzPerRow.end(), 0); for (Teuchos::ArrayRCP<int>::const_iterator ri=rowinds.begin(); ri != rowinds.end(); ++ri) { // count each row index towards its row ++nnzPerRow[*ri-1]; } if (isSymmetric) { // count each column toward the corresponding row as well for (int c=0; c < numCols; ++c) { // the diagonal was already counted; neglect it, if it exists for (int i=colptrs[c]-1; i != colptrs[c+1]-1; ++i) { if (rowinds[i] != c+1) { ++nnzPerRow[c]; ++numNZ; } } } } // allocate/set new matrix data svals = Teuchos::arcp<Scalar>(numNZ); colinds = Teuchos::arcp<GlobalOrdinal>(numNZ); rowptrs = Teuchos::arcp<int>(numRows+1); rowptrs[0] = 0; #ifdef HAVE_TPETRA_DEBUG Teuchos::ArrayRCP<size_t> nnzPerRow_debug(nnzPerRow.size()); std::copy(nnzPerRow.begin(), nnzPerRow.end(), nnzPerRow_debug.begin()); #endif for (int j=1; j <= numRows; ++j) { rowptrs[j] = rowptrs[j-1] + nnzPerRow[j-1]; nnzPerRow[j-1] = 0; } // translate from column-oriented to row-oriented for (int col=0; col<numCols; ++col) { for (int i=colptrs[col]-1; i != colptrs[col+1]-1; ++i) { const int row = rowinds[i]-1; // add entry to (row,col), with value dvals[i] const size_t entry = rowptrs[row] + nnzPerRow[row]; svals[entry] = Teuchos::as<Scalar>(dvals[i]); colinds[entry] = Teuchos::as<GlobalOrdinal>(col); ++nnzPerRow[row]; if (isSymmetric && row != col) { // add entry to (col,row), with value dvals[i] const size_t symentry = rowptrs[col] + nnzPerRow[col]; svals[symentry] = Teuchos::as<Scalar>(dvals[i]); colinds[symentry] = Teuchos::as<GlobalOrdinal>(row); ++nnzPerRow[col]; } } } #ifdef HAVE_TPETRA_DEBUG { bool isequal = true; typename Teuchos::ArrayRCP<size_t>::const_iterator it1, it2; for (it1 = nnzPerRow.begin(), it2 = nnzPerRow_debug.begin(); it1 != nnzPerRow.end(); ++it1, ++it2) { if (*it1 != *it2) { isequal = false; break; } } TEUCHOS_TEST_FOR_EXCEPTION(!isequal || nnzPerRow.size() != nnzPerRow_debug.size(), std::logic_error, "Tpetra::Utils::readHBMatrix(): Logic error."); } #endif } // std::cout << "Matrix " << filename << " of type " << type << ": " << numRows << " by " << numCols << ", " << numNZ << " nonzeros" << std::endl; } // check for read errors broadcast(*comm,0,&fail); TEUCHOS_TEST_FOR_EXCEPTION(fail == 1, std::runtime_error, "Tpetra::Utils::readHBMatrix() can only read Real matrices."); // distribute global matrix info broadcast(*comm,0,&numRows); broadcast(*comm,0,&numCols); // create map with uniform partitioning if (rowMap == Teuchos::null) { rowMap = Teuchos::rcp(new Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node>((global_size_t)numRows,(GlobalOrdinal)0,comm,GloballyDistributed,node)); } else { TEUCHOS_TEST_FOR_EXCEPTION( rowMap->getGlobalNumElements() != (global_size_t)numRows, std::runtime_error, "Tpetra::Utils::readHBMatrix(): specified map has incorrect number of elements."); TEUCHOS_TEST_FOR_EXCEPTION( rowMap->isDistributed() == false && comm->getSize() > 1, std::runtime_error, "Tpetra::Utils::readHBMatrix(): specified map is not distributed."); } Teuchos::ArrayRCP<size_t> myNNZ; if (rowMap->getNodeNumElements()) { myNNZ = Teuchos::arcp<size_t>(rowMap->getNodeNumElements()); } if (myRank == 0) { LocalOrdinal numRowsAlreadyDistributed = rowMap->getNodeNumElements(); std::copy(nnzPerRow.begin(), nnzPerRow.begin()+numRowsAlreadyDistributed,myNNZ); for (int p=1; p < Teuchos::size(*comm); ++p) { size_t numRowsForP; Teuchos::receive(*comm,p,&numRowsForP); if (numRowsForP) { Teuchos::send<int,size_t>(*comm,numRowsForP,nnzPerRow(numRowsAlreadyDistributed,numRowsForP).getRawPtr(),p); numRowsAlreadyDistributed += numRowsForP; } } } else { const size_t numMyRows = rowMap->getNodeNumElements(); Teuchos::send(*comm,numMyRows,0); if (numMyRows) { Teuchos::receive<int,size_t>(*comm,0,numMyRows,myNNZ(0,numMyRows).getRawPtr()); } } nnzPerRow = Teuchos::null; // create column map Teuchos::RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > domMap; if (numRows == numCols) { domMap = rowMap; } else { domMap = createUniformContigMapWithNode<LocalOrdinal,GlobalOrdinal,Node>(numCols,comm,node); } A = rcp(new Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>(rowMap,myNNZ,Tpetra::StaticProfile)); // free this locally, A will keep it allocated as long as it is needed by A (up until allocation of nonzeros) myNNZ = Teuchos::null; if (myRank == 0 && numNZ > 0) { for (int r=0; r < numRows; ++r) { const LocalOrdinal nnz = rowptrs[r+1] - rowptrs[r]; if (nnz > 0) { Teuchos::ArrayView<const GlobalOrdinal> inds = colinds(rowptrs[r],nnz); Teuchos::ArrayView<const Scalar> vals = svals( rowptrs[r],nnz); A->insertGlobalValues(r, inds, vals); } } } // don't need these anymore colinds = Teuchos::null; svals = Teuchos::null; rowptrs = Teuchos::null; A->fillComplete(domMap,rowMap,params); }
TEUCHOS_UNIT_TEST(Zoltan, Build) { typedef Teuchos::ScalarTraits<Scalar> ST; out << "version: " << MueLu::Version() << std::endl; out << std::endl; out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl; out << "that has a random number of nonzeros per row. Good results have been precomputed" << std::endl; out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl; out << "once the Zoltan repartitioning has been applied." << std::endl; out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl; RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm(); if (comm->getSize() > 5) { out << std::endl; out << "This test must be run on 1 to 5 processes." << std::endl; TEST_EQUALITY(true, true); return; } Level level; RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager()); level.SetFactoryManager(factoryHandler); int nx=7; int ny=nx; GO numGlobalElements = nx*ny; size_t maxEntriesPerRow=30; // Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row. RCP<const Map> map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm); const size_t numMyElements = map->getNodeNumElements(); Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList(); RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more // memory on the fly. While not super efficient, this // ensures that no zeros are being stored. Thus, from // Zoltan's perspective the matrix is imbalanced. // Create a vector with random integer entries in [1,maxEntriesPerRow]. ST::seedrandom(666*comm->getRank()); RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false); Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0); for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) { *i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1)); } RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); fos->setOutputToRootOnly(-1); Teuchos::Array<Scalar> vals(maxEntriesPerRow); Teuchos::Array<GO> cols(maxEntriesPerRow); for (size_t i = 0; i < numMyElements; ++i) { Teuchos::ArrayView<SC> av(&vals[0],eprData[i]); Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]); //stick in ones for values for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one(); //figure out valid column indices GO start = std::max(myGlobalElements[i]-eprData[i]+1,0); for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j; A->insertGlobalValues(myGlobalElements[i], iv, av); } A->fillComplete(); level.Set("A",A); //build coordinates RCP<const Map> rowMap = A->getRowMap(); Teuchos::ParameterList list; list.set("nx",nx); list.set("ny",ny); RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",rowMap,list); level.Set("Coordinates",XYZ); LO numPartitions = comm->getSize(); level.Set("number of partitions",numPartitions); RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface()); //zoltan->SetNumberOfPartitions(numPartitions); //zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints level.Request("Partition",zoltan.get()); zoltan->Build(level); RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get()); /* //TODO temporary code to have the trivial decomposition (no change) ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0); for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i) *i = comm->getRank(); decompEntries=Teuchos::null; */ //TODO end of temporary code //Create vector whose local length is the global number of partitions. //This vector will record the local number of nonzeros associated with each partition. Teuchos::Array<GO> parts(numPartitions); for (int i=0; i<numPartitions; ++i) parts[i] = i; Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions); RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView, map->getIndexBase(),comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap); //For the local rows in each partition, tally up the number of nonzeros. This is what //Zoltan should be load-balancing. Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0); Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0); for (size_t i=0; i<decomposition->getLocalLength();++i) { Teuchos::ArrayView<const LO> c; Teuchos::ArrayView<const SC> v; A->getLocalRowView(i,c,v); lpvData[decompData[i]] += v.size(); } lpvData = Teuchos::null; decompData = Teuchos::null; //localPartsVec->describe(*fos,Teuchos::VERB_EXTREME); //Send the local nnz tallies to pid 0, which can report the global sums. size_t mysize=1; if (comm->getRank() == 0) mysize = numPartitions; RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), mysize, map->getIndexBase(), comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap); RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap); globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD); ArrayRCP<GO> expectedResults(numPartitions); switch (comm->getSize()) { case 1: expectedResults[0] = 807; break; case 2: expectedResults[0] = 364; expectedResults[1] = 363; break; case 3: expectedResults[0] = 277; expectedResults[1] = 261; expectedResults[2] = 269; break; case 4: expectedResults[0] = 195; expectedResults[1] = 186; expectedResults[2] = 177; expectedResults[3] = 168; break; case 5: expectedResults[0] = 161; expectedResults[1] = 145; expectedResults[2] = 148; expectedResults[3] = 159; expectedResults[4] = 157; break; default: break; }; //FIXME cool ... this next line causes a hang if locally the globalyTallyVec has no data. //FIXME I get around this by making mysize (above) 1 instead of 0. Is this a bug or feature //FIXME in getData? ArrayRCP<const LO> gtvData = globalTallyVec->getData(0); #ifdef __linux__ out << "Checking results..." << std::endl; for (int i=0; i<numPartitions; ++i) { if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]); } #endif #ifdef MUELU_VISUALIZE_REPARTITIONING // //Now write everything to a comma-separate list that ParaView can grok // Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0); Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1); Teuchos::ArrayRCP<const GO> D = decomposition->getData(0); RCP<std::ofstream> outFile; std::string fileName = "zoltanResults.csv"; //write header information if (comm->getRank() == 0) { outFile = rcp(new std::ofstream(fileName.c_str())); *outFile << "x coord, y coord, z coord, scalar" << std::endl; } comm->barrier(); //append coordinates for (int j=0; j<comm->getSize(); ++j) { int mypid = comm->getRank(); if (mypid == j) { outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app)); for (int i=0; i < D.size(); ++i) { *outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i] << std::endl; } } } //for (int i=0; i<comm->getSize(); ++i) out << std::endl; out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl; out << " 1) Load the data file " << fileName << "." << std::endl; out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl; out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl; out << " 4) Split screen horizontally (Icon, top right)." << std::endl; out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl; out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl; out << std::endl; out << " To display row weights next to each point:" << std::endl; out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl; out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl; out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl; #endif } //Build
TEUCHOS_UNIT_TEST(Zoltan, Build3PDEs) { typedef Teuchos::ScalarTraits<Scalar> ST; out << "version: " << MueLu::Version() << std::endl; out << std::endl; out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl; out << "that has a random number of nonzeros per row and 3 DOFs per mesh point. Good results have been precomputed" << std::endl; out << "for up to 5 processors. The results are the number of nonzeros in the local matrix" << std::endl; out << "once the Zoltan repartitioning has been applied." << std::endl; out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl; RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm(); if (comm->getSize() > 5) { out << std::endl; out << "This test must be run on 1 to 5 processes." << std::endl; TEST_EQUALITY(true, true); return; } Level level; RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager()); level.SetFactoryManager(factoryHandler); int nx=9; int ny=nx; int dofsPerNode = 3; GO numGlobalElements = nx*ny*dofsPerNode; size_t maxEntriesPerRow=30; RCP<const Map> map; int numMyNodes = numGlobalElements / dofsPerNode; if (comm->getSize() > 1) { // In parallel, make sure that the dof's associated with a node all // reside on the same processor. int numNodes = numGlobalElements / dofsPerNode; TEUCHOS_TEST_FOR_EXCEPTION( (numGlobalElements - numNodes * dofsPerNode) != 0, MueLu::Exceptions::RuntimeError, "Number of matrix rows is not divisible by #dofs" ); int nproc = comm->getSize(); if (comm->getRank() < nproc-1) numMyNodes = numNodes / nproc; else numMyNodes = numNodes - (numNodes/nproc) * (nproc-1); map = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, numMyNodes*dofsPerNode, comm); } else { map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm); } const size_t numMyElements = map->getNodeNumElements(); Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList(); RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more // memory on the fly. While not super efficient, this // ensures that no zeros are being stored. Thus, from // Zoltan's perspective the matrix is imbalanced. // Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row. // Create a vector with random integer entries in [1,maxEntriesPerRow]. ST::seedrandom(666*comm->getRank()); RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false); Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0); for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) { *i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1)); } RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout)); fos->setOutputToRootOnly(-1); Teuchos::Array<Scalar> vals(maxEntriesPerRow); Teuchos::Array<GO> cols(maxEntriesPerRow); for (size_t i = 0; i < numMyElements; ++i) { Teuchos::ArrayView<SC> av(&vals[0],eprData[i]); Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]); //stick in ones for values for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one(); //figure out valid column indices GO start = std::max(myGlobalElements[i]-eprData[i]+1,0); for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j; A->insertGlobalValues(myGlobalElements[i], iv, av); } A->fillComplete(); // Now treat the matrix as if it has 3 DOFs per node. A->SetFixedBlockSize(dofsPerNode); level.Set("A",A); //build coordinates Teuchos::ParameterList list; list.set("nx",nx); list.set("ny",ny); RCP<const Map> coalescedMap = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements/dofsPerNode, numMyNodes, comm); RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",coalescedMap,list); // XYZ are the "coalesce" coordinates as it has been generated for 1 DOF/node and we are using them for 3 DOFS/node // level.Set("Coordinates",XYZ); "Coordinates" == uncoalesce. "X,Y,ZCoordinates" == coalesce { RCP<MultiVector> coordinates = XYZ; // making a copy because I don't want to keep 'open' the Xpetra_MultiVector if (coordinates->getNumVectors() >= 1) { Teuchos::ArrayRCP<const SC> coord = coordinates->getData(0); Teuchos::ArrayRCP<SC> coordCpy(coord.size()); for(int i=0; i<coord.size(); i++) { coordCpy[i] = coord[i]; } level.Set("XCoordinates", coordCpy); //std::cout << coordCpy << std::endl; } if (coordinates->getNumVectors() >= 2) { Teuchos::ArrayRCP<const SC> coord = coordinates->getData(1); Teuchos::ArrayRCP<SC> coordCpy(coord.size()); for(int i=0; i<coord.size(); i++) { coordCpy[i] = coord[i]; } level.Set("YCoordinates", coordCpy); } /*if (coordinates->getNumVectors() >= 3) { Teuchos::ArrayRCP<const SC> coord = coordinates->getData(2); Teuchos::ArrayRCP<SC> coordCpy(coord.size()); for(int i=0; i<coord.size(); i++) { coordCpy[i] = coord[i]; } level.Set("ZCoordinates", coordCpy); }*/ } //coalescedMap->describe(*fos,Teuchos::VERB_EXTREME); //sleep(1); comm->barrier(); //XYZ->describe(*fos,Teuchos::VERB_EXTREME); LO numPartitions = comm->getSize(); level.Set("number of partitions",numPartitions); RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface()); //zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints level.Request("Partition",zoltan.get()); zoltan->Build(level); RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get()); /* //temporary code to have the trivial decomposition (no change) ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0); for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i) *i = comm->getRank(); decompEntries=Teuchos::null; */ //Create vector whose local length is the global number of partitions. //This vector will record the local number of nonzeros associated with each partition. Teuchos::Array<GO> parts(numPartitions); for (int i=0; i<numPartitions; ++i) parts[i] = i; Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions); RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView, map->getIndexBase(),comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap); RCP<Xpetra::Vector<LO,LO,GO,NO> > nnzPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(A->getRowMap()); Teuchos::ArrayRCP<GO> nnzData = nnzPerRow->getDataNonConst(0); //For the local rows in each partition, tally up the number of nonzeros. This is what //Zoltan should be load-balancing. Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0); Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0); for (size_t i=0; i<decomposition->getLocalLength();++i) { Teuchos::ArrayView<const LO> c; Teuchos::ArrayView<const SC> v; A->getLocalRowView(i,c,v); lpvData[decompData[i]] += v.size(); nnzData[i] = v.size(); } lpvData = Teuchos::null; decompData = Teuchos::null; nnzData = Teuchos::null; /* if (comm->getRank() == 0) std::cout << "nnz per row" << std::endl; nnzPerRow->describe(*fos,Teuchos::VERB_EXTREME); if (comm->getRank() == 0) std::cout << "Row-to-partition assignment (from Zoltan)" << std::endl; decomposition->describe(*fos,Teuchos::VERB_EXTREME); if (comm->getRank() == 0) std::cout << "#nonzeros per partition" << std::endl; localPartsVec->describe(*fos,Teuchos::VERB_EXTREME); */ //Send the local nnz tallies to pid 0, which can report the global sums. size_t mysize=1; if (comm->getRank() == 0) mysize = numPartitions; RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(), Teuchos::OrdinalTraits<global_size_t>::invalid(), mysize, map->getIndexBase(), comm); RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap); RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap); globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD); ArrayRCP<GO> expectedResults(numPartitions); switch (comm->getSize()) { case 1: expectedResults[0] = 3951; break; case 2: expectedResults[0] = 1955; expectedResults[1] = 1910; break; case 3: expectedResults[0] = 1326; expectedResults[1] = 1340; expectedResults[2] = 1321; break; case 4: expectedResults[0] = 950; expectedResults[1] = 922; expectedResults[2] = 908; expectedResults[3] = 936; break; case 5: expectedResults[0] = 774; expectedResults[1] = 735; expectedResults[2] = 726; expectedResults[3] = 771; expectedResults[4] = 759; break; default: break; }; ArrayRCP<const LO> gtvData = globalTallyVec->getData(0); #ifdef __linux__ out << "Checking results..." << std::endl; for (int i=0; i<numPartitions; ++i) { if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]); } #endif #ifdef MUELU_VISUALIZE_REPARTITIONING // //Now write everything to a comma-separate list that ParaView can grok // Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0); Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1); Teuchos::ArrayRCP<const GO> D = decomposition->getData(0); RCP<std::ofstream> outFile; std::string fileName = "zoltanResults.csv"; //write header information if (comm->getRank() == 0) { outFile = rcp(new std::ofstream(fileName.c_str())); *outFile << "x coord, y coord, z coord, partition, row weight" << std::endl; } comm->barrier(); //append coordinates nnzData = nnzPerRow->getDataNonConst(0); for (int j=0; j<comm->getSize(); ++j) { int mypid = comm->getRank(); if (mypid == j) { outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app)); int blockSize = A->GetFixedBlockSize(); //Coordinates are for coalesced system, D is for uncoalesced for (int i=0; i < D.size()/blockSize; ++i) { int nnz=0; for (int k=0; k<blockSize; ++k) nnz += nnzData[i*blockSize+k]; *outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i*blockSize] << ", " << nnz << std::endl; } } } //for (int i=0; i<comm->getSize(); ++i) out << std::endl; out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl; out << " 1) Load the data file " << fileName << "." << std::endl; out << " 2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl; out << " 3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl; out << " 4) Split screen horizontally (Icon, top right)." << std::endl; out << " 5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl; out << " 6) Under the Display tab, you can color points by scalar value and resize them." << std::endl; out << std::endl; out << " To display row weights next to each point:" << std::endl; out << " 1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl; out << " 2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl; out << " 3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl; #endif } //Build3PDEs