Пример #1
0
bool Tpetra::Utils::parseRfmt(Teuchos::ArrayRCP<char> fmt, int &perline, int &width, int &prec, char &valformat) {
  TEUCHOS_TEST_FOR_EXCEPT(fmt.size() != 0 && fmt[fmt.size()-1] != '\0');
  std::transform(fmt.begin(), fmt.end(), fmt, static_cast < int(*)(int) > (std::toupper));
  // find the first left paren '(' and the last right paren ')'
  Teuchos::ArrayRCP<char>::iterator firstLeftParen = std::find( fmt.begin(),  fmt.end(), '(');
  Teuchos::ArrayRCP<char>::iterator lastRightParen = std::find(std::reverse_iterator<Teuchos::ArrayRCP<char>::iterator>(fmt.end()), 
                                                               std::reverse_iterator<Teuchos::ArrayRCP<char>::iterator>(fmt.begin()), 
                                                               ')').base()-1;
  // select the substring between the parens, including them
  // if neither was found, set the string to empty
  if (firstLeftParen == fmt.end() || lastRightParen == fmt.begin()) {
    fmt.resize(0 + 1);
    fmt[0] = '\0';
  }
  else {
    fmt += (firstLeftParen - fmt.begin());
    size_t newLen = lastRightParen - firstLeftParen + 1;
    fmt.resize(newLen + 1);
    fmt[newLen] = '\0';
  }
  if (std::find(fmt.begin(),fmt.end(),'P') != fmt.end()) {
    // not supported
    return true;
  }
  bool error = true;
  if (std::sscanf(fmt.getRawPtr(),"(%d%c%d.%d)",&perline,&valformat,&width,&prec) == 4) {
    if (valformat == 'E' || valformat == 'D' || valformat == 'F') {
      error = false;
    }
  } 
  return error;
}
Пример #2
0
void dump(const MV& v, const std::string& name) {
  std::cout << name << std::endl;
  Teuchos::ArrayRCP<const Scalar> view = v.get1dView();
  for (Teuchos::ArrayRCP<const Scalar>::iterator it = view.begin(); it != view.end(); ++it) {
    std::cout << *it << std::endl;
  }
  std::cout << std::endl;
}
Пример #3
0
void Tpetra::Utils::readHBHeader(std::ifstream &fin, Teuchos::ArrayRCP<char> &Title, Teuchos::ArrayRCP<char> &Key, Teuchos::ArrayRCP<char> &Type, 
                           int &Nrow, int &Ncol, int &Nnzero, int &Nrhs,
                           Teuchos::ArrayRCP<char> &Ptrfmt, Teuchos::ArrayRCP<char> &Indfmt, Teuchos::ArrayRCP<char> &Valfmt, Teuchos::ArrayRCP<char> &Rhsfmt, 
                           int &Ptrcrd, int &Indcrd, int &Valcrd, int &Rhscrd, Teuchos::ArrayRCP<char> &Rhstype) {
  int Totcrd, Neltvl, Nrhsix;
  const int MAXLINE = 81;
  char line[MAXLINE];
  //
  Title.resize(72 + 1);  std::fill(Title.begin(),  Title.end(),  '\0');
  Key.resize(8 + 1);     std::fill(Key.begin(),    Key.end(),    '\0');
  Type.resize(3 + 1);    std::fill(Type.begin(),   Type.end(),   '\0');
  Ptrfmt.resize(16 + 1); std::fill(Ptrfmt.begin(), Ptrfmt.end(), '\0');
  Indfmt.resize(16 + 1); std::fill(Indfmt.begin(), Indfmt.end(), '\0');
  Valfmt.resize(20 + 1); std::fill(Valfmt.begin(), Valfmt.end(), '\0');
  Rhsfmt.resize(20 + 1); std::fill(Rhsfmt.begin(), Rhsfmt.end(), '\0');
  //
  const std::string errStr("Tpetra::Utils::readHBHeader(): Improperly formatted H/B file: ");
  /*  First line:   (A72,A8) */
  fin.getline(line,MAXLINE);
  TEUCHOS_TEST_FOR_EXCEPTION( std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
  (void)std::sscanf(line, "%72c%8[^\n]", Title.getRawPtr(), Key.getRawPtr());
  /*  Second line:  (5I14) or (4I14) */
  fin.getline(line,MAXLINE);
  TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
  if ( std::sscanf(line,"%14d%14d%14d%14d%14d",&Totcrd,&Ptrcrd,&Indcrd,&Valcrd,&Rhscrd) != 5 ) {
    Rhscrd = 0;
    TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%14d%14d%14d%14d",&Totcrd,&Ptrcrd,&Indcrd,&Valcrd) != 4, std::runtime_error, errStr << "error reading pointers (line 2)");
  }
  /*  Third line:   (A3, 11X, 4I14) */
  fin.getline(line,MAXLINE);
  TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
  TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line, "%3c%14i%14i%14i%14i", Type.getRawPtr(),&Nrow,&Ncol,&Nnzero,&Neltvl) != 5 , std::runtime_error, errStr << "error reading matrix meta-data (line 3)");
  std::transform(Type.begin(), Type.end(), Type.begin(), static_cast < int(*)(int) > (std::toupper));
  /*  Fourth line:  */
  fin.getline(line,MAXLINE);
  TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
  if (Rhscrd != 0) {
    TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%16c%16c%20c%20c",Ptrfmt.getRawPtr(),Indfmt.getRawPtr(),Valfmt.getRawPtr(),Rhsfmt.getRawPtr()) != 4, std::runtime_error, errStr << "error reading formats (line 4)");
  }
  else {
    TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%16c%16c%20c",Ptrfmt.getRawPtr(),Indfmt.getRawPtr(),Valfmt.getRawPtr()) != 3,                        std::runtime_error, errStr << "error reading formats (line 4)");
  }
  /*  (Optional) Fifth line: */
  if (Rhscrd != 0 ) { 
    Rhstype.resize(3 + 1,'\0');
    fin.getline(line,MAXLINE);
    TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%*s") < 0, std::runtime_error, errStr << "error buffering line.");
    TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(line,"%3c%14d%14d", Rhstype.getRawPtr(), &Nrhs, &Nrhsix) != 3, std::runtime_error, errStr << "error reading right-hand-side meta-data (line 5)");
  }
}
Пример #4
0
//-----------------------------------------------------------------------------
double TpetraVector::sum() const
{
  dolfin_assert(!_x.is_null());

  Teuchos::ArrayRCP<const double> arr = _x->getData(0);
  const double _sum = std::accumulate(arr.begin(), arr.end(), 0.0);

  return MPI::sum(mpi_comm(), _sum);
}
Пример #5
0
bool Tpetra::Utils::parseIfmt(Teuchos::ArrayRCP<char> fmt, int &perline, int &width) {
  TEUCHOS_TEST_FOR_EXCEPT(fmt.size() != 0 && fmt[fmt.size()-1] != '\0');
  // parses integers n and d out of (nId)
  bool error = true;
  std::transform(fmt.begin(), fmt.end(), fmt, static_cast < int(*)(int) > (std::toupper));
  if (std::sscanf(fmt.getRawPtr(),"(%dI%d)",&perline,&width) == 2) {
    error = false;
  }
  return error;
}
void 
Filtered_UniqueGlobalIndexer<LocalOrdinalT,GlobalOrdinalT>::
getOwnedAndGhostedNotFilteredIndicator(std::vector<int> & indicator) const
{
  using Teuchos::RCP;

  typedef GlobalOrdinalT GO;
  typedef LocalOrdinalT LO;
  typedef panzer::TpetraNodeType Node;
  typedef Tpetra::Map<LO, GO, Node> Map;
  typedef Tpetra::Vector<GO,LO,GO,Node> Vector;
  typedef Tpetra::Import<LO,GO,Node> Import;

  std::vector<GlobalOrdinalT> ownedIndices;
  std::vector<GlobalOrdinalT> ghostedIndices;

  // build owned and ghosted maps
  getOwnedIndices(ownedIndices);
  getOwnedAndGhostedIndices(ghostedIndices);

  RCP<const Map> ownedMap 
      = Tpetra::createNonContigMap<LO,GO>(ownedIndices,getComm());
  RCP<const Map> ghostedMap 
      = Tpetra::createNonContigMap<LO,GO>(ghostedIndices,getComm());

  // allocate the owned vector, mark those GIDs as unfiltered
  // (they are by definition)
  Vector ownedActive(ownedMap);
  ownedActive.putScalar(1);

  // Initialize all indices to zero
  Vector ghostedActive(ghostedMap);
  ghostedActive.putScalar(0);

  // do communication, marking unfiltered indices as 1 (filtered
  // indices locally are marked as zero)
  Import importer(ownedMap,ghostedMap);
  ghostedActive.doImport(ownedActive,importer,Tpetra::INSERT);

  Teuchos::ArrayRCP<const GO> data = ghostedActive.getData();

  // copy communicated data (clear it out first)
  indicator.clear();
  indicator.insert(indicator.end(),data.begin(),data.end()); 
}
Пример #7
0
//---------------------------------------------------------------------------//
TEUCHOS_UNIT_TEST( FieldTools, tensor_field_test )
{
    using namespace DataTransferKit;

    // Setup random numbers for the test.
    int num_rand = 4;
    double rand_max = 10.0;
    Teuchos::Array<double> random_numbers( num_rand );
    std::srand( 1 );
    for ( int i = 0; i < num_rand; ++i )
    {
	random_numbers[i] = rand_max * (double) std::rand() / RAND_MAX;
    }

    // Setup communication.
    Teuchos::RCP< const Teuchos::Comm<int> > comm = getDefaultComm<int>();
    int my_rank = comm->getRank();
    int my_size = comm->getSize();

    // Setup a field manager.
    int field_dim = 9;
    typedef FieldTraits<ArrayField> FT;
    Teuchos::RCP<ArrayField> array_field = 
	Teuchos::rcp( new ArrayField( field_dim*(my_rank+1), field_dim ) );
    FieldManager<ArrayField> field_manager( array_field, comm );

    // Test the field tools.
    typedef FieldTools<ArrayField> Tools;

    // Dimension iterators.
    FT::iterator dim_begin, dim_end;
    for ( int d = 0; d < field_dim; ++d )
    {
	dim_begin = Tools::dimBegin( *field_manager.field(), d );
	dim_end = Tools::dimEnd( *field_manager.field(), d );
	TEST_ASSERT( std::distance( dim_begin, dim_end ) == my_rank+1 );
    }

    FT::const_iterator const_dim_begin, const_dim_end;
    for ( int d = 0; d < field_dim; ++d )
    {
	const_dim_begin = Tools::dimBegin( *field_manager.field(), d );
	const_dim_end = Tools::dimEnd( *field_manager.field(), d );
	TEST_ASSERT( std::distance( const_dim_begin, const_dim_end ) 
		     == my_rank+1 );
    }
  
    // Length.
    int global_size = 0;
    for ( int i = 0; i < my_size; ++i )
    {
	global_size += field_dim*(i+1);
    }
    TEST_ASSERT( 
	Tools::globalSize( *field_manager.field(), field_manager.comm() )
	== global_size );
  
    // Scalar filling.
    Tools::putScalar( *field_manager.field(), random_numbers[0] );

    // Views.
    double local_val;
    Teuchos::ArrayRCP<const double> const_view = 
	Tools::view( *field_manager.field() );
    Teuchos::ArrayRCP<const double>::const_iterator const_view_iterator;
    for ( const_view_iterator = const_view.begin();
	  const_view_iterator != const_view.end();
	  ++const_view_iterator )
    {
	local_val = random_numbers[0];
	TEST_ASSERT( *const_view_iterator == local_val );
    }

    Teuchos::ArrayRCP<double> view = 
	Tools::nonConstView( *field_manager.field() );
    Teuchos::ArrayRCP<double>::iterator view_iterator;
    for ( view_iterator = view.begin();
	  view_iterator != view.end();
	  ++view_iterator )
    {
	local_val = random_numbers[0];
	TEST_ASSERT( *view_iterator == local_val );
    }
    
    Teuchos::ArrayRCP<double> copy = 
	Tools::copy( *field_manager.field() );
    Teuchos::ArrayRCP<double>::const_iterator copy_iterator;
    for ( copy_iterator = copy.begin();
	  copy_iterator != copy.end();
	  ++copy_iterator )
    {
	local_val = random_numbers[0];
	TEST_ASSERT( *copy_iterator == local_val );
    }

    // Multiple scalar filling.
    Teuchos::Array<double> fillers( field_dim );
    for ( int d = 0; d < field_dim; ++d )
    {
	fillers[d] = (d+1)*random_numbers[1];
    }
    Tools::putScalar( *field_manager.field(), fillers() );

    // Dim iterators.
    FT::const_iterator const_dim_iterator;
    for ( int d = 0; d < field_dim; ++d )
    {
	for ( const_dim_iterator = Tools::dimBegin( *field_manager.field(), d );
	      const_dim_iterator != Tools::dimEnd( *field_manager.field(), d );
	      ++const_dim_iterator )
	{
	    local_val = (d+1) * random_numbers[1];
	    TEST_ASSERT( *const_dim_iterator == local_val );
	}
    }

    FT::iterator dim_iterator;
    for ( int d = 0; d < field_dim; ++d )
    {
	for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d );
	      dim_iterator != Tools::dimEnd( *field_manager.field(), d );
	      ++dim_iterator )
	{
	    local_val = (d+1) * random_numbers[1];
	    TEST_ASSERT( *dim_iterator == local_val );
	}
    }

    // Scaling.
    Tools::scale( *field_manager.field(), random_numbers[2] );
    for ( int d = 0; d < field_dim; ++d )
    {
	for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d );
	      dim_iterator != Tools::dimEnd( *field_manager.field(), d );
	      ++dim_iterator )
	{
	    local_val = (d+1) * random_numbers[1] * random_numbers[2];
	    TEST_ASSERT( *dim_iterator == local_val );
	}
    }

    // Multiple scaling.
    Teuchos::Array<double> multipliers( field_dim );
    for ( int d = 0; d < field_dim; ++d )
    {
	multipliers[d] = -(d+1) - random_numbers[3];
    }
    Tools::scale( *field_manager.field(), multipliers() );
    for ( int d = 0; d < field_dim; ++d )
    {
	for ( dim_iterator = Tools::dimBegin( *field_manager.field(), d );
	      dim_iterator != Tools::dimEnd( *field_manager.field(), d );
	      ++dim_iterator )
	{
	    local_val = (d+1) * random_numbers[1] 
			* random_numbers[2] * (-(d+1) - random_numbers[3]);
	    TEST_ASSERT( *dim_iterator == local_val );
	}
    }

    // Norms.
    Teuchos::Array<double> inf_norms;
    Tools::normInf( *field_manager.field(), field_manager.comm(), inf_norms );
    for ( int d = 0; d < field_dim; ++d )
    {
	local_val = (d+1) * random_numbers[1] * random_numbers[2] 
		    * (-(d+1) - random_numbers[3]);
	TEST_ASSERT( softEquivalence( inf_norms[d], std::abs( local_val ) ) );
    }

    Teuchos::Array<double> one_norms;
    Tools::norm1( *field_manager.field(), field_manager.comm(), one_norms );
    for ( int d = 0; d < field_dim; ++d )
    {
	local_val = ( global_size / FT::dim( *field_manager.field() ) ) * 
		   (d+1) * random_numbers[1] 
		   * random_numbers[2] * (-(d+1) - random_numbers[3]);
	TEST_ASSERT( softEquivalence( one_norms[d], std::abs( local_val ) ) );
    }

    Teuchos::Array<double> two_norms;
    Tools::norm2( *field_manager.field(), field_manager.comm(), two_norms );
    for ( int d = 0; d < field_dim; ++d )
    {
	local_val = ( global_size / FT::dim( *field_manager.field() ) ) * 
		   std::pow( 
		       std::abs( (d+1) * random_numbers[1] * random_numbers[2] 
				 * (-(d+1) - random_numbers[3]) ), 2.0 );
	local_val = std::pow( local_val, 1.0/2.0 );
	TEST_ASSERT( softEquivalence( two_norms[d], local_val ) );
    }

    Teuchos::Array<double> q_norms;
    int q = std::floor( (10 * std::rand()) / RAND_MAX ) + 1;
    Tools::normQ( *field_manager.field(), field_manager.comm(), q, q_norms );
    for ( int d = 0; d < field_dim; ++d )
    {
	local_val = ( global_size / FT::dim( *field_manager.field() ) ) * 
		    std::pow( 
			std::abs( (d+1) * random_numbers[1] * random_numbers[2] 
				  * (-(d+1) - random_numbers[3]) ), q );
	local_val = std::pow( local_val, 1.0/q );
	TEST_ASSERT( softEquivalence( q_norms[d], local_val ) );
    }

    // Average.
    Teuchos::Array<double> averages;
    Tools::average( *field_manager.field(), field_manager.comm(), averages );
    for ( int d = 0; d < field_dim; ++d )
    {
	local_val = (d+1) * random_numbers[1] * random_numbers[2] 
		    * (-(d+1) - random_numbers[3]);
	TEST_ASSERT( softEquivalence( averages[d], local_val ) );
    }
}
Пример #8
0
void Tpetra::Utils::readHBMatDouble(const std::string &filename, int &numRows, int &numCols, int &numNZ, std::string &type, Teuchos::ArrayRCP<int> &colPtrs, Teuchos::ArrayRCP<int> &rowInds, Teuchos::ArrayRCP<double> &vals) {
  // NOTE: if complex, allocate enough space for 2*NNZ and interleave real and imaginary parts (real,imag)
  //       if pattern, don't touch parameter vals; do not allocate space space for values
  try {
    std::ifstream fin;
    int ptrCrd, indCrd, valCrd;
    Teuchos::ArrayRCP<char> Title, Key, Ptrfmt, Indfmt, Valfmt;
    const int MAXSIZE = 81;
    char lineBuf[MAXSIZE];
    // nitty gritty
    int ptrsPerLine, ptrWidth, indsPerLine, indWidth, valsPerLine, valWidth, valPrec;
    char valFlag;
    // 
    fin.open(filename.c_str(),std::ifstream::in);
    {
      // we don't care about RHS-related stuff, so declare those vars in an expiring scope
      int Nrhs, rhsCrd;
      Teuchos::ArrayRCP<char> Rhstype, Rhsfmt;
      Teuchos::ArrayRCP<char> TypeArray;
      Tpetra::Utils::readHBHeader(fin, Title, Key, TypeArray, numRows, numCols, numNZ, Nrhs,
                                  Ptrfmt, Indfmt, Valfmt, Rhsfmt, 
                                  ptrCrd, indCrd, valCrd, rhsCrd, Rhstype);
      if (TypeArray.size() > 0) {
        type.resize(TypeArray.size()-1);
        std::copy(TypeArray.begin(), TypeArray.end(), type.begin());
      }
    }
    const std::string errStr("Tpetra::Utils::readHBHeader(): Improperly formatted H/B file.");
    const bool readPatternOnly = (type[0] == 'P' || type[0] == 'p');
    const bool readComplex     = (type[0] == 'C' || type[0] == 'c');
    /*  Parse the array input formats from Line 3 of HB file  */
    TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseIfmt(Ptrfmt,ptrsPerLine,ptrWidth) == true, std::runtime_error,
        "Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format.");
    TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseIfmt(Indfmt,indsPerLine,indWidth) == true, std::runtime_error,
        "Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format.");
    if (readPatternOnly == false) {
      TEUCHOS_TEST_FOR_EXCEPTION( Tpetra::Utils::parseRfmt(Valfmt,valsPerLine,valWidth,valPrec,valFlag) == true, std::runtime_error,
          "Tpetra::Utils::readHBMatDouble(): error parsing. Invalid/unsupported file format.");
    }
    // special case this: the reason is that the number of colPtrs read is numCols+1, which is non-zero even if numCols == 0
    // furthermore, if numCols == 0, there is nothing of interest to read
    if (numCols == 0) return;
    // allocate space for column pointers, row indices, and values
    // if the file is empty, do not touch these ARCPs
    colPtrs = Teuchos::arcp<int>(numCols+1);
    if (numNZ > 0) {
      rowInds = Teuchos::arcp<int>(numNZ);  
      if (readPatternOnly == false) {
        if (readComplex) {
          vals = Teuchos::arcp<double>(2*numNZ); 
        }
        else {
          vals = Teuchos::arcp<double>(numNZ); 
        }
      }
    }
    /* Read column pointer array:
       Specifically, read ptrCrd number of lines, and on each line, read ptrsPerLine number of integers, each of width ptrWidth
       Store these in colPtrs */
    {
      int colPtrsRead = 0;
      char NullSub = '\0';
      for (int lno=0; lno < ptrCrd; ++lno) {
        fin.getline(lineBuf, MAXSIZE);
        TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr);
        char *linePtr = lineBuf;
        for (int ptr=0; ptr < ptrsPerLine; ++ptr) {
          if (colPtrsRead == numCols + 1) break;
          int cptr;
          // terminate the string at the end of the current ptr block, saving the character in that location
          std::swap(NullSub,linePtr[ptrWidth]);
          // read the ptr
          std::sscanf(linePtr, "%d", &cptr);
          // put the saved character back, and put the '\0' back into NullSub for use again
          std::swap(NullSub,linePtr[ptrWidth]);
          linePtr += ptrWidth;
          colPtrs[colPtrsRead++] = cptr;
        }
      }
      TEUCHOS_TEST_FOR_EXCEPT(colPtrsRead != numCols + 1);
    }
    /* Read row index array:
       Specifically, read indCrd number of lines, and on each line, read indsPerLine number of integers, each of width indWidth
       Store these in rowInds */
    {
      char NullSub = '\0';
      int indicesRead = 0;
      for (int lno=0; lno < indCrd; ++lno) {
        fin.getline(lineBuf, MAXSIZE);
        TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr);
        char *linePtr = lineBuf;
        for (int indcntr=0; indcntr < indsPerLine; ++indcntr) {
          if (indicesRead == numNZ) break;
          int ind;
          // terminate the string at the end of the current ind block, saving the character in that location
          std::swap(NullSub,linePtr[indWidth]);
          // read the ind
          std::sscanf(linePtr, "%d", &ind);
          // put the saved character back, and put the '\0' back into NullSub for use again
          std::swap(NullSub,linePtr[indWidth]);
          linePtr += indWidth;
          rowInds[indicesRead++] = ind;
        }
      }
      TEUCHOS_TEST_FOR_EXCEPT(indicesRead != numNZ);
    }
    /* Read array of values:
       Specifically, read valCrd number of lines, and on each line, read valsPerLine number of real values, each of width/precision valWidth/valPrec
       Store these in vals
       If readComplex, then read twice as many non-zeros, and interleave real,imag into vals */
    if (readPatternOnly == false) {
      int totalNumVals;
      if (readComplex) {
        totalNumVals = 2*numNZ;
      }
      else {
        totalNumVals = numNZ;
      }
      char NullSub = '\0';
      int valsRead = 0;
      for (int lno=0; lno < valCrd; ++lno) {
        fin.getline(lineBuf, MAXSIZE);
        TEUCHOS_TEST_FOR_EXCEPTION(std::sscanf(lineBuf,"%*s") < 0, std::runtime_error, errStr);
        // if valFlag == 'D', then we need to convert [dD] in fp vals into [eE] that scanf can parse
        if (valFlag == 'D') std::replace_if(lineBuf, lineBuf+MAXSIZE, std::bind2nd(std::equal_to<char>(), 'D'), 'E'); 
        char *linePtr = lineBuf;
        for (int valcntr=0; valcntr < valsPerLine; ++valcntr) {
          if (valsRead == totalNumVals) break;
          double val;
          // terminate the string at the end of the current val block, saving the character in that location
          std::swap(NullSub,linePtr[valWidth]);
          // read the val
          std::sscanf(linePtr, "%le", &val);
          // put the saved character back, and put the '\0' back into NullSub for use again
          std::swap(NullSub,linePtr[valWidth]);
          linePtr += valWidth;
          vals[valsRead++] = val;
        }
      }
      TEUCHOS_TEST_FOR_EXCEPT(valsRead != totalNumVals);
    }
    fin.close();
  }
  catch (std::exception &e) {
    TEUCHOS_TEST_FOR_EXCEPTION(true, std::runtime_error, 
        "Tpetra::Utils::readHBInfo() of filename \"" << filename << "\" caught exception: " << std::endl
        << e.what() << std::endl);
  }
}
Пример #9
0
void
Tpetra::Utils::readHBMatrix(const std::string &filename,
                            const Teuchos::RCP<const Teuchos::Comm<int> > &comm,
                            const Teuchos::RCP<Node> &node,
                            Teuchos::RCP< Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node,LocalMatOps> > &A,
                            Teuchos::RCP< const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > rowMap,
                            const Teuchos::RCP<ParameterList> &params)
{
    const int myRank = comm->getRank();
    int numRows,numCols,numNZ;
    Teuchos::ArrayRCP<Scalar> svals;
    Teuchos::ArrayRCP<GlobalOrdinal> colinds;
    Teuchos::ArrayRCP<int>           rowptrs;
    Teuchos::ArrayRCP<size_t>        nnzPerRow;
    int fail = 0;
    if (myRank == 0) {
        bool isSymmetric=false;
        Teuchos::ArrayRCP<double> dvals;
        Teuchos::ArrayRCP<int> colptrs, rowinds;
        std::string type;
        Tpetra::Utils::readHBMatDouble(filename,numRows,numCols,numNZ,type,colptrs,rowinds,dvals);
        TEUCHOS_TEST_FOR_EXCEPT(type.size() != 3);
        if (type[0] != 'R' && type[0] != 'r') {
            // only real matrices right now
            fail = 1;
        }
        if (fail == 0 && numNZ > 0) {
            if (type[1] == 'S' || type[1] == 's') {
                isSymmetric = true;
            }
            else {
                isSymmetric = false;
            }
        }
        if (fail == 0 && numNZ > 0) {
            // find num non-zero per row
            nnzPerRow = Teuchos::arcp<size_t>(numRows);
            std::fill(nnzPerRow.begin(), nnzPerRow.end(), 0);
            for (Teuchos::ArrayRCP<int>::const_iterator ri=rowinds.begin(); ri != rowinds.end(); ++ri) {
                // count each row index towards its row
                ++nnzPerRow[*ri-1];
            }
            if (isSymmetric) {
                // count each column toward the corresponding row as well
                for (int c=0; c < numCols; ++c) {
                    // the diagonal was already counted; neglect it, if it exists
                    for (int i=colptrs[c]-1; i != colptrs[c+1]-1; ++i) {
                        if (rowinds[i] != c+1) {
                            ++nnzPerRow[c];
                            ++numNZ;
                        }
                    }
                }
            }
            // allocate/set new matrix data
            svals = Teuchos::arcp<Scalar>(numNZ);
            colinds = Teuchos::arcp<GlobalOrdinal>(numNZ);
            rowptrs = Teuchos::arcp<int>(numRows+1);
            rowptrs[0] = 0;
#ifdef HAVE_TPETRA_DEBUG
            Teuchos::ArrayRCP<size_t> nnzPerRow_debug(nnzPerRow.size());
            std::copy(nnzPerRow.begin(), nnzPerRow.end(), nnzPerRow_debug.begin());
#endif
            for (int j=1; j <= numRows; ++j) {
                rowptrs[j] = rowptrs[j-1] + nnzPerRow[j-1];
                nnzPerRow[j-1] = 0;
            }
            // translate from column-oriented to row-oriented
            for (int col=0; col<numCols; ++col) {
                for (int i=colptrs[col]-1; i != colptrs[col+1]-1; ++i) {
                    const int row = rowinds[i]-1;
                    // add entry to (row,col), with value dvals[i]
                    const size_t entry = rowptrs[row] + nnzPerRow[row];
                    svals[entry] = Teuchos::as<Scalar>(dvals[i]);
                    colinds[entry] = Teuchos::as<GlobalOrdinal>(col);
                    ++nnzPerRow[row];
                    if (isSymmetric && row != col) {
                        // add entry to (col,row), with value dvals[i]
                        const size_t symentry = rowptrs[col] + nnzPerRow[col];
                        svals[symentry] = Teuchos::as<Scalar>(dvals[i]);
                        colinds[symentry] = Teuchos::as<GlobalOrdinal>(row);
                        ++nnzPerRow[col];
                    }
                }
            }
#ifdef HAVE_TPETRA_DEBUG
            {
                bool isequal = true;
                typename Teuchos::ArrayRCP<size_t>::const_iterator it1, it2;
                for (it1 = nnzPerRow.begin(), it2 = nnzPerRow_debug.begin(); it1 != nnzPerRow.end(); ++it1, ++it2) {
                    if (*it1 != *it2) {
                        isequal = false;
                        break;
                    }
                }
                TEUCHOS_TEST_FOR_EXCEPTION(!isequal || nnzPerRow.size() != nnzPerRow_debug.size(), std::logic_error,
                                           "Tpetra::Utils::readHBMatrix(): Logic error.");
            }
#endif
        }
        // std::cout << "Matrix " << filename << " of type " << type << ": " << numRows << " by " << numCols << ", " << numNZ << " nonzeros" << std::endl;
    }
    // check for read errors
    broadcast(*comm,0,&fail);
    TEUCHOS_TEST_FOR_EXCEPTION(fail == 1, std::runtime_error, "Tpetra::Utils::readHBMatrix() can only read Real matrices.");
    // distribute global matrix info
    broadcast(*comm,0,&numRows);
    broadcast(*comm,0,&numCols);
    // create map with uniform partitioning
    if (rowMap == Teuchos::null) {
        rowMap = Teuchos::rcp(new Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node>((global_size_t)numRows,(GlobalOrdinal)0,comm,GloballyDistributed,node));
    }
    else {
        TEUCHOS_TEST_FOR_EXCEPTION( rowMap->getGlobalNumElements() != (global_size_t)numRows, std::runtime_error,
                                    "Tpetra::Utils::readHBMatrix(): specified map has incorrect number of elements.");
        TEUCHOS_TEST_FOR_EXCEPTION( rowMap->isDistributed() == false && comm->getSize() > 1, std::runtime_error,
                                    "Tpetra::Utils::readHBMatrix(): specified map is not distributed.");
    }
    Teuchos::ArrayRCP<size_t> myNNZ;
    if (rowMap->getNodeNumElements()) {
        myNNZ = Teuchos::arcp<size_t>(rowMap->getNodeNumElements());
    }
    if (myRank == 0) {
        LocalOrdinal numRowsAlreadyDistributed = rowMap->getNodeNumElements();
        std::copy(nnzPerRow.begin(), nnzPerRow.begin()+numRowsAlreadyDistributed,myNNZ);
        for (int p=1; p < Teuchos::size(*comm); ++p) {
            size_t numRowsForP;
            Teuchos::receive(*comm,p,&numRowsForP);
            if (numRowsForP) {
                Teuchos::send<int,size_t>(*comm,numRowsForP,nnzPerRow(numRowsAlreadyDistributed,numRowsForP).getRawPtr(),p);
                numRowsAlreadyDistributed += numRowsForP;
            }
        }
    }
    else {
        const size_t numMyRows = rowMap->getNodeNumElements();
        Teuchos::send(*comm,numMyRows,0);
        if (numMyRows) {
            Teuchos::receive<int,size_t>(*comm,0,numMyRows,myNNZ(0,numMyRows).getRawPtr());
        }
    }
    nnzPerRow = Teuchos::null;
    // create column map
    Teuchos::RCP<const Tpetra::Map<LocalOrdinal,GlobalOrdinal,Node> > domMap;
    if (numRows == numCols) {
        domMap = rowMap;
    }
    else {
        domMap = createUniformContigMapWithNode<LocalOrdinal,GlobalOrdinal,Node>(numCols,comm,node);
    }
    A = rcp(new Tpetra::CrsMatrix<Scalar,LocalOrdinal,GlobalOrdinal,Node>(rowMap,myNNZ,Tpetra::StaticProfile));
    // free this locally, A will keep it allocated as long as it is needed by A (up until allocation of nonzeros)
    myNNZ = Teuchos::null;
    if (myRank == 0 && numNZ > 0) {
        for (int r=0; r < numRows; ++r) {
            const LocalOrdinal nnz = rowptrs[r+1] - rowptrs[r];
            if (nnz > 0) {
                Teuchos::ArrayView<const GlobalOrdinal> inds = colinds(rowptrs[r],nnz);
                Teuchos::ArrayView<const        Scalar> vals = svals(  rowptrs[r],nnz);
                A->insertGlobalValues(r, inds, vals);
            }
        }
    }
    // don't need these anymore
    colinds = Teuchos::null;
    svals   = Teuchos::null;
    rowptrs = Teuchos::null;
    A->fillComplete(domMap,rowMap,params);
}
Пример #10
0
  TEUCHOS_UNIT_TEST(Zoltan, Build)
  {
    typedef Teuchos::ScalarTraits<Scalar> ST;

    out << "version: " << MueLu::Version() << std::endl;
    out << std::endl;
    out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
    out << "that has a random number of nonzeros per row.  Good results have been precomputed" << std::endl;
    out << "for up to 5 processors.  The results are the number of nonzeros in the local matrix" << std::endl;
    out << "once the Zoltan repartitioning has been applied." << std::endl;
    out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;

    RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();

    if (comm->getSize() > 5) {
      out << std::endl;
      out << "This test must be run on 1 to 5 processes." << std::endl;
      TEST_EQUALITY(true, true);
      return;
    }

    Level level;
    RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
    level.SetFactoryManager(factoryHandler);
    int nx=7;
    int ny=nx;
    GO numGlobalElements = nx*ny;
    size_t maxEntriesPerRow=30;

    // Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
    RCP<const Map> map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
    const size_t numMyElements = map->getNodeNumElements();
    Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
    RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
                                                    // memory on the fly.  While not super efficient, this
                                                    // ensures that no zeros are being stored.  Thus, from
                                                    // Zoltan's perspective the matrix is imbalanced.
    // Create a vector with random integer entries in [1,maxEntriesPerRow].
    ST::seedrandom(666*comm->getRank());
    RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
    Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
    for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
      *i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
    }

    RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
    fos->setOutputToRootOnly(-1);

    Teuchos::Array<Scalar> vals(maxEntriesPerRow);
    Teuchos::Array<GO> cols(maxEntriesPerRow);
    for (size_t i = 0; i < numMyElements; ++i) {
      Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
      Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
      //stick in ones for values
      for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
      //figure out valid column indices
      GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
      for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
      A->insertGlobalValues(myGlobalElements[i], iv, av);
    }

    A->fillComplete();
    level.Set("A",A);

    //build coordinates
    RCP<const Map> rowMap = A->getRowMap();
    Teuchos::ParameterList list;
    list.set("nx",nx);
    list.set("ny",ny);
    RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",rowMap,list);
    level.Set("Coordinates",XYZ);

    LO numPartitions = comm->getSize();
    level.Set("number of partitions",numPartitions);
    RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
    //zoltan->SetNumberOfPartitions(numPartitions);
    //zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
    level.Request("Partition",zoltan.get());
    zoltan->Build(level);

    RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
    /* //TODO temporary code to have the trivial decomposition (no change)
    ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
    for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
      *i = comm->getRank();
    decompEntries=Teuchos::null;
    */ //TODO end of temporary code

    //Create vector whose local length is the global number of partitions.
    //This vector will record the local number of nonzeros associated with each partition.
    Teuchos::Array<GO> parts(numPartitions);
    for (int i=0; i<numPartitions; ++i) parts[i] = i;
    Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
    RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
                                                    Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
                                                    map->getIndexBase(),comm);
    RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);

    //For the local rows in each partition, tally up the number of nonzeros.  This is what
    //Zoltan should be load-balancing.
    Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
    Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
    for (size_t i=0; i<decomposition->getLocalLength();++i) {
      Teuchos::ArrayView<const LO> c;
      Teuchos::ArrayView<const SC> v;
      A->getLocalRowView(i,c,v);
      lpvData[decompData[i]] += v.size();
    }

    lpvData = Teuchos::null;
    decompData = Teuchos::null;

    //localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);

    //Send the local nnz tallies to pid 0, which can report the global sums.
    size_t mysize=1;
    if (comm->getRank() == 0) mysize = numPartitions;
    RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
                                                Teuchos::OrdinalTraits<global_size_t>::invalid(),
                                                mysize,
                                                map->getIndexBase(),
                                                comm);
    RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
    RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
    globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);

    ArrayRCP<GO> expectedResults(numPartitions);
    switch (comm->getSize()) {
       case 1:
         expectedResults[0] = 807;
         break;

       case 2:
         expectedResults[0] = 364;
         expectedResults[1] = 363;
         break;

       case 3:
         expectedResults[0] = 277;
         expectedResults[1] = 261;
         expectedResults[2] = 269;
         break;

       case 4:
         expectedResults[0] = 195;
         expectedResults[1] = 186;
         expectedResults[2] = 177;
         expectedResults[3] = 168;
         break;

       case 5:
         expectedResults[0] = 161;
         expectedResults[1] = 145;
         expectedResults[2] = 148;
         expectedResults[3] = 159;
         expectedResults[4] = 157;
         break;

       default:
         break;
    };

    //FIXME cool ... this next line causes a hang if locally the globalyTallyVec has no data.
    //FIXME I get around this by making mysize (above) 1 instead of 0. Is this a bug or feature
    //FIXME in getData?
    ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);

#ifdef __linux__
    out << "Checking results..." << std::endl;
    for (int i=0; i<numPartitions; ++i) {
      if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
    }
#endif

#ifdef MUELU_VISUALIZE_REPARTITIONING
    //
    //Now write everything to a comma-separate list that ParaView can grok
    //
    Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
    Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
    Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
    RCP<std::ofstream> outFile;
    std::string fileName = "zoltanResults.csv";

    //write header information
    if (comm->getRank() == 0) {
      outFile = rcp(new std::ofstream(fileName.c_str()));
      *outFile << "x coord, y coord, z coord, scalar" << std::endl;
    }
    comm->barrier();

    //append coordinates
    for (int j=0; j<comm->getSize(); ++j) {
      int mypid = comm->getRank();
      if (mypid == j) {
        outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
        for (int i=0; i < D.size(); ++i) {
          *outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", " << D[i] << std::endl;
        }
      }
    } //for (int i=0; i<comm->getSize(); ++i)

    out << std::endl;
    out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
    out << "   1) Load the data file " << fileName << "." << std::endl;
    out << "   2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
    out << "   3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
    out << "   4) Split screen horizontally (Icon, top right)." << std::endl;
    out << "   5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
    out << "   6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
    out << std::endl;
    out << " To display row weights next to each point:" << std::endl;
    out << "   1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
    out << "   2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
    out << "   3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif

  } //Build
Пример #11
0
  TEUCHOS_UNIT_TEST(Zoltan, Build3PDEs)
  {

    typedef Teuchos::ScalarTraits<Scalar> ST;

    out << "version: " << MueLu::Version() << std::endl;
    out << std::endl;
    out << "This tests that the partitioning produced by Zoltan is \"reasonable\" for a matrix" << std::endl;
    out << "that has a random number of nonzeros per row and 3 DOFs per mesh point.  Good results have been precomputed" << std::endl;
    out << "for up to 5 processors.  The results are the number of nonzeros in the local matrix" << std::endl;
    out << "once the Zoltan repartitioning has been applied." << std::endl;
    out << "The results can be viewed in Paraview by enabling code guarded by the macro MUELU_VISUALIZE_REPARTITIONING" << std::endl;

    RCP<const Teuchos::Comm<int> > comm = TestHelpers::Parameters::getDefaultComm();

    if (comm->getSize() > 5) {
      out << std::endl;
      out << "This test must be run on 1 to 5 processes." << std::endl;
      TEST_EQUALITY(true, true);
      return;
    }

    Level level;
    RCP<FactoryManagerBase> factoryHandler = rcp(new FactoryManager());
    level.SetFactoryManager(factoryHandler);
    int nx=9;
    int ny=nx;
    int dofsPerNode = 3;
    GO numGlobalElements = nx*ny*dofsPerNode;
    size_t maxEntriesPerRow=30;

    RCP<const Map> map;
    int numMyNodes = numGlobalElements / dofsPerNode;
    if (comm->getSize() > 1) {
      // In parallel, make sure that the dof's associated with a node all
      // reside on the same processor.
      int numNodes = numGlobalElements / dofsPerNode;
      TEUCHOS_TEST_FOR_EXCEPTION( (numGlobalElements - numNodes * dofsPerNode) != 0, MueLu::Exceptions::RuntimeError,
                                  "Number of matrix rows is not divisible by #dofs" );
      int nproc = comm->getSize();
      if (comm->getRank() < nproc-1) numMyNodes = numNodes / nproc;
      else numMyNodes = numNodes - (numNodes/nproc) * (nproc-1);
      map = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, numMyNodes*dofsPerNode, comm);
    } else {
      map = MapFactory::createUniformContigMap(TestHelpers::Parameters::getLib(), numGlobalElements, comm);
    }

    const size_t numMyElements = map->getNodeNumElements();
    Teuchos::ArrayView<const GlobalOrdinal> myGlobalElements = map->getNodeElementList();
    RCP<Matrix> A = rcp(new CrsMatrixWrap(map, 1)); // Force underlying linear algebra library to allocate more
                                                    // memory on the fly.  While not super efficient, this
                                                    // ensures that no zeros are being stored.  Thus, from
                                                    // Zoltan's perspective the matrix is imbalanced.
    // Populate CrsMatrix with random number of entries (up to maxEntriesPerRow) per row.
    // Create a vector with random integer entries in [1,maxEntriesPerRow].
    ST::seedrandom(666*comm->getRank());
    RCP<Xpetra::Vector<LO,LO,GO,NO> > entriesPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(map,false);
    Teuchos::ArrayRCP<LO> eprData = entriesPerRow->getDataNonConst(0);
    for (Teuchos::ArrayRCP<LO>::iterator i=eprData.begin(); i!=eprData.end(); ++i) {
      *i = (LO)(std::floor(((ST::random()+1)*0.5*maxEntriesPerRow)+1));
    }

    RCP<Teuchos::FancyOStream> fos = Teuchos::fancyOStream(Teuchos::rcpFromRef(std::cout));
    fos->setOutputToRootOnly(-1);

    Teuchos::Array<Scalar> vals(maxEntriesPerRow);
    Teuchos::Array<GO> cols(maxEntriesPerRow);
    for (size_t i = 0; i < numMyElements; ++i) {
      Teuchos::ArrayView<SC> av(&vals[0],eprData[i]);
      Teuchos::ArrayView<GO> iv(&cols[0],eprData[i]);
      //stick in ones for values
      for (LO j=0; j< eprData[i]; ++j) vals[j] = ST::one();
      //figure out valid column indices
      GO start = std::max(myGlobalElements[i]-eprData[i]+1,0);
      for (LO j=0; j< eprData[i]; ++j) cols[j] = start+j;
      A->insertGlobalValues(myGlobalElements[i], iv, av);
    }

    A->fillComplete();

    // Now treat the matrix as if it has 3 DOFs per node.
    A->SetFixedBlockSize(dofsPerNode);
    level.Set("A",A);

    //build coordinates
    Teuchos::ParameterList list;
    list.set("nx",nx);
    list.set("ny",ny);
    RCP<const Map> coalescedMap = MapFactory::createContigMap(TestHelpers::Parameters::getLib(), numGlobalElements/dofsPerNode, numMyNodes, comm);
    RCP<MultiVector> XYZ = Galeri::Xpetra::Utils::CreateCartesianCoordinates<SC,LO,GO,Map,MultiVector>("2D",coalescedMap,list);

    // XYZ are the "coalesce" coordinates as it has been generated for 1 DOF/node and we are using them for 3 DOFS/node
    // level.Set("Coordinates",XYZ); "Coordinates" == uncoalesce. "X,Y,ZCoordinates" == coalesce
    {
      RCP<MultiVector> coordinates = XYZ;

      // making a copy because I don't want to keep 'open' the Xpetra_MultiVector
      if (coordinates->getNumVectors() >= 1) {
        Teuchos::ArrayRCP<const SC> coord = coordinates->getData(0);
        Teuchos::ArrayRCP<SC> coordCpy(coord.size());
        for(int i=0; i<coord.size(); i++) {
          coordCpy[i] = coord[i];
        }
        level.Set("XCoordinates", coordCpy);
        //std::cout << coordCpy << std::endl;
      }

      if (coordinates->getNumVectors() >= 2) {
        Teuchos::ArrayRCP<const SC> coord = coordinates->getData(1);
        Teuchos::ArrayRCP<SC> coordCpy(coord.size());
        for(int i=0; i<coord.size(); i++) {
          coordCpy[i] = coord[i];
        }
        level.Set("YCoordinates", coordCpy);
      }

      /*if (coordinates->getNumVectors() >= 3) {
        Teuchos::ArrayRCP<const SC> coord = coordinates->getData(2);
        Teuchos::ArrayRCP<SC> coordCpy(coord.size());
        for(int i=0; i<coord.size(); i++) {
          coordCpy[i] = coord[i];
        }
        level.Set("ZCoordinates", coordCpy);
        }*/
    }

    //coalescedMap->describe(*fos,Teuchos::VERB_EXTREME);
    //sleep(1); comm->barrier();
    //XYZ->describe(*fos,Teuchos::VERB_EXTREME);

    LO numPartitions = comm->getSize();
    level.Set("number of partitions",numPartitions);
    RCP<ZoltanInterface> zoltan = rcp(new ZoltanInterface());
    //zoltan->SetOutputLevel(0); //options are 0=none, 1=summary, 2=every pid prints
    level.Request("Partition",zoltan.get());
    zoltan->Build(level);

    RCP<Xpetra::Vector<GO,LO,GO,NO> > decomposition = level.Get<RCP<Xpetra::Vector<GO,LO,GO,NO> > >("Partition",zoltan.get());
    /* //temporary code to have the trivial decomposition (no change)
    ArrayRCP<GO> decompEntries = decomposition->getDataNonConst(0);
    for (ArrayRCP<GO>::iterator i = decompEntries.begin(); i != decompEntries.end(); ++i)
      *i = comm->getRank();
    decompEntries=Teuchos::null;
    */

    //Create vector whose local length is the global number of partitions.
    //This vector will record the local number of nonzeros associated with each partition.
    Teuchos::Array<GO> parts(numPartitions);
    for (int i=0; i<numPartitions; ++i) parts[i] = i;
    Teuchos::ArrayView<GO> partsView(&parts[0],numPartitions);
    RCP<const Map> partitionMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
                                                    Teuchos::OrdinalTraits<global_size_t>::invalid(), partsView,
                                                    map->getIndexBase(),comm);
    RCP<Xpetra::Vector<LO,LO,GO,NO> > localPartsVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(partitionMap);

    RCP<Xpetra::Vector<LO,LO,GO,NO> > nnzPerRow = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(A->getRowMap());
    Teuchos::ArrayRCP<GO> nnzData = nnzPerRow->getDataNonConst(0);
    //For the local rows in each partition, tally up the number of nonzeros.  This is what
    //Zoltan should be load-balancing.
    Teuchos::ArrayRCP<GO> lpvData = localPartsVec->getDataNonConst(0);
    Teuchos::ArrayRCP<const GO> decompData = decomposition->getData(0);
    for (size_t i=0; i<decomposition->getLocalLength();++i) {
      Teuchos::ArrayView<const LO> c;
      Teuchos::ArrayView<const SC> v;
      A->getLocalRowView(i,c,v);
      lpvData[decompData[i]] += v.size();
      nnzData[i] = v.size();
    }

    lpvData = Teuchos::null;
    decompData = Teuchos::null;
    nnzData = Teuchos::null;

    /*
    if (comm->getRank() == 0)
      std::cout << "nnz per row" << std::endl;
    nnzPerRow->describe(*fos,Teuchos::VERB_EXTREME);

    if (comm->getRank() == 0)
      std::cout << "Row-to-partition assignment (from Zoltan)" << std::endl;
    decomposition->describe(*fos,Teuchos::VERB_EXTREME);

    if (comm->getRank() == 0)
      std::cout << "#nonzeros per partition" << std::endl;
    localPartsVec->describe(*fos,Teuchos::VERB_EXTREME);
    */

    //Send the local nnz tallies to pid 0, which can report the global sums.
    size_t mysize=1;
    if (comm->getRank() == 0) mysize = numPartitions;
    RCP<const Map> globalTallyMap = MapFactory::Build(TestHelpers::Parameters::getLib(),
                                                Teuchos::OrdinalTraits<global_size_t>::invalid(),
                                                mysize,
                                                map->getIndexBase(),
                                                comm);
    RCP<Xpetra::Vector<LO,LO,GO,NO> > globalTallyVec = Xpetra::VectorFactory<LO,LO,GO,NO>::Build(globalTallyMap);
    RCP<const Export> exporter = ExportFactory::Build( partitionMap, globalTallyMap);
    globalTallyVec->doExport(*localPartsVec,*exporter,Xpetra::ADD);

    ArrayRCP<GO> expectedResults(numPartitions);
    switch (comm->getSize()) {
       case 1:
         expectedResults[0] = 3951;
         break;

       case 2:
         expectedResults[0] = 1955;
         expectedResults[1] = 1910;
         break;

       case 3:
         expectedResults[0] = 1326;
         expectedResults[1] = 1340;
         expectedResults[2] = 1321;
         break;

       case 4:
         expectedResults[0] = 950;
         expectedResults[1] = 922;
         expectedResults[2] = 908;
         expectedResults[3] = 936;
         break;

       case 5:
         expectedResults[0] = 774;
         expectedResults[1] = 735;
         expectedResults[2] = 726;
         expectedResults[3] = 771;
         expectedResults[4] = 759;
         break;

       default:
         break;
    };

    ArrayRCP<const LO> gtvData = globalTallyVec->getData(0);

#ifdef __linux__
    out << "Checking results..." << std::endl;
    for (int i=0; i<numPartitions; ++i) {
      if (comm->getRank() == 0) TEST_EQUALITY( expectedResults[i], gtvData[i]);
    }
#endif

#ifdef MUELU_VISUALIZE_REPARTITIONING
    //
    //Now write everything to a comma-separate list that ParaView can grok
    //
    Teuchos::ArrayRCP<const Scalar> X = XYZ->getData(0);
    Teuchos::ArrayRCP<const Scalar> Y = XYZ->getData(1);
    Teuchos::ArrayRCP<const GO> D = decomposition->getData(0);
    RCP<std::ofstream> outFile;
    std::string fileName = "zoltanResults.csv";

    //write header information
    if (comm->getRank() == 0) {
      outFile = rcp(new std::ofstream(fileName.c_str()));
      *outFile << "x coord, y coord, z coord, partition, row weight" << std::endl;
    }
    comm->barrier();

    //append coordinates
    nnzData = nnzPerRow->getDataNonConst(0);
    for (int j=0; j<comm->getSize(); ++j) {
      int mypid = comm->getRank();
      if (mypid == j) {
        outFile = rcp(new std::ofstream(fileName.c_str(),std::ios::app));
        int blockSize = A->GetFixedBlockSize();
        //Coordinates are for coalesced system, D is for uncoalesced
        for (int i=0; i < D.size()/blockSize; ++i) {
          int nnz=0;
          for (int k=0; k<blockSize; ++k)  nnz += nnzData[i*blockSize+k];
            *outFile << X[i] << ", " << Y[i] << ", " << ST::zero() << ", "
                     << D[i*blockSize] << ", " << nnz << std::endl;
        }
      }
    } //for (int i=0; i<comm->getSize(); ++i)

    out << std::endl;
    out << "You can view the Zoltan decomposition in ParaView 3.10.1 or later:" << std::endl;
    out << "   1) Load the data file " << fileName << "." << std::endl;
    out << "   2) Run the filter Filters/ Alphabetical/ Table To Points." << std::endl;
    out << "   3) Tell ParaView what columns are the X, Y and Z coordinates." << std::endl;
    out << "   4) Split screen horizontally (Icon, top right)." << std::endl;
    out << "   5) Click on the eyeball in the Pipeline Browser to see the points." << std::endl;
    out << "   6) Under the Display tab, you can color points by scalar value and resize them." << std::endl;
    out << std::endl;
    out << " To display row weights next to each point:" << std::endl;
    out << "   1) Click the \"Select Points Through\" button (2nd row) and select all points." << std::endl;
    out << "   2) Under View pull-down menu, choose the \"Selection Inspector\"." << std::endl;
    out << "   3) Under the Point Label, check the Visible box and set the Label Mode to \"row weight\"." << std::endl;
#endif

  } //Build3PDEs