void 
TpetraLinearObjFactory<Traits,ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT>::
ghostToGlobalTpetraMatrix(const Tpetra::CrsMatrix<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> & in,
                          Tpetra::CrsMatrix<ScalarT,LocalOrdinalT,GlobalOrdinalT,NodeT> & out) const
{
   using Teuchos::RCP;

   // do the global distribution
   RCP<ExportType> exporter = getGhostedExport();
   
   out.resumeFill();
   out.setAllToScalar(0.0);
   out.doExport(in,*exporter,Tpetra::ADD);
   out.fillComplete();
}
Beispiel #2
0
int main(int argc, char* argv[]){
        Teuchos::oblackholestream blackhole;
        Teuchos::GlobalMPISession mpiSession(&argc,&argv,&blackhole);
        typedef double Scalar;
        typedef int Ordinal;
        using Tpetra::global_size_t;

        Teuchos::RCP<const Teuchos::Comm<int> > comm = Tpetra::DefaultPlatform::getDefaultPlatform().getComm();
  Teuchos::RCP<Teuchos::FancyOStream> out = Teuchos::fancyOStream(Teuchos::rcp(&std::cout,false));
  //out->setOutputToRootOnly(comm->getRank());

        size_t myRank = comm->getRank();
        size_t numProc = comm->getSize();
        bool verbose = (myRank==0);

        std::cout << *comm;

        const global_size_t numGlobalElements = 4;
        if (numGlobalElements < numProc) {
                if (verbose) {
                        std::cout << "numGlobalBlocks = " << numGlobalElements
                        << " cannot be less than the number of processors = " << numProc << std::endl;
                }
                return -1;
        }

        // Construct a Map that puts approximately the same number of equations on each processor.

        Teuchos::RCP<const Tpetra::Map<Ordinal> > map = Tpetra::createUniformContigMap<Ordinal,Ordinal>(numGlobalElements, comm);

        // Get update list and number of local equations from newly created map.

        const size_t numMyElements = map->getNodeNumElements();

        Teuchos::ArrayView<const Ordinal> myGlobalElements = map->getNodeElementList();

        // Create an OTeger vector NumNz that is used to build the Petra Matrix.
        // NumNz[i] is the Number of OFF-DIAGONAL term for the ith global equation
        // on this processor

        Teuchos::ArrayRCP<size_t> NumNz = Teuchos::arcp<size_t>(numMyElements);

        // We are building a tridiagonal matrix where each row has (-1 2 -1)
        // So we need 2 off-diagonal terms (except for the first and last equation)

        for (size_t i=0; i < numMyElements; ++i) {
                if (myGlobalElements[i] == 0 || static_cast<global_size_t>(myGlobalElements[i]) == numGlobalElements-1) {
                // boundary
                        NumNz[i] = 2;
                }
                else {
                        NumNz[i] = 3;
                }
        }

        // Create a Tpetra::Matrix using the Map, with a static allocation dictated by NumNz
        Tpetra::CrsMatrix<Scalar,Ordinal>  A (map, NumNz, Tpetra::StaticProfile);
        Tpetra::CrsMatrix<Scalar,Ordinal>  AT(map, NumNz, Tpetra::StaticProfile);
        Teuchos::RCP< Tpetra::CrsMatrix<Scalar,Ordinal> > TestMatrix = Teuchos::null;

        // We are done with NumNZ
        NumNz = Teuchos::null;

        // Add  rows one-at-a-time
        // Off diagonal values will always be -1
        const Scalar two    = static_cast<Scalar>( 2.0);
        const Scalar negOne = static_cast<Scalar>(-1.0);
        const Scalar three = static_cast<Scalar>(3.0);
        for (size_t i=0; i<numMyElements; i++) {
                if (myGlobalElements[i] == 0) {
                        A.insertGlobalValues( myGlobalElements[i],
                        Teuchos::tuple<Ordinal>( myGlobalElements[i], myGlobalElements[i]+1 ),
                        Teuchos::tuple<Scalar> ( two, negOne ) );
                }
                else if (static_cast<global_size_t>(myGlobalElements[i]) == numGlobalElements-1) {
                        A.insertGlobalValues( myGlobalElements[i],
                        Teuchos::tuple<Ordinal>( myGlobalElements[i]-1, myGlobalElements[i] ),
                        Teuchos::tuple<Scalar> ( negOne, two ) );
                }
                else {
                        A.insertGlobalValues( myGlobalElements[i],
                        Teuchos::tuple<Ordinal>( myGlobalElements[i]-1, myGlobalElements[i], myGlobalElements[i]+1 ),
                        Teuchos::tuple<Scalar> ( three, two, negOne ) );
                }
        }


        for (size_t i=0; i<numMyElements; i++) {
                if (myGlobalElements[i] == 0) {
                        AT.insertGlobalValues( myGlobalElements[i],
                        Teuchos::tuple<Ordinal>( myGlobalElements[i], myGlobalElements[i]+1 ),
                        Teuchos::tuple<Scalar> ( two, three ) );
                }
                else if (static_cast<global_size_t>(myGlobalElements[i]) == numGlobalElements-1) {
                        AT.insertGlobalValues( myGlobalElements[i],
                        Teuchos::tuple<Ordinal>( myGlobalElements[i]-1, myGlobalElements[i] ),
                        Teuchos::tuple<Scalar> ( negOne, two ) );
                }
                else if(static_cast<global_size_t>(myGlobalElements[i])==1){
                        AT.insertGlobalValues( myGlobalElements[i],
                        Teuchos::tuple<Ordinal>( myGlobalElements[i]-1, myGlobalElements[i], myGlobalElements[i]+1 ),
                        Teuchos::tuple<Scalar> ( negOne, two, three ) );
                }
                else if(static_cast<global_size_t>(myGlobalElements[i])==2){
                        AT.insertGlobalValues( myGlobalElements[i],
                        Teuchos::tuple<Ordinal>( myGlobalElements[i]-1, myGlobalElements[i], myGlobalElements[i]+1 ),
                        Teuchos::tuple<Scalar> ( negOne, two, negOne ) );
                }
        }

        // Finish up
        A.fillComplete();
        AT.fillComplete();


        Tpetra::RowMatrixTransposer<Scalar, Ordinal> transposer (Teuchos::rcpFromRef (A));
        TestMatrix = transposer.createTranspose(); //, TestMatrix/*, tMap*/);

  Teuchos::RCP<Tpetra::CrsMatrix<Scalar, Ordinal> > diffMatrix = Tpetra::createCrsMatrix<Scalar, Ordinal>(TestMatrix->getRowMap());
  //Apparently there is a problem with ADD because while these two matricies are
  //identical when I add them together I don't get 0 like I should. Infact
  //I just get a matrix that has the exact same entries and sparsity structure.
  //I'll have to come back to this later. But RowMatrixTransposer is working right.
  //And all my other tests are telling me ADD works right too.
  //KLN 06/14/2011
  Tpetra::MatrixMatrix::Add(AT,false,-1.0,*TestMatrix,false, 1.0,diffMatrix);
  diffMatrix->fillComplete();
  //diffMatrix->describe(*out, Teuchos::VERB_EXTREME);
  double diffNorm = getNorm(*diffMatrix);
  double realNorm = getNorm(AT);
  double epsilon = diffNorm/realNorm;
  if(epsilon > 1e-10){
    *out << "The calculated A transpose and the real one don't match!" << std::endl;
    *out << "Diff Norm: " << diffNorm << std::endl;
    *out << "Real Norm: " << realNorm << std::endl;
    *out << "Epsilon: " << epsilon << std::endl;
    return 1;
  }




        return 0;
}
  RCP<BlockCrsMatrix<Scalar, LO, GO, Node>>
  convertToBlockCrsMatrix(const Tpetra::CrsMatrix<Scalar, LO, GO, Node>& pointMatrix, const LO &blockSize)
  {

      /*
        ASSUMPTIONS:

           1) In point matrix, all entries associated with a little block are present (even if they are zero).
           2) For given mesh DOF, point DOFs appear consecutively and in ascending order in row & column maps.
           3) Point column map and block column map are ordered consistently.
      */

      using Teuchos::ArrayView;
      using Teuchos::Array;

      typedef Tpetra::Experimental::BlockCrsMatrix<Scalar,LO,GO,Node> block_crs_matrix_type;
      typedef Tpetra::Map<LO,GO,Node>                                 map_type;
      typedef Tpetra::CrsGraph<LO,GO,Node>                            crs_graph_type;

      const map_type &pointRowMap = *(pointMatrix.getRowMap());
      RCP<const map_type> meshRowMap = createMeshMap<LO,GO,Node>(blockSize, pointRowMap);

      const map_type &pointColMap = *(pointMatrix.getColMap());
      RCP<const map_type> meshColMap = createMeshMap<LO,GO,Node>(blockSize, pointColMap);

      const map_type &pointDomainMap = *(pointMatrix.getDomainMap());
      RCP<const map_type> meshDomainMap = createMeshMap<LO,GO,Node>(blockSize, pointDomainMap);

      const map_type &pointRangeMap = *(pointMatrix.getRangeMap());
      RCP<const map_type> meshRangeMap = createMeshMap<LO,GO,Node>(blockSize, pointRangeMap);

      // Use graph ctor that provides column map and upper bound on nonzeros per row.
      // We can use static profile because the point graph should have at least as many entries per
      // row as the mesh graph.
      RCP<crs_graph_type> meshCrsGraph = rcp(new crs_graph_type(meshRowMap, meshColMap,
                                                 pointMatrix.getGlobalMaxNumRowEntries(), Tpetra::StaticProfile));
      // Fill the graph by walking through the matrix.  For each mesh row, we query the collection of point
      // rows associated with it. The point column ids are converted to mesh column ids and put into an array.
      // As each point row collection is finished, the mesh column ids are sorted, made unique, and inserted
      // into the mesh graph.
      ArrayView<const LO> pointColInds;
      ArrayView<const Scalar> pointVals;
      Array<GO> meshColGids;
      meshColGids.reserve(pointMatrix.getGlobalMaxNumRowEntries());
      //again, I assume that point GIDs associated with a mesh GID are consecutive.
      //if they are not, this will break!!
      for (size_t i=0; i<pointMatrix.getNodeNumRows()/blockSize; i++) {
        for (int j=0; j<blockSize; ++j) {
          LO rowLid = i*blockSize+j;
          pointMatrix.getLocalRowView(rowLid,pointColInds,pointVals); //TODO optimization: Since I don't care about values,
                                                                      //TODO I should use the graph instead.
          for (int k=0; k<pointColInds.size(); ++k) {
            GO meshColInd = pointColMap.getGlobalElement(pointColInds[k]) / blockSize;
            meshColGids.push_back(meshColInd);
          }
        }
        //List of mesh GIDs probably contains duplicates because we looped over all point rows in the block.
        //Sort and make unique.
        std::sort(meshColGids.begin(), meshColGids.end());
        meshColGids.erase( std::unique(meshColGids.begin(), meshColGids.end()), meshColGids.end() );
        meshCrsGraph->insertGlobalIndices(meshRowMap->getGlobalElement(i), meshColGids());
        meshColGids.clear();
      }
      meshCrsGraph->fillComplete(meshDomainMap,meshRangeMap);

      //create and populate the block matrix
      RCP<block_crs_matrix_type> blockMatrix = rcp(new block_crs_matrix_type(*meshCrsGraph, blockSize));

      //preallocate the maximum number of (dense) block entries needed by any row
      int maxBlockEntries = blockMatrix->getNodeMaxNumRowEntries();
      Array<Array<Scalar>> blocks(maxBlockEntries);
      for (int i=0; i<maxBlockEntries; ++i)
        blocks[i].reserve(blockSize*blockSize);
      std::map<int,int> bcol2bentry;             //maps block column index to dense block entries
      std::map<int,int>::iterator iter;
      //Fill the block matrix.  We must do this in local index space.
      //TODO: Optimization: We assume the blocks are fully populated in the point matrix.  This means
      //TODO: on the first point row in the block row, we know that we're hitting new block col indices.
      //TODO: on other rows, we know the block col indices have all been seen before
      //int offset;
      //if (pointMatrix.getIndexBase()) offset = 0;
      //else                     offset = 1;
      for (size_t i=0; i<pointMatrix.getNodeNumRows()/blockSize; i++) {
        int blkCnt=0; //how many unique block entries encountered so far in current block row
        for (int j=0; j<blockSize; ++j) {
          LO rowLid = i*blockSize+j;
          pointMatrix.getLocalRowView(rowLid,pointColInds,pointVals);
          for (int k=0; k<pointColInds.size(); ++k) {
            //convert point column to block col
            LO meshColInd = pointColInds[k] / blockSize;
            iter = bcol2bentry.find(meshColInd);
            if (iter == bcol2bentry.end()) {
              //new block column
              bcol2bentry[meshColInd] = blkCnt;
              blocks[blkCnt].push_back(pointVals[k]);
              blkCnt++;
            } else {
              //block column found previously
              int littleBlock = iter->second;
              blocks[littleBlock].push_back(pointVals[k]);
            }
          }
        }
        // TODO This inserts the blocks one block entry at a time.  It is probably more efficient to
        // TODO store all the blocks in a block row contiguously so they can be inserted with a single call.
        for (iter=bcol2bentry.begin(); iter != bcol2bentry.end(); ++iter) {
          LO localBlockCol = iter->first;
          Scalar *vals = (blocks[iter->second]).getRawPtr();
          blockMatrix->replaceLocalValues(i, &localBlockCol, vals, 1);
        }

        //Done with block row.  Zero everything out.
        for (int j=0; j<maxBlockEntries; ++j)
          blocks[j].clear();
        blkCnt = 0;
        bcol2bentry.clear();
      }

      return blockMatrix;

  }
void 
Redistributor<Node>::redistribute(const ::Tpetra::CrsMatrix<double,int,int,Node>& inputMatrix, 
                                  ::Tpetra::CrsMatrix<double,int,int,Node> * &outputMatrix, bool callFillComplete)
{
  if (!created_importer_) {
    create_importer(inputMatrix.RowMap());
  }

  // First obtain the length of each of my new rows

  int myOldRows = inputMatrix.NumMyRows();
  int myNewRows = target_map_->NumMyElements();

  double *nnz = new double [myOldRows];
  for (int i=0; i < myOldRows; i++){
    nnz[i] = inputMatrix.NumMyEntries(i);
  }

  Teuchos::ArrayView<double>::ArrayView   myNNZView(nnz, myOldRows);

  ::Tpetra::Vector<double,int,int,Node> oldRowSizes(inputMatrix.RowMap(), myNNZView);

  if (myOldRows)
    delete [] nnz;

  ::Tpetra::Vector<double,int,int,Node> newRowSizes(*target_map_);

  newRowSizes.Import(oldRowSizes, *importer_, ::Tpetra::INSERT);

  int *rowSize=0;
  if(myNewRows){
    rowSize = new int [myNewRows];
    for (int i=0; i< myNewRows; i++){
      rowSize[i] = static_cast<int>(newRowSizes[i]);
    }
  }

  Teuchos::ArrayView<int>::ArrayView rowSizeView(rowSize, myNewRows);

  // Receive new rows, send old rows

  outputMatrix = new ::Tpetra::CrsMatrix<double,int,int,Node> (*target_map_, rowSizeView, true);

  if (myNewRows)
    delete [] rowSize;

  outputMatrix->Import(inputMatrix, *importer_, ::Tpetra::INSERT);

  // Set the new domain map such that
  // (a) if old DomainMap == old RangeMap, preserve this property,
  // (b) otherwise, let the new DomainMap be the old DomainMap 
  const ::Tpetra::Map<int,int,Node> *newDomainMap;
  if (inputMatrix.DomainMap().SameAs(inputMatrix.RangeMap()))
     newDomainMap = &(outputMatrix->RangeMap());
  else
     newDomainMap = &(inputMatrix.DomainMap());

  if (callFillComplete && (!outputMatrix->Filled()))
    outputMatrix->FillComplete(*newDomainMap,  *target_map_);

  return;
}