//----------------------------------------------------------------------------
snl_fei::LinearSystem_General::LinearSystem_General(fei::SharedPtr<fei::MatrixGraph>& matrixGraph)
  : fei::LinearSystem(matrixGraph),
    comm_(matrixGraph->getRowSpace()->getCommunicator()),
    essBCvalues_(NULL),
    resolveConflictRequested_(false),
    bcs_trump_slaves_(false),
    explicitBCenforcement_(false),
    BCenforcement_no_column_mod_(false),
    localProc_(0),
    numProcs_(1),
    name_(),
    named_loadcomplete_counter_(),
    iwork_(),
    dwork_(),
    dbgprefix_("LinSysG: ")
{
  localProc_ = fei::localProc(comm_);
  numProcs_  = fei::numProcs(comm_);

  fei::SharedPtr<fei::VectorSpace> vecSpace = matrixGraph->getRowSpace();

  std::vector<int> offsets;
  vecSpace->getGlobalIndexOffsets(offsets);

  firstLocalOffset_ = offsets[localProc_];
  lastLocalOffset_ = offsets[localProc_+1]-1;

  setName("dbg");
}
Пример #2
0
fei::Vector_Impl<T>::Vector_Impl(fei::SharedPtr<fei::VectorSpace> vecSpace,
			   T* vector, int numLocalEqns,
			   bool isSolutionVector,
                           bool deleteVector)
  : Vector_core(vecSpace, numLocalEqns),
    vector_(vector),
    isSolution_(isSolutionVector),
    deleteVector_(deleteVector),
    localProc_(0),
    numProcs_(1),
    dbgprefix_("VecImpl: ")
{
  if (strcmp(snl_fei::FEVectorTraits<T>::typeName(), "unsupported")) {
    setFEVector(true);
  }
  else {
    setFEVector(false);
  }

  localProc_ = fei::localProc(vecSpace->getCommunicator());
  numProcs_ = fei::numProcs(vecSpace->getCommunicator());

  if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) {
    FEI_OSTREAM& os = *output_stream_;
    os << dbgprefix_<<" ctor, numLocalEqns="<<numLocalEqns
       <<", typeName: "<<typeName()<<FEI_ENDL;
  }
}
void getOwnedIndices_T<int>(const fei::SharedPtr<fei::VectorSpace> & vs,std::vector<int> & indices) 
{
   int numIndices, ni;
   numIndices = vs->getNumIndices_Owned();
   indices.resize(numIndices);

   // directly write to int indices
   vs->getIndices_Owned(numIndices,&indices[0],ni);
}
Пример #4
0
fei::SharedPtr<fei::Matrix>
Factory_Aztec::createMatrix(fei::SharedPtr<fei::MatrixGraph> matrixGraph)
{
  fei::SharedPtr<fei::Matrix> feimat;
  int globalNumSlaves = matrixGraph->getGlobalNumSlaveConstraints();

  if (globalNumSlaves > 0 && reducer_.get()==NULL) {
    reducer_ = matrixGraph->getReducer();
  }

  return feimat;
}
Пример #5
0
//==============================================================================
AztecDVBR_Matrix::AztecDVBR_Matrix(fei::SharedPtr<Aztec_BlockMap> map)
  : amap_(map),
    Amat_(NULL),
    N_update_(map->getNumLocalBlocks()),
    external_(NULL),
    extern_index_(NULL),
    update_index_(NULL),
    data_org_(NULL),
    orderingUpdate_(NULL),
    isLoaded_(false),
    isAllocated_(false),
    localNNZ_(0),
    nnzPerRow_(NULL),
    numRemoteBlocks_(0),
    remoteInds_(NULL),
    remoteBlockSizes_(NULL)
{
    nnzPerRow_ = new int[N_update_];

    for(int i=0; i<N_update_; i++) {
       nnzPerRow_[i] = 0;
    }

    Amat_ = AZ_matrix_create(N_update_);
    Amat_->matrix_type = AZ_VBR_MATRIX;
    Amat_->matvec =
            (void(*)(double*,double*,AZ_MATRIX_STRUCT*,int*))AZ_VBR_matvec_mult;

    //now we can allocate and fill the rpntr array.
    Amat_->rpntr = NULL;
    calcRpntr();
}
Пример #6
0
fei::Vector_core::Vector_core(fei::SharedPtr<fei::VectorSpace> vecSpace,
                              int numLocalEqns)
  : eqnComm_(),
    vecSpace_(vecSpace),
    comm_(vecSpace->getCommunicator()),
    firstLocalOffset_(0),
    lastLocalOffset_(0),
    numLocal_(0),
    work_indices_(),
    work_indices2_(),
    haveFEVector_(false),
    remotelyOwned_(),
    overlapAlreadySet_(false),
    dbgprefix_("Vcore: ")
{
  eqnComm_.reset(new fei::EqnComm(comm_,numLocalEqns));
  remotelyOwned_.resize(fei::numProcs(comm_));
  for(unsigned i=0; i<remotelyOwned_.size(); ++i) {
    remotelyOwned_[i] = new CSVec;
  }

  const std::vector<int>& offsets = eqnComm_->getGlobalOffsets();
  firstLocalOffset_ = offsets[fei::localProc(comm_)];
  lastLocalOffset_ = offsets[fei::localProc(comm_)+1] - 1;
  numLocal_ = lastLocalOffset_ - firstLocalOffset_ + 1;

  if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) {
    FEI_OSTREAM& os = *output_stream_;
    os<<dbgprefix_<<" ctor firstLocal="<<firstLocalOffset_<<", lastLocal="
     <<lastLocalOffset_<<FEI_ENDL;
  }
}
Пример #7
0
//----------------------------------------------------------------------------
snl_fei::Factory::Factory(MPI_Comm comm,
                          fei::SharedPtr<LibraryWrapper> wrapper)
  : fei::Factory(comm),
    comm_(comm),
    broker_(),
    matrixGraph_(),
    nodeIDType_(0),
    lsc_(),
    feData_(),
    wrapper_(wrapper),
    outputLevel_(0),
    blockMatrix_(false)
{
  if (wrapper_.get() != NULL) {
    lsc_ = wrapper->getLinearSystemCore();
    feData_ = wrapper->getFiniteElementData();
  }
}
Пример #8
0
void test_Matrix::matrix_test1(fei::SharedPtr<fei::Matrix> mat)
{
  if (localProc_==0)
    FEI_COUT << "  matrix_test1: testing fei::Matrix with type '"
	   << mat->typeName() << "':"<<FEI_ENDL;

  fei::SharedPtr<fei::MatrixGraph> mgraph = mat->getMatrixGraph();

  fei::SharedPtr<fei::VectorSpace> rspace = mgraph->getRowSpace();

  if (localProc_==0)
    FEI_COUT << "   testing get{Global/Local}NumRows,getRowLength...";

  int mglobalrows = mat->getGlobalNumRows();
  int vglobaleqns = rspace->getGlobalNumIndices();

  if (mglobalrows != vglobaleqns) {
    throw std::runtime_error("mat reports different num rows than vector-space eqns");
  }

  std::vector<int> global_offsets;
  rspace->getGlobalIndexOffsets(global_offsets);

  int my_num_rows = mat->getLocalNumRows();
  if (my_num_rows != global_offsets[localProc_+1]-global_offsets[localProc_]) {
    throw std::runtime_error("num-local-rows mis-match between mat and vector-space");
  }

  int i, my_first_row = global_offsets[localProc_];
  std::vector<int> row_lengths(my_num_rows);

  for(i=0; i<my_num_rows; ++i) {
    int errcode = mat->getRowLength(i+my_first_row, row_lengths[i]);
    if (errcode != 0) {
      throw std::runtime_error("nonzero errcode from mat->getRowLength");
    }
  }

  if (localProc_==0) FEI_COUT << "ok" << FEI_ENDL;
}
Пример #9
0
fei::SharedPtr<fei::Vector>
Factory_Aztec::createVector(fei::SharedPtr<fei::VectorSpace> vecSpace,
                               bool isSolutionVector,
                               int numVectors)
{
  std::vector<int> indices;
  int err = 0, localSize = 0;
  if (reducer_.get() != NULL) {
    indices = reducer_->getLocalReducedEqns();
    localSize = indices.size();
  }
  else {
    if (blockEntryMatrix_) {
      localSize = vecSpace->getNumBlkIndices_Owned();
      indices.resize(localSize*2);
      err = vecSpace->getBlkIndices_Owned(localSize, &indices[0], &indices[localSize], localSize);
    }
    else {
      localSize = vecSpace->getNumIndices_Owned();
      err = vecSpace->getIndices_Owned(indices);
    }
  }
  if (err != 0) {
    throw std::runtime_error("fei::Factory_Aztec: error in vecSpace->getIndices_Owned");
  }

  fei::SharedPtr<fei::Vector> feivec, tmpvec;

  if (reducer_.get() != NULL) {
    feivec.reset(new fei::VectorReducer(reducer_,
                                        tmpvec, isSolutionVector));
  }
  else {
    feivec = tmpvec;
  }

  return(feivec);
}
Пример #10
0
fei::SharedPtr<fei::Vector>
Factory_Aztec::createVector(fei::SharedPtr<fei::MatrixGraph> matrixGraph,
                               bool isSolutionVector,
                               int numVectors)
{
  int globalNumSlaves = matrixGraph->getGlobalNumSlaveConstraints();

  if (globalNumSlaves > 0 && reducer_.get()==NULL) {
    reducer_ = matrixGraph->getReducer();
  }

  fei::SharedPtr<fei::Vector> feivec, tmpvec;

  std::vector<int> indices;
  int err = 0, localSize;
  fei::SharedPtr<fei::VectorSpace> vecSpace = matrixGraph->getRowSpace();
  if (reducer_.get() != NULL) {
    indices = reducer_->getLocalReducedEqns();
    localSize = indices.size();
  }
  else {
    localSize = vecSpace->getNumIndices_Owned();
    indices.resize(localSize);
    err = vecSpace->getIndices_Owned(indices);
  }
  if (err != 0) {
    throw std::runtime_error("error in vecSpace->getIndices_Owned");
  }

  if (reducer_.get() != NULL) {
    feivec.reset(new fei::VectorReducer(reducer_, tmpvec, isSolutionVector));
  }
  else {
    feivec = tmpvec;
  }

  return(feivec);
}
Пример #11
0
fei::SharedPtr<fei::Matrix>
test_Matrix::create_matrix(fei::SharedPtr<fei::Factory> factory)
{
  testData test_data(localProc_, numProcs_);

  fei::SharedPtr<fei::VectorSpace> vspace =
    test_VectorSpace::create_VectorSpace(comm_, &test_data, localProc_, numProcs_,
					 false, false, (const char*)0, factory);
  int err = vspace->initComplete();
  if (err != 0) {
    FEI_COUT << "ERROR, failed to create valid fei::VectorSpace." << FEI_ENDL;
    throw std::runtime_error("test_Vector::vector_test1: ERROR, failed to create valid fei::VectorSpace.");
  }

  fei::SharedPtr<fei::MatrixGraph> mgraph =
    factory->createMatrixGraph(vspace, vspace, NULL);

  std::vector<int>& fieldIDs = test_data.fieldIDs;
  std::vector<int>& idTypes = test_data.idTypes;
  std::vector<int>& ids = test_data.ids;

  int numIDs = ids.size();
  int fieldID = fieldIDs[0];
  int idType = idTypes[0];

  int patternID = mgraph->definePattern(numIDs, idType, fieldID);

  mgraph->initConnectivityBlock(0, 1, patternID);

  mgraph->initConnectivity(0, 0, &ids[0]);

  mgraph->initComplete();

  fei::SharedPtr<fei::Matrix> matrix = factory->createMatrix(mgraph);
  return(matrix);
}
//----------------------------------------------------------------------------
snl_fei::Broker_LinSysCore::Broker_LinSysCore(fei::SharedPtr<LinearSystemCore> lsc,
			      fei::SharedPtr<fei::MatrixGraph> matrixGraph,
                              fei::SharedPtr<fei::Reducer> reducer,
                              bool blockMatrix)
  : linsyscore_(lsc),
    matrixGraph_(matrixGraph),
    reducer_(reducer),
    lookup_(NULL),
    setGlobalOffsets_(false),
    numLocalEqns_(0),
    setMatrixStructure_(false),
    blockMatrix_(blockMatrix)
{
  int dummyID = -1;
  lsc->setNumRHSVectors(1, &dummyID);
}
Пример #13
0
MatrixReducer::MatrixReducer(fei::SharedPtr<fei::Reducer> reducer,
                             fei::SharedPtr<fei::Matrix> target)
 : reducer_(reducer),
   target_(target),
   globalAssembleCalled_(false),
   changedSinceMark_(false)
{
  fei::SharedPtr<fei::VectorSpace> vspace =
    target->getMatrixGraph()->getRowSpace();
  MPI_Comm comm = vspace->getCommunicator();
  int numLocal = reducer_->getLocalReducedEqns().size();
  fei::SharedPtr<fei::EqnComm> eqnComm(new fei::EqnComm(comm, numLocal));
  fei::Matrix_core* target_core =
    dynamic_cast<fei::Matrix_core*>(target_.get());
    if (target_core == NULL) {
    throw std::runtime_error("fei::MatrixReducer ERROR, target matrix not dynamic_cast-able to fei::Matrix_core.");
  }

  target_core->setEqnComm(eqnComm);
}
Пример #14
0
//----------------------------------------------------------------------------
VectorReducer::VectorReducer(fei::SharedPtr<fei::Reducer> reducer,
                             fei::SharedPtr<fei::Vector> target,
                             bool isSolutionVector)
  : reducer_(reducer),
    target_(target),
    isSolution_(isSolutionVector)
{
  localProc_ = fei::localProc(target->getVectorSpace()->getCommunicator());
  numProcs_ = fei::numProcs(target->getVectorSpace()->getCommunicator());

  fei::Vector_core* target_core = dynamic_cast<fei::Vector_core*>(target.get());
  if (target_core == NULL) {
    throw std::runtime_error("fei::VectorReducer ERROR, target vector not dynamic_cast-able to fei::Vector_core.");
  }

  fei::SharedPtr<fei::VectorSpace> vecspace = target->getVectorSpace();
  int numEqns = vecspace->getNumIndices_SharedAndOwned();
  std::vector<int> eqns;
  vecspace->getIndices_SharedAndOwned(eqns);

  std::vector<int> overlap;
  for(int i=0; i<numEqns; ++i) {
    if (!reducer->isSlaveEqn(eqns[i])) {
      overlap.push_back(reducer->translateToReducedEqn(eqns[i]));
    }
    else {
      std::vector<int> masters;
      reducer->getSlaveMasterEqns(eqns[i], masters);
      for(unsigned j=0; j<masters.size(); ++j) {
        overlap.push_back(reducer->translateToReducedEqn(masters[j]));
      }
    }
  }

  target_core->setOverlap(overlap.size(), &overlap[0]);
}
Пример #15
0
fei::Vector_Impl<T>::Vector_Impl(fei::SharedPtr<fei::VectorSpace> vecSpace,
			   T* vector, int numLocalEqns,
			   bool isSolutionVector,
                           bool deleteVector)
  : Vector_core(vecSpace, numLocalEqns),
    vector_(vector),
    isSolution_(isSolutionVector),
    deleteVector_(deleteVector),
    localProc_(0),
    numProcs_(1),
    dbgprefix_("VecImpl: ")
{
  if (strcmp(snl_fei::FEVectorTraits<T>::typeName(), "unsupported")) {
    setFEVector(true);
  }
  else {
    setFEVector(false);
  }

  localProc_ = fei::localProc(vecSpace->getCommunicator());
  numProcs_ = fei::numProcs(vecSpace->getCommunicator());

  if (output_level_ >= fei::BRIEF_LOGS && output_stream_ != NULL) {
    FEI_OSTREAM& os = *output_stream_;
    os << dbgprefix_<<" ctor, numLocalEqns="<<numLocalEqns
       <<", typeName: "<<typeName()<<FEI_ENDL;
  }

  std::vector<int> idTypes;
  vecSpace->getIDTypes(idTypes);
  std::vector<int> eqns;
  std::vector<double> zeros;
  for(size_t i=0; i<idTypes.size(); ++i) {
    int idType = idTypes[i];
    fei::SharedIDs<int>& sharedIDs = vecSpace->getSharedIDs(idType);
    const fei::SharedIDs<int>::map_type& idMap = sharedIDs.getSharedIDs();
    fei::SharedIDs<int>::map_type::const_iterator
      iter = idMap.begin(), iterEnd = idMap.end();
    for(; iter!=iterEnd; ++iter) {
      int ID = iter->first;
      int eqn;
      vecSpace->getGlobalIndex(idType, ID, eqn);
      int ndof = vecSpace->getNumDegreesOfFreedom(idType, ID);
      eqns.resize(ndof);
      zeros.resize(ndof, 0.0);
      for(int j=0; j<ndof; ++j) eqns[j] = eqn+j;
      if (!isSolutionVector) {
        sumIn(ndof, &eqns[0], &zeros[0]);
      }
      else {
        copyIn(ndof, &eqns[0], &zeros[0]);
      }
    }
  }

  setCommSizes();
  std::vector<CSVec*>& remoteVecs = remotelyOwned();
  for(size_t i=0; i<remoteVecs.size(); ++i) {
    remoteVecs[i]->clear();
  }
}
void getOwnedAndSharedIndices_T<int>(const fei::SharedPtr<fei::VectorSpace> & vs,std::vector<int> & indices) 
{
   // get the global indices
   vs->getIndices_SharedAndOwned(indices);
}
Пример #17
0
fei::SharedPtr<fei::MatrixGraph> test_MatrixGraph::create_MatrixGraph(testData* testdata,
					 int localProc, int numProcs,
					 bool bothFields, bool nonSymmetric,
					 const char* name,
					 fei::SharedPtr<fei::VectorSpace> vectorSpacePtr,
					 fei::SharedPtr<fei::Factory> factory,
                                         const std::string& path,
					 bool turnOnDebugOutput)
{
  //
  //This function creates a MatrixGraph object, and initializes it as follows:
  //
  //setRowSpace(vectorSpacePtr)
  //
  //definePattern patternID=0, numIDs=4, idType=testdata->idTypes[0]
  //      fieldID=testdata->fieldIDs[0] if !bothFields, else
  //      fieldIDs=testdata->fieldIDs
  //
  //initConnectivityBlock blockID=0, numConnectivityLists=1
  //
  //initConnectivity blockID, 0, testdata->ids
  //
  //If nonSymmetric==true, then also do the following:
  //  definePattern patternID=1, numIDs=1, idType=testdata->idTypes[0]
  //     fieldID=testdata->fieldIDs[0] if !bothFields, else
  //      fieldIDs=testdata->fieldIDs
  //  definePattern patternID=2, numIDs=4, idType=testdata->idTypes[0]
  //     fieldID=testdata->fieldIDs[0] if !bothFields, else
  //      fieldIDs=testdata->fieldIDs
  //
  //initConnectivityBlock blockID=1, patterns 1 and 2
  //
  //initConnectivity blockID, 0, testdata->ids
  //
  fei::SharedPtr<fei::MatrixGraph> mgptr;
  if (factory.get() == NULL) {
    fei::SharedPtr<fei::MatrixGraph> tmp(new fei::MatrixGraph_Impl2(vectorSpacePtr,
							      vectorSpacePtr, name));
    mgptr = tmp;
  }
  else {
    mgptr = factory->createMatrixGraph(vectorSpacePtr, vectorSpacePtr, name);
  }

  fei::ParameterSet paramset;
  fei::Param param1("name", name);
  paramset.add(param1);
  if (turnOnDebugOutput) {
    if (path.empty()) {
      fei::Param param2("debugOutput", ".");
      paramset.add(param2);
    }
    else {
      fei::Param param2("debugOutput", path.c_str());
      paramset.add(param2);
    }
  }

  fei::MatrixGraph* matrixGraphPtr = mgptr.get();

  matrixGraphPtr->setParameters(paramset);

  matrixGraphPtr->setRowSpace(vectorSpacePtr);

  int patternID = 0;
  int numIDs = 4;
  int idType = testdata->idTypes[0];
  int fieldID = testdata->fieldIDs[0];

  if (bothFields) {
    std::vector<int> numFieldsPerID(numIDs, 2);
    std::vector<int> fieldIDsArray(numIDs*2);
    for(int i=0; i<numIDs; ++i) {
      fieldIDsArray[i*2] = testdata->fieldIDs[0];
      fieldIDsArray[i*2+1] = testdata->fieldIDs[1];
    }

    patternID = matrixGraphPtr->definePattern(numIDs, idType,
					 &numFieldsPerID[0],
					 &fieldIDsArray[0]);
  }
  else {
    patternID = matrixGraphPtr->definePattern(numIDs, idType, fieldID);
  }

  int blockID = 0;
  int numConnectivityLists = 1;

  matrixGraphPtr->initConnectivityBlock(blockID,
					   numConnectivityLists,
					   patternID);

  matrixGraphPtr->initConnectivity(blockID, 0, &(testdata->ids[0]));

  if (!nonSymmetric) {
    return(mgptr);
  }

  int patternID1 = 1, patternID2 = 2;
  int numRowIDs = 1, numColIDs = 4;

  if (bothFields) {
    std::vector<int> numFieldsPerID(numIDs, 2);
    std::vector<int> fieldIDsArray(numIDs*2);
    for(int i=0; i<numIDs; ++i) {
      fieldIDsArray[i*2] = testdata->fieldIDs[0];
      fieldIDsArray[i*2+1] = testdata->fieldIDs[1];
    }

    patternID1 = matrixGraphPtr->definePattern(numRowIDs, idType,
					 &numFieldsPerID[0],
					 &fieldIDsArray[0]);
    patternID2 = matrixGraphPtr->definePattern(numColIDs, idType,
					 &numFieldsPerID[0],
					 &fieldIDsArray[0]);
  }
  else {
    patternID1 = matrixGraphPtr->definePattern(numRowIDs,
					   idType, fieldID);
    patternID2 = matrixGraphPtr->definePattern(numColIDs,
					   idType, fieldID);
  }

  blockID = 1;

  matrixGraphPtr->initConnectivityBlock(blockID,
					   numConnectivityLists,
					   patternID1, patternID2);

  matrixGraphPtr->initConnectivity(blockID, 0,
					  &(testdata->ids[0]),
					  &(testdata->ids[0]));

  return(mgptr);
}
//----------------------------------------------------------------------------
int extractDirichletBCs(fei::DirichletBCManager* bcManager,
                fei::SharedPtr<fei::MatrixGraph> matrixGraph,
                fei::CSVec* essBCvalues,
                bool resolveConflictRequested,
                bool bcs_trump_slaves)
{
//  int numLocalBCs = bcManager->getNumBCRecords();
//  int globalNumBCs = 0;
//  MPI_Comm comm = matrixGraph->getRowSpace()->getCommunicator();
//  fei::GlobalSum(comm, numLocalBCs, globalNumBCs);
//  if (globalNumBCs == 0) {
//    return(0);
//  }

  fei::SharedPtr<fei::FillableMat> localBCeqns(new fei::FillableMat);
  fei::SharedPtr<fei::Matrix_Impl<fei::FillableMat> > bcEqns;
//  matrixGraph->getRowSpace()->initComplete();
  int numSlaves = matrixGraph->getGlobalNumSlaveConstraints();
  fei::SharedPtr<fei::Reducer> reducer = matrixGraph->getReducer();

  int numIndices = numSlaves>0 ?
    reducer->getLocalReducedEqns().size() :
    matrixGraph->getRowSpace()->getNumIndices_Owned();

  bool zeroSharedRows = false;
  bcEqns.reset(new fei::Matrix_Impl<fei::FillableMat>(localBCeqns, matrixGraph, numIndices, zeroSharedRows));
  fei::SharedPtr<fei::Matrix> bcEqns_reducer;
  if (numSlaves > 0) {
    bcEqns_reducer.reset(new fei::MatrixReducer(reducer, bcEqns));
  }

  fei::Matrix& bcEqns_mat = bcEqns_reducer.get()==NULL ?
      *bcEqns : *bcEqns_reducer;

  CHK_ERR( bcManager->finalizeBCEqns(bcEqns_mat, bcs_trump_slaves) );

  if (resolveConflictRequested) {
    fei::SharedPtr<fei::FillableMat> mat = bcEqns->getMatrix();
    std::vector<int> bcEqnNumbers;
    fei::get_row_numbers(*mat, bcEqnNumbers);
    CHK_ERR( snl_fei::resolveConflictingCRs(*matrixGraph, bcEqns_mat,
                                            bcEqnNumbers) );
  }

  std::vector<int> essEqns;
  std::vector<double> values;

  std::map<int,fei::FillableMat*>& remotes = bcEqns->getRemotelyOwnedMatrices();
  std::map<int,fei::FillableMat*>::iterator
    it = remotes.begin(),
    it_end = remotes.end();
  for(; it!=it_end; ++it) {
    fei::impl_utils::separate_BC_eqns( *(it->second), essEqns, values);
  }

//  CHK_ERR( bcEqns->gatherFromOverlap(false) );

  fei::impl_utils::separate_BC_eqns( *(bcEqns->getMatrix()), essEqns, values);

  if (essEqns.size() > 0) {
    int* essEqnsPtr = &essEqns[0];
    double* valuesPtr = &values[0];

    for(unsigned i=0; i<essEqns.size(); ++i) {
      int eqn = essEqnsPtr[i];
      double value = valuesPtr[i];
      fei::put_entry(*essBCvalues, eqn, value);
    }
  }

  return(0);
}