void ISVDUDV::Initialize( const Teuchos::RCP< Teuchos::ParameterList >& params,
const Teuchos::RCP< const Epetra_MultiVector >& ss,
const Teuchos::RCP< RBGen::FileIOHandler< Epetra_Operator > >& fileio)
{
workU_ = Teuchos::rcp( new Epetra_MultiVector(ss->Map(),maxBasisSize_,false) );
Epetra_LocalMap lclmap(ss->NumVectors(),0,ss->Comm());
workV_ = Teuchos::rcp( new Epetra_MultiVector(lclmap,maxBasisSize_,false) );
}
ZoltanLibClass::ZoltanLibClass(Teuchos::RCP<const Epetra_MultiVector> input_coords,
Teuchos::RCP<const Epetra_MultiVector> weights,
int inputType):
Library(input_coords, weights, inputType)
{
int weightDim = weights->NumVectors();
if (weightDim > 1){
if (input_coords->Comm().MyPID() == 0){
std::cout << "WARNING: Zoltan will only use the first weight of the "<< weightDim << " supplied for each object" << std::endl;
}
}
}
//EpetraMap_To_TpetraMap: takes in Epetra_Map object, converts it to its equivalent Tpetra::Map object,
//and returns an RCP pointer to this Tpetra::Map
Teuchos::RCP<const Tpetra_Map> Petra::EpetraMap_To_TpetraMap(const Teuchos::RCP<const Epetra_Map>& epetraMap_,
const Teuchos::RCP<const Teuchos::Comm<int> >& commT_)
{
const std::size_t numElements = Teuchos::as<std::size_t>(epetraMap_->NumMyElements());
const auto indexBase = Teuchos::as<GO>(epetraMap_->IndexBase());
if (epetraMap_->DistributedGlobal() || epetraMap_->Comm().NumProc() == Teuchos::OrdinalTraits<int>::one()) {
Teuchos::Array<Tpetra_GO> indices(numElements);
int *epetra_indices = epetraMap_->MyGlobalElements();
for(LO i=0; i < numElements; i++)
indices[i] = epetra_indices[i];
const Tpetra::global_size_t computeGlobalElements = Teuchos::OrdinalTraits<Tpetra::global_size_t>::invalid();
return Teuchos::rcp(new Tpetra_Map(computeGlobalElements, indices, indexBase, commT_));
} else {
return Teuchos::rcp(new Tpetra_Map(numElements, indexBase, commT_, Tpetra::LocallyReplicated));
}
}
LOCA::Epetra::CompactWYOp::CompactWYOp(
const Teuchos::RCP<LOCA::GlobalData>& global_data,
const Teuchos::RCP<const Epetra_Operator>& jacOperator,
const Teuchos::RCP<const Epetra_MultiVector>& A_multiVec,
const Teuchos::RCP<const Epetra_MultiVector>& Y_x_multiVec,
const Teuchos::RCP<const NOX::Abstract::MultiVector::DenseMatrix>& Y_p_matrix,
const Teuchos::RCP<const NOX::Abstract::MultiVector::DenseMatrix>& T_matrix) :
globalData(global_data),
label("LOCA::Epetra::CompactWYOp"),
localMap(Y_x_multiVec->NumVectors(), 0, jacOperator->Comm()),
J(jacOperator),
A(A_multiVec),
Y_x(Y_x_multiVec),
Y_p(View, localMap, Y_p_matrix->values(), Y_p_matrix->stride(),
Y_p_matrix->numCols()),
T(View, localMap, T_matrix->values(), T_matrix->stride(),
T_matrix->numCols()),
tmpMat1(NULL),
tmpMV(NULL),
dblas()
{
}
NOX::Abstract::Group::ReturnType
LOCA::Epetra::Group::computeComplex(double frequency)
{
std::string callingFunction =
"LOCA::Epetra::Group::computeComplex()";
// We must have the time-dependent interface
if (userInterfaceTime == Teuchos::null)
return NOX::Abstract::Group::BadDependency;
#ifdef HAVE_NOX_EPETRAEXT
if (isValidComplex)
return NOX::Abstract::Group::Ok;
NOX::Abstract::Group::ReturnType finalStatus = NOX::Abstract::Group::Ok;
NOX::Abstract::Group::ReturnType status;
// Get Jacobian matrix
Teuchos::RCP<Epetra_RowMatrix> jac = Teuchos::rcp_dynamic_cast<Epetra_RowMatrix>(sharedLinearSystem.getObject(this)->getJacobianOperator());
// Create complex matrix
if (complexMatrix == Teuchos::null) {
std::vector< std::vector<int> >rowStencil(2);
std::vector<int> rowIndex;
rowStencil[0].push_back(0); rowStencil[0].push_back(1);
rowStencil[1].push_back(-1); rowStencil[1].push_back(0);
rowIndex.push_back(0); rowIndex.push_back(1);
complexMatrix = Teuchos::rcp(new EpetraExt::BlockCrsMatrix(*jac,
rowStencil,
rowIndex,
jac->Comm()));
// Construct global solution vector, the overlap vector, and importer between them
complexVec =
Teuchos::rcp(new EpetraExt::BlockVector(jac->RowMatrixRowMap(),
complexMatrix->RowMap()));
}
// Get mass matrix M
Teuchos::RCP<Epetra_RowMatrix> mass = Teuchos::rcp_dynamic_cast<Epetra_RowMatrix>(shiftedSharedLinearSystem->getObject(this)->getJacobianOperator());
// Compute w*M
status = computeShiftedMatrix(0.0, frequency);
finalStatus =
globalData->locaErrorCheck->combineAndCheckReturnTypes(status,
finalStatus,
callingFunction);
// Load w*M in complex matrix
complexMatrix->LoadBlock(*mass, 1, 0);
// Compute -w*M
status = computeShiftedMatrix(0.0, -frequency);
finalStatus =
globalData->locaErrorCheck->combineAndCheckReturnTypes(status,
finalStatus,
callingFunction);
// Load -w*M in complex matrix
complexMatrix->LoadBlock(*mass, 0, 1);
// Compute J
status = computeJacobian();
finalStatus =
globalData->locaErrorCheck->combineAndCheckReturnTypes(status,
finalStatus,
callingFunction);
// Load J in complex matrix
complexMatrix->LoadBlock(*jac, 0, 0);
complexMatrix->LoadBlock(*jac, 1, 1);
// Create linear system
if (complexSharedLinearSystem == Teuchos::null) {
NOX::Epetra::Vector nev(complexVec, NOX::Epetra::Vector::CreateView);
Teuchos::RCP<Teuchos::ParameterList> lsParams =
globalData->parsedParams->getSublist("Linear Solver");
// Create the Linear System
Teuchos::RCP<NOX::Epetra::Interface::Required> iReq;
Teuchos::RCP<NOX::Epetra::Interface::Jacobian> iJac;
Teuchos::RCP<NOX::Epetra::LinearSystem> complexLinSys =
Teuchos::rcp(new NOX::Epetra::LinearSystemAztecOO(printParams,
*lsParams,
iReq,
iJac,
complexMatrix,
nev));
complexLinSys->setJacobianOperatorForSolve(complexMatrix);
complexSharedLinearSystem =
Teuchos::rcp(new NOX::SharedObject<NOX::Epetra::LinearSystem,
LOCA::Epetra::Group>(complexLinSys));
}
if (finalStatus == NOX::Abstract::Group::Ok)
isValidComplex = true;
return finalStatus;
#else
globalData->locaErrorCheck->throwError(callingFunction,
"Must have EpetraExt support for Hopf tracking. Configure trilinos with --enable-epetraext");
return NOX::Abstract::Group::BadDependency;
#endif
}
void IncSVDPOD::Initialize( const Teuchos::RCP< Teuchos::ParameterList >& params,
const Teuchos::RCP< const Epetra_MultiVector >& ss,
const Teuchos::RCP< RBGen::FileIOHandler< Epetra_Operator > >& fileio ) {
using Teuchos::rcp;
// Get the "Reduced Basis Method" sublist.
Teuchos::ParameterList rbmethod_params = params->sublist( "Reduced Basis Method" );
// Get the maximum basis size
maxBasisSize_ = rbmethod_params.get<int>("Max Basis Size");
TEUCHOS_TEST_FOR_EXCEPTION(maxBasisSize_ < 2,std::invalid_argument,"\"Max Basis Size\" must be at least 2.");
// Get a filter
filter_ = rbmethod_params.get<Teuchos::RCP<Filter<double> > >("Filter",Teuchos::null);
if (filter_ == Teuchos::null) {
int k = rbmethod_params.get("Rank",(int)5);
filter_ = rcp( new RangeFilter<double>(LARGEST,k,k) );
}
// Get convergence tolerance
tol_ = rbmethod_params.get<double>("Convergence Tolerance",tol_);
// Get debugging flag
debug_ = rbmethod_params.get<bool>("IncSVD Debug",debug_);
// Get verbosity level
verbLevel_ = rbmethod_params.get<int>("IncSVD Verbosity Level",verbLevel_);
// Get an Anasazi orthomanager
if (rbmethod_params.isType<
Teuchos::RCP< Anasazi::OrthoManager<double,Epetra_MultiVector> >
>("Ortho Manager")
)
{
ortho_ = rbmethod_params.get<
Teuchos::RCP<Anasazi::OrthoManager<double,Epetra_MultiVector> >
>("Ortho Manager");
TEUCHOS_TEST_FOR_EXCEPTION(ortho_ == Teuchos::null,std::invalid_argument,"User specified null ortho manager.");
}
else {
std::string omstr = rbmethod_params.get("Ortho Manager","DGKS");
if (omstr == "SVQB") {
ortho_ = rcp( new Anasazi::SVQBOrthoManager<double,Epetra_MultiVector,Epetra_Operator>() );
}
else { // if omstr == "DGKS"
ortho_ = rcp( new Anasazi::BasicOrthoManager<double,Epetra_MultiVector,Epetra_Operator>() );
}
}
// Lmin,Lmax,Kstart
lmin_ = rbmethod_params.get("Min Update Size",1);
TEUCHOS_TEST_FOR_EXCEPTION(lmin_ < 1 || lmin_ >= maxBasisSize_,std::invalid_argument,
"Method requires 1 <= min update size < max basis size.");
lmax_ = rbmethod_params.get("Max Update Size",maxBasisSize_);
TEUCHOS_TEST_FOR_EXCEPTION(lmin_ > lmax_,std::invalid_argument,"Max update size must be >= min update size.");
startRank_ = rbmethod_params.get("Start Rank",lmin_);
TEUCHOS_TEST_FOR_EXCEPTION(startRank_ < 1 || startRank_ > maxBasisSize_,std::invalid_argument,
"Starting rank must be in [1,maxBasisSize_)");
// MaxNumPasses
maxNumPasses_ = rbmethod_params.get("Maximum Number Passes",maxNumPasses_);
TEUCHOS_TEST_FOR_EXCEPTION(maxNumPasses_ != -1 && maxNumPasses_ <= 0, std::invalid_argument,
"Maximum number passes must be -1 or > 0.");
// Save the pointer to the snapshot matrix
TEUCHOS_TEST_FOR_EXCEPTION(ss == Teuchos::null,std::invalid_argument,"Input snapshot matrix cannot be null.");
A_ = ss;
// MaxNumCols
maxNumCols_ = A_->NumVectors();
maxNumCols_ = rbmethod_params.get("Maximum Number Columns",maxNumCols_);
TEUCHOS_TEST_FOR_EXCEPTION(maxNumCols_ < A_->NumVectors(), std::invalid_argument,
"Maximum number of columns must be at least as many as in the initializing data set.");
// V locally replicated or globally distributed
// this must be true for now
// Vlocal_ = rbmethod_params.get("V Local",Vlocal_);
// Allocate space for the factorization
sigma_.reserve( maxBasisSize_ );
U_ = Teuchos::null;
V_ = Teuchos::null;
U_ = rcp( new Epetra_MultiVector(ss->Map(),maxBasisSize_,false) );
if (Vlocal_) {
Epetra_LocalMap lclmap(maxNumCols_,0,ss->Comm());
V_ = rcp( new Epetra_MultiVector(lclmap,maxBasisSize_,false) );
}
else {
Epetra_Map gblmap(maxNumCols_,0,ss->Comm());
V_ = rcp( new Epetra_MultiVector(gblmap,maxBasisSize_,false) );
}
B_ = rcp( new Epetra_SerialDenseMatrix(maxBasisSize_,maxBasisSize_) );
resNorms_.reserve(maxBasisSize_);
// clear counters
numProc_ = 0;
curNumPasses_ = 0;
// we are now initialized, albeit with null rank
isInitialized_ = true;
}
static int run_test(Teuchos::RCP<Epetra_CrsMatrix> matrix,
bool verbose, // display the graph before & after
bool contract, // set global number of partitions to 1/2 num procs
int partitioningType, // hypergraph or graph partitioning, or simple
int vertexWeightType, // use vertex weights?
int edgeWeightType, // use edge/hyperedge weights?
int objectType) // use isorropia's CrsMatrix or CrsGraph
{
int rc=0, fail = 0;
#ifdef HAVE_EPETRAEXT
int localProc = 0;
double balance1, balance2, cutn1, cutn2, cutl1, cutl2;
double balance3, cutn3, cutl3;
double cutWgt1, cutWgt2, cutWgt3;
int numCuts1, numCuts2, numCuts3, valid;
int numPartitions = 0;
int keepDenseEdges = 0;
int numProcs = 1;
#ifdef HAVE_MPI
const Epetra_MpiComm &Comm = dynamic_cast<const Epetra_MpiComm &>(matrix->Comm());
localProc = Comm.MyPID();
numProcs = Comm.NumProc();
#else
const Epetra_SerialComm &Comm = dynamic_cast<const Epetra_SerialComm &>(matrix->Comm());
#endif
int numRows = matrix->NumGlobalRows();
if (numRows < (numProcs * 100)){
// By default Zoltan throws out dense edges, defined as those
// whose number of non-zeros exceeds 25% of the number of vertices.
//
// If dense edges are thrown out of a small matrix, there may be nothing left.
keepDenseEdges = 1;
}
double myShareBefore = 1.0 / numProcs;
double myShare = myShareBefore;
if (contract){
numPartitions = numProcs / 2;
if (numPartitions > numRows)
numPartitions = numRows;
if (numPartitions > 0){
if (localProc < numPartitions){
myShare = 1.0 / numPartitions;
}
else{
myShare = 0.0;
}
}
else{
contract = 0;
}
}
// If we want Zoltan's or Isorropia's default weights, then we don't
// need to supply a CostDescriber object to createBalancedCopy,
// so we get to test the API functions that don't take a CostDescriber.
bool noCosts = ((vertexWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS) &&
(edgeWeightType == NO_APPLICATION_SUPPLIED_WEIGHTS));
// Test the interface that has no parameters, if possible
bool noParams =
((partitioningType == HYPERGRAPH_PARTITIONING) && // default, so requires no params
(numPartitions == 0) && // >0 would require a parameter
(keepDenseEdges == 0)); // >0 would require a parameter
// Maps for original object
const Epetra_Map &sourceRowMap = matrix->RowMap();
const Epetra_Map &sourceRangeMap = matrix->RangeMap();
// const Epetra_Map &sourceColMap = matrix->ColMap();
const Epetra_Map &sourceDomainMap = matrix->DomainMap();
int numCols = matrix->NumGlobalCols();
int nMyRows = sourceRowMap.NumMyElements();
int base = sourceRowMap.IndexBase();
// Compute vertex and edge weights
Isorropia::Epetra::CostDescriber costs;
Teuchos::RCP<Epetra_Vector> vptr;
Teuchos::RCP<Epetra_CrsMatrix> eptr;
Teuchos::RCP<Epetra_Vector> hyperEdgeWeights;
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
// Create graph edge weights.
eptr = Teuchos::rcp(new Epetra_CrsMatrix(*matrix));
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
eptr->PutScalar(1.0); // set all nonzeros to 1.0
}
else{
int maxRowSize = eptr->MaxNumEntries();
double *newVal = NULL;
if (maxRowSize > 0){
newVal = new double [maxRowSize];
for (int j=0; j<maxRowSize; j++){
newVal[j] = localProc + 1 + j;
}
}
int numEntries;
int *idx;
double *val;
for (int i=0; i<nMyRows; i++){
rc = eptr->ExtractMyRowView(i, numEntries, val, idx);
for (int j=0; j<numEntries; j++){
val[j] = newVal[j];
}
}
if (newVal) delete [] newVal;
}
eptr->FillComplete(sourceDomainMap, sourceRangeMap);
costs.setGraphEdgeWeights(eptr);
}
else{
// Create hyperedge weights. (Note that the list of hyperedges that a
// process provides weights for has no relation to the columns
// that it has non-zeroes for, or the rows that is has. Hypergraphs
// in general are not square. Also more than one process can provide
// a weight for the same edge. Zoltan combines the weights according
// to the value of the PHG_EDGE_WEIGHT_OPERATION parameter. The default
// for this parameter is to use the maximum edge weight provided by any
// process for a given hyperedge.)
Epetra_Map hyperEdgeMap(numCols, base, Comm);
hyperEdgeWeights = Teuchos::rcp(new Epetra_Vector(hyperEdgeMap));
int *edgeGIDs = NULL;
double *weights = NULL;
int numHEweights = hyperEdgeMap.NumMyElements();
if (numHEweights){
edgeGIDs = new int [numHEweights];
weights = new double [numHEweights];
if (edgeWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
weights[i] = 1.0;
}
}
else{
int hiVolumeStart = matrix->NumGlobalCols() / 3;
int hiVolumeEnd = hiVolumeStart * 2;
for (int i=0; i<numHEweights; i++){
edgeGIDs[i] = hyperEdgeMap.GID(i);
if ((edgeGIDs[i] < hiVolumeStart) || (edgeGIDs[i] >= hiVolumeEnd)){
weights[i] = 1.0;
}
else{
weights[i] = 3.0;
}
}
}
hyperEdgeWeights->ReplaceGlobalValues(numHEweights, weights, edgeGIDs);
}
if (weights){
delete [] weights;
delete [] edgeGIDs;
}
costs.setHypergraphEdgeWeights(hyperEdgeWeights);
}
}
bool need_importer = false;
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
need_importer = true; // to redistribute row weights
double *val = NULL;
if (nMyRows){
val = new double [nMyRows];
if (vertexWeightType == SUPPLY_EQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0;
}
}
else if (vertexWeightType == SUPPLY_UNEQUAL_WEIGHTS){
for (int i=0; i<nMyRows; i++){
val[i] = 1.0 + ((localProc+1) / 2);
}
}
}
vptr = Teuchos::rcp(new Epetra_Vector(Copy, sourceRowMap, val));
if (val) delete [] val;
costs.setVertexWeights(vptr);
}
// Calculate partition quality metrics before calling Zoltan
if (partitioningType == GRAPH_PARTITIONING){
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShare, balance1, numCuts1, cutWgt1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_graph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, numCuts3, cutWgt3, cutn3, cutl3);
}
}
else{
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShare, balance1, cutn1, cutl1);
if (contract){
// balance wrt target of balancing weight over *all* procs
rc = ispatest::compute_hypergraph_metrics(matrix->Graph(), costs,
myShareBefore, balance3, cutn3, cutl3);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
Teuchos::ParameterList params;
#ifdef HAVE_ISORROPIA_ZOLTAN
if (!noParams){
// We're using Zoltan for partitioning and supplying
// parameters, overriding defaults.
Teuchos::ParameterList &sublist = params.sublist("Zoltan");
if (partitioningType == GRAPH_PARTITIONING){
params.set("PARTITIONING METHOD", "GRAPH");
sublist.set("GRAPH_PACKAGE", "PHG");
}
else{
params.set("PARTITIONING METHOD", "HYPERGRAPH");
sublist.set("LB_APPROACH", "PARTITION");
sublist.set("PHG_CUT_OBJECTIVE", "CONNECTIVITY"); // "cutl"
}
if (keepDenseEdges){
// only throw out rows that have no zeroes, default is to
// throw out if .25 or more of the columns are non-zero
sublist.set("PHG_EDGE_SIZE_THRESHOLD", "1.0");
}
if (numPartitions > 0){
// test #Partitions < #Processes
std::ostringstream os;
os << numPartitions;
std::string s = os.str();
// sublist.set("NUM_GLOBAL_PARTS", s);
params.set("NUM PARTS", s);
}
//sublist.set("DEBUG_LEVEL", "1"); // Zoltan will print out parameters
//sublist.set("DEBUG_LEVEL", "5"); // proc 0 will trace Zoltan calls
//sublist.set("DEBUG_MEMORY", "2"); // Zoltan will trace alloc & free
}
#else
ERROREXIT((localProc==0),
"Zoltan partitioning required but Zoltan not available.")
#endif
// Function scope values
Teuchos::RCP<Epetra_Vector> newvwgts;
Teuchos::RCP<Epetra_CrsMatrix> newewgts;
// Function scope values required for LinearProblem
Epetra_LinearProblem *problem = NULL;
Epetra_Map *LHSmap = NULL;
Epetra_MultiVector *RHS = NULL;
Epetra_MultiVector *LHS = NULL;
// Reference counted pointer to balanced object
Epetra_CrsMatrix *matrixPtr=NULL;
Epetra_CrsGraph *graphPtr=NULL;
Epetra_RowMatrix *rowMatrixPtr=NULL;
Epetra_LinearProblem *problemPtr=NULL;
// Row map for balanced object
const Epetra_BlockMap *targetBlockRowMap=NULL; // for input CrsGraph
const Epetra_Map *targetRowMap=NULL; // for all other inputs
// Column map for balanced object
const Epetra_BlockMap *targetBlockColMap=NULL; // for input CrsGraph
const Epetra_Map *targetColMap=NULL; // for all other inputs
if (objectType == EPETRA_CRSMATRIX){
if (noParams && noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
matrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(matrixPtr->RowMap());
targetColMap = &(matrixPtr->ColMap());
}
else if (objectType == EPETRA_CRSGRAPH){
const Epetra_CrsGraph graph = matrix->Graph();
if (noParams && noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph);
}
else if (noCosts){
graphPtr = Isorropia::Epetra::createBalancedCopy(graph, params);
}
targetBlockRowMap = &(graphPtr->RowMap());
targetBlockColMap = &(graphPtr->ColMap());
}
else if (objectType == EPETRA_ROWMATRIX){
if (noParams && noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix);
}
else if (noCosts){
rowMatrixPtr = Isorropia::Epetra::createBalancedCopy(*matrix, params);
}
targetRowMap = &(rowMatrixPtr->RowMatrixRowMap());
targetColMap = &(rowMatrixPtr->RowMatrixColMap());
}
else if (objectType == EPETRA_LINEARPROBLEM){
// Create a linear problem with this matrix.
LHSmap = new Epetra_Map(numCols, base, Comm);
int myRHSsize = sourceRowMap.NumMyElements();
int myLHSsize = LHSmap->NumMyElements();
int valSize = ((myRHSsize > myLHSsize) ? myRHSsize : myLHSsize);
double *vals = NULL;
if (valSize){
vals = new double [valSize];
}
if (valSize){
for (int i=0; i < valSize; i++){
// put my rank in my portion of LHS and my portion of RHS
vals[i] = localProc;
}
}
RHS = new Epetra_MultiVector(Copy, sourceRowMap, vals, 1, 1);
LHS = new Epetra_MultiVector(Copy, *LHSmap, vals, 1, 1);
if (valSize){
delete [] vals;
}
problem = new Epetra_LinearProblem(matrix.get(), LHS, RHS);
Epetra_LinearProblem lp = *problem;
if (lp.CheckInput()){
ERROREXIT((localProc==0), "Error creating a LinearProblem");
}
if (noParams && noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp);
}
else if (noCosts){
problemPtr = Isorropia::Epetra::createBalancedCopy(lp, params);
}
targetRowMap = &(problemPtr->GetMatrix()->RowMatrixRowMap());
targetColMap = &(problemPtr->GetMatrix()->RowMatrixColMap());
}
// Redistribute the edge weights
// Comment this out since we don't redistribute columns
if (edgeWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS){
if (partitioningType == GRAPH_PARTITIONING){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *graphPtr));
targetRowMap = &(newewgts->RowMap());
targetColMap = &(newewgts->ColMap());
}
else{
newewgts = Teuchos::rcp(new Epetra_CrsMatrix(Copy, *targetRowMap, *targetColMap, 0));
}
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
newewgts->Import(*eptr, *importer, Insert);
newewgts->FillComplete(*targetColMap, *targetRowMap);
costs.setGraphEdgeWeights(newewgts);
}
}
// Redistribute the vertex weights
if ((vertexWeightType != NO_APPLICATION_SUPPLIED_WEIGHTS)){
Epetra_Import *importer = NULL;
if (objectType == EPETRA_CRSGRAPH){
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetBlockRowMap));
importer = new Epetra_Import(*targetBlockRowMap, sourceRowMap);
}
else{
newvwgts = Teuchos::rcp(new Epetra_Vector(*targetRowMap));
importer = new Epetra_Import(*targetRowMap, sourceRowMap);
}
newvwgts->Import(*vptr, *importer, Insert);
costs.setVertexWeights(newvwgts);
}
if (localProc == 0){
test_type(numPartitions, partitioningType, vertexWeightType, edgeWeightType, objectType);
}
if (verbose){
// Picture of problem before balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("Before load balancing", *problem, Comm);
}
else{
ispatest::show_matrix("Before load balancing", matrix->Graph(), Comm);
}
// Picture of problem after balancing
if (objectType == EPETRA_LINEARPROBLEM){
ispatest::show_matrix("After load balancing (x in Ax=b is not redistributed)", *problemPtr, Comm);
}
else if (objectType == EPETRA_ROWMATRIX){
ispatest::show_matrix("After load balancing", *rowMatrixPtr, Comm);
}
else if (objectType == EPETRA_CRSMATRIX){
ispatest::show_matrix("After load balancing", matrixPtr->Graph(), Comm);
}
else if (objectType == EPETRA_CRSGRAPH){
ispatest::show_matrix("After load balancing", *graphPtr, Comm);
}
}
// After partitioning, recompute the metrics
if (partitioningType == GRAPH_PARTITIONING){
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_graph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_graph_metrics(*rowMatrixPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_graph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
else {
rc = ispatest::compute_graph_metrics(*graphPtr, costs,
myShare, balance2, numCuts2, cutWgt2, cutn2, cutl2);
}
}
else{
if (objectType == EPETRA_LINEARPROBLEM){
rc = ispatest::compute_hypergraph_metrics(*(problemPtr->GetMatrix()), costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_ROWMATRIX){
rc = ispatest::compute_hypergraph_metrics(*rowMatrixPtr, costs,
myShare, balance2, cutn2, cutl2);
}
else if (objectType == EPETRA_CRSMATRIX){
rc = ispatest::compute_hypergraph_metrics(matrixPtr->Graph(), costs,
myShare, balance2, cutn2, cutl2);
}
else{
rc = ispatest::compute_hypergraph_metrics(*graphPtr, costs,
myShare, balance2, cutn2, cutl2);
}
}
if (rc){
ERROREXIT((localProc==0), "Error in computing partitioning metrics")
}
std::string why;
if (partitioningType == GRAPH_PARTITIONING){
fail = (cutWgt2 > cutWgt1);
why = "New weighted edge cuts are worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << " Total edge cuts: " << numCuts1;
std::cout << " Total weighted edge cuts: " << cutWgt1 << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
std::cout << " Total edge cuts: " << numCuts2;
std::cout << " Total weighted edge cuts: " << cutWgt2 << std::endl;
}
}
else{
fail = (cutl2 > cutl1);
why = "New cutl is worse";
if (localProc == 0){
std::cout << "Before partitioning: Balance " << balance1 ;
std::cout << " cutn " << cutn1 ;
std::cout << " cutl " << cutl1 ;
if (contract){
std::cout << " (wrt balancing over " << numPartitions << " partitions)" << std::endl;
std::cout << "Before partitioning: Balance " << balance3 ;
std::cout << " cutn " << cutn3 ;
std::cout << " cutl " << cutl3 ;
std::cout << " (wrt balancing over " << numProcs << " partitions)" ;
}
std::cout << std::endl;
std::cout << "After partitioning: Balance " << balance2 ;
std::cout << " cutn " << cutn2 ;
std::cout << " cutl " << cutl2 << std::endl;
}
}
if (fail){
if (localProc == 0) std::cout << "ERROR: "+why << std::endl;
}
// Check that input matrix is valid. This test constructs an "x"
// with the matrix->DomainMap() and a "y" with matrix->RangeMap()
// and then calculates y = Ax.
if (objectType == EPETRA_LINEARPROBLEM){
valid = ispatest::test_matrix_vector_multiply(*problemPtr);
}
else if (objectType == EPETRA_ROWMATRIX){
valid = ispatest::test_row_matrix_vector_multiply(*rowMatrixPtr);
}
else if (objectType == EPETRA_CRSMATRIX){
valid = ispatest::test_matrix_vector_multiply(*matrixPtr);
}
else{
valid = ispatest::test_matrix_vector_multiply(*graphPtr);
}
if (!valid){
if (localProc == 0) std::cout << "Rebalanced matrix is not a valid Epetra matrix" << std::endl;
fail = 1;
}
else{
if (localProc == 0) std::cout << "Rebalanced matrix is a valid Epetra matrix" << std::endl;
}
if (localProc == 0)
std::cout << std::endl;
#else
std::cout << "test_simple main: currently can only test "
<< "with Epetra and EpetraExt enabled." << std::endl;
rc = -1;
#endif
return fail;
}
// helper routines
bool SplitMatrix2x2(Teuchos::RCP<const Epetra_CrsMatrix> A,
const Epetra_Map& A11rowmap,
const Epetra_Map& A22rowmap,
Teuchos::RCP<Epetra_CrsMatrix>& A11,
Teuchos::RCP<Epetra_CrsMatrix>& A12,
Teuchos::RCP<Epetra_CrsMatrix>& A21,
Teuchos::RCP<Epetra_CrsMatrix>& A22)
{
if (A==Teuchos::null)
{
std::cout << "ERROR: SplitMatrix2x2: A==null on entry" << std::endl;
return false;
}
const Epetra_Comm& Comm = A->Comm();
const Epetra_Map& A22map = A22rowmap;
const Epetra_Map& A11map = A11rowmap;
//----------------------------- create a parallel redundant map of A22map
std::map<int,int> a22gmap;
{
std::vector<int> a22global(A22map.NumGlobalElements());
int count=0;
for (int proc=0; proc<Comm.NumProc(); ++proc)
{
int length = 0;
if (proc==Comm.MyPID())
{
for (int i=0; i<A22map.NumMyElements(); ++i)
{
a22global[count+length] = A22map.GID(i);
++length;
}
}
Comm.Broadcast(&length,1,proc);
Comm.Broadcast(&a22global[count],length,proc);
count += length;
}
if (count != A22map.NumGlobalElements())
{
std::cout << "ERROR SplitMatrix2x2: mismatch in dimensions" << std::endl;
return false;
}
// create the map
for (int i=0; i<count; ++i)
a22gmap[a22global[i]] = 1;
a22global.clear();
}
//--------------------------------------------------- create matrix A22
A22 = Teuchos::rcp(new Epetra_CrsMatrix(Copy,A22map,100));
{
std::vector<int> a22gcindices(100);
std::vector<double> a22values(100);
for (int i=0; i<A->NumMyRows(); ++i)
{
const int grid = A->GRID(i);
if (A22map.MyGID(grid)==false)
continue;
int numentries;
double* values;
int* cindices;
int err = A->ExtractMyRowView(i,numentries,values,cindices);
if (err)
{
std::cout << "ERROR: SplitMatrix2x2: A->ExtractMyRowView returned " << err << std::endl;
return false;
}
if (numentries>(int)a22gcindices.size())
{
a22gcindices.resize(numentries);
a22values.resize(numentries);
}
int count=0;
for (int j=0; j<numentries; ++j)
{
const int gcid = A->ColMap().GID(cindices[j]);
// see whether we have gcid in a22gmap
std::map<int,int>::iterator curr = a22gmap.find(gcid);
if (curr==a22gmap.end()) continue;
//std::cout << gcid << " ";
a22gcindices[count] = gcid;
a22values[count] = values[j];
++count;
}
//std::cout << std::endl; fflush(stdout);
// add this filtered row to A22
err = A22->InsertGlobalValues(grid,count,&a22values[0],&a22gcindices[0]);
if (err<0)
{
std::cout << "ERROR: SplitMatrix2x2: A->InsertGlobalValues returned " << err << std::endl;
return false;
}
} //for (int i=0; i<A->NumMyRows(); ++i)
a22gcindices.clear();
a22values.clear();
}
A22->FillComplete();
A22->OptimizeStorage();
//----------------------------------------------------- create matrix A11
A11 = Teuchos::rcp(new Epetra_CrsMatrix(Copy,A11map,100));
{
std::vector<int> a11gcindices(100);
std::vector<double> a11values(100);
for (int i=0; i<A->NumMyRows(); ++i)
{
const int grid = A->GRID(i);
if (A11map.MyGID(grid)==false) continue;
int numentries;
double* values;
int* cindices;
int err = A->ExtractMyRowView(i,numentries,values,cindices);
if (err)
{
std::cout << "ERROR: SplitMatrix2x2: A->ExtractMyRowView returned " << err << std::endl;
return false;
}
if (numentries>(int)a11gcindices.size())
{
a11gcindices.resize(numentries);
a11values.resize(numentries);
}
int count=0;
for (int j=0; j<numentries; ++j)
{
const int gcid = A->ColMap().GID(cindices[j]);
// see whether we have gcid as part of a22gmap
std::map<int,int>::iterator curr = a22gmap.find(gcid);
if (curr!=a22gmap.end()) continue;
a11gcindices[count] = gcid;
a11values[count] = values[j];
++count;
}
err = A11->InsertGlobalValues(grid,count,&a11values[0],&a11gcindices[0]);
if (err<0)
{
std::cout << "ERROR: SplitMatrix2x2: A->InsertGlobalValues returned " << err << std::endl;
return false;
}
} // for (int i=0; i<A->NumMyRows(); ++i)
a11gcindices.clear();
a11values.clear();
}
A11->FillComplete();
A11->OptimizeStorage();
//---------------------------------------------------- create matrix A12
A12 = Teuchos::rcp(new Epetra_CrsMatrix(Copy,A11map,100));
{
std::vector<int> a12gcindices(100);
std::vector<double> a12values(100);
for (int i=0; i<A->NumMyRows(); ++i)
{
const int grid = A->GRID(i);
if (A11map.MyGID(grid)==false) continue;
int numentries;
double* values;
int* cindices;
int err = A->ExtractMyRowView(i,numentries,values,cindices);
if (err)
{
std::cout << "ERROR: SplitMatrix2x2: A->ExtractMyRowView returned " << err << std::endl;
return false;
}
if (numentries>(int)a12gcindices.size())
{
a12gcindices.resize(numentries);
a12values.resize(numentries);
}
int count=0;
for (int j=0; j<numentries; ++j)
{
const int gcid = A->ColMap().GID(cindices[j]);
// see whether we have gcid as part of a22gmap
std::map<int,int>::iterator curr = a22gmap.find(gcid);
if (curr==a22gmap.end()) continue;
a12gcindices[count] = gcid;
a12values[count] = values[j];
++count;
}
err = A12->InsertGlobalValues(grid,count,&a12values[0],&a12gcindices[0]);
if (err<0)
{
std::cout << "ERROR: SplitMatrix2x2: A->InsertGlobalValues returned " << err << std::endl;
return false;
}
} // for (int i=0; i<A->NumMyRows(); ++i)
a12values.clear();
a12gcindices.clear();
}
A12->FillComplete(A22map,A11map);
A12->OptimizeStorage();
//----------------------------------------------------------- create A21
A21 = Teuchos::rcp(new Epetra_CrsMatrix(Copy,A22map,100));
{
std::vector<int> a21gcindices(100);
std::vector<double> a21values(100);
for (int i=0; i<A->NumMyRows(); ++i)
{
const int grid = A->GRID(i);
if (A22map.MyGID(grid)==false) continue;
int numentries;
double* values;
int* cindices;
int err = A->ExtractMyRowView(i,numentries,values,cindices);
if (err)
{
std::cout << "ERROR: SplitMatrix2x2: A->ExtractMyRowView returned " << err << std::endl;
return false;
}
if (numentries>(int)a21gcindices.size())
{
a21gcindices.resize(numentries);
a21values.resize(numentries);
}
int count=0;
for (int j=0; j<numentries; ++j)
{
const int gcid = A->ColMap().GID(cindices[j]);
// see whether we have gcid as part of a22gmap
std::map<int,int>::iterator curr = a22gmap.find(gcid);
if (curr!=a22gmap.end()) continue;
a21gcindices[count] = gcid;
a21values[count] = values[j];
++count;
}
err = A21->InsertGlobalValues(grid,count,&a21values[0],&a21gcindices[0]);
if (err<0)
{
std::cout << "ERROR: SplitMatrix2x2: A->InsertGlobalValues returned " << err << std::endl;
return false;
}
} // for (int i=0; i<A->NumMyRows(); ++i)
a21values.clear();
a21gcindices.clear();
}
A21->FillComplete(A11map,A22map);
A21->OptimizeStorage();
//-------------------------------------------------------------- tidy up
a22gmap.clear();
return true;
}
void PeridigmNS::Block::createMapsFromGlobalMaps(Teuchos::RCP<const Epetra_BlockMap> globalOwnedScalarPointMap,
Teuchos::RCP<const Epetra_BlockMap> globalOverlapScalarPointMap,
Teuchos::RCP<const Epetra_BlockMap> globalOwnedVectorPointMap,
Teuchos::RCP<const Epetra_BlockMap> globalOverlapVectorPointMap,
Teuchos::RCP<const Epetra_BlockMap> globalOwnedScalarBondMap,
Teuchos::RCP<const Epetra_Vector> globalBlockIds,
Teuchos::RCP<const PeridigmNS::NeighborhoodData> globalNeighborhoodData,
Teuchos::RCP<const PeridigmNS::NeighborhoodData> globalContactNeighborhoodData)
{
double* globalBlockIdsPtr;
globalBlockIds->ExtractView(&globalBlockIdsPtr);
// Create a list of all the on-processor elements that are part of this block
vector<int> IDs;
IDs.reserve(globalOverlapScalarPointMap->NumMyElements()); // upper bound
vector<int> bondIDs;
bondIDs.reserve(globalOverlapScalarPointMap->NumMyElements());
vector<int> bondElementSize;
bondElementSize.reserve(globalOwnedScalarPointMap->NumMyElements());
for(int iLID=0 ; iLID<globalOwnedScalarPointMap->NumMyElements() ; ++iLID){
if(globalBlockIdsPtr[iLID] == blockID) {
int globalID = globalOwnedScalarPointMap->GID(iLID);
IDs.push_back(globalID);
}
}
// Record the size of these elements in the bond map
// Note that if an element has no bonds, it has no entry in the bondMap
// So, the bond map and the scalar map can have a different number of entries (different local IDs)
for(int iLID=0 ; iLID<globalOwnedScalarBondMap->NumMyElements() ; ++iLID){
int globalID = globalOwnedScalarBondMap->GID(iLID);
int localID = globalOwnedScalarPointMap->LID(globalID);
if(globalBlockIdsPtr[localID] == blockID){
bondIDs.push_back(globalID);
bondElementSize.push_back(globalOwnedScalarBondMap->ElementSize(iLID));
}
}
// Create the owned scalar point map, the owned vector point map, and the owned scalar bond map
int numGlobalElements = -1;
int numMyElements = IDs.size();
int* myGlobalElements = 0;
if(numMyElements > 0)
myGlobalElements = &IDs.at(0);
int elementSize = 1;
int indexBase = 0;
ownedScalarPointMap =
Teuchos::rcp(new Epetra_BlockMap(numGlobalElements, numMyElements, myGlobalElements, elementSize, indexBase, globalOwnedScalarPointMap->Comm()));
elementSize = 3;
ownedVectorPointMap =
Teuchos::rcp(new Epetra_BlockMap(numGlobalElements, numMyElements, myGlobalElements, elementSize, indexBase, globalOwnedScalarPointMap->Comm()));
numMyElements = bondElementSize.size();
myGlobalElements = 0;
int* elementSizeList = 0;
if(numMyElements > 0){
myGlobalElements = &bondIDs.at(0);
elementSizeList = &bondElementSize.at(0);
}
ownedScalarBondMap =
Teuchos::rcp(new Epetra_BlockMap(numGlobalElements, numMyElements, myGlobalElements, elementSizeList, indexBase, globalOwnedScalarPointMap->Comm()));
// Create a list of nodes that need to be ghosted (both across material boundaries and across processor boundaries)
set<int> ghosts;
// Check the neighborhood list for things that need to be ghosted
int* const globalNeighborhoodList = globalNeighborhoodData->NeighborhoodList();
int globalNeighborhoodListIndex = 0;
for(int iLID=0 ; iLID<globalNeighborhoodData->NumOwnedPoints() ; ++iLID){
int numNeighbors = globalNeighborhoodList[globalNeighborhoodListIndex++];
if(globalBlockIdsPtr[iLID] == blockID) {
for(int i=0 ; i<numNeighbors ; ++i){
int neighborGlobalID = globalOverlapScalarPointMap->GID( globalNeighborhoodList[globalNeighborhoodListIndex + i] );
ghosts.insert(neighborGlobalID);
}
}
globalNeighborhoodListIndex += numNeighbors;
}
// Check the contact neighborhood list for things that need to be ghosted
if(!globalContactNeighborhoodData.is_null()){
int* const globalContactNeighborhoodList = globalContactNeighborhoodData->NeighborhoodList();
int globalContactNeighborhoodListIndex = 0;
for(int iLID=0 ; iLID<globalContactNeighborhoodData->NumOwnedPoints() ; ++iLID){
int numNeighbors = globalContactNeighborhoodList[globalContactNeighborhoodListIndex++];
if(globalBlockIdsPtr[iLID] == blockID) {
for(int i=0 ; i<numNeighbors ; ++i){
int neighborGlobalID = globalOverlapScalarPointMap->GID( globalContactNeighborhoodList[globalContactNeighborhoodListIndex + i] );
ghosts.insert(neighborGlobalID);
}
}
globalContactNeighborhoodListIndex += numNeighbors;
}
}
// Remove entries from ghosts that are already in IDs
for(unsigned int i=0 ; i<IDs.size() ; ++i)
ghosts.erase(IDs[i]);
// Copy IDs, this is the owned global ID list
vector<int> ownedIDs(IDs.begin(), IDs.end());
// Append ghosts to IDs
// This creates the overlap global ID list
for(set<int>::iterator it=ghosts.begin() ; it!=ghosts.end() ; ++it)
IDs.push_back(*it);
// Create the overlap scalar point map and the overlap vector point map
numMyElements = IDs.size();
myGlobalElements = 0;
if(numMyElements > 0)
myGlobalElements = &IDs.at(0);
elementSize = 1;
overlapScalarPointMap =
Teuchos::rcp(new Epetra_BlockMap(numGlobalElements, numMyElements, myGlobalElements, elementSize, indexBase, globalOwnedScalarPointMap->Comm()));
elementSize = 3;
overlapVectorPointMap =
Teuchos::rcp(new Epetra_BlockMap(numGlobalElements, numMyElements, myGlobalElements, elementSize, indexBase, globalOwnedScalarPointMap->Comm()));
// Invalidate the importers
oneDimensionalImporter = Teuchos::RCP<Epetra_Import>();
threeDimensionalImporter = Teuchos::RCP<Epetra_Import>();
}
AmesosBTFGlobal_LinearProblem::NewTypeRef
AmesosBTFGlobal_LinearProblem::
operator()( OriginalTypeRef orig )
{
origObj_ = &orig;
// Extract the matrix and vectors from the linear problem
OldRHS_ = Teuchos::rcp( orig.GetRHS(), false );
OldLHS_ = Teuchos::rcp( orig.GetLHS(), false );
OldMatrix_ = Teuchos::rcp( dynamic_cast<Epetra_CrsMatrix *>( orig.GetMatrix() ), false );
int nGlobal = OldMatrix_->NumGlobalRows();
int n = OldMatrix_->NumMyRows();
// Check if the matrix is on one processor.
int myMatProc = -1, matProc = -1;
int myPID = OldMatrix_->Comm().MyPID();
int numProcs = OldMatrix_->Comm().NumProc();
const Epetra_BlockMap& oldRowMap = OldMatrix_->RowMap();
// Get some information about the parallel distribution.
int maxMyRows = 0;
std::vector<int> numGlobalElem( numProcs );
OldMatrix_->Comm().GatherAll(&n, &numGlobalElem[0], 1);
OldMatrix_->Comm().MaxAll(&n, &maxMyRows, 1);
for (int proc=0; proc<numProcs; proc++)
{
if (OldMatrix_->NumGlobalNonzeros() == OldMatrix_->NumMyNonzeros())
myMatProc = myPID;
}
OldMatrix_->Comm().MaxAll( &myMatProc, &matProc, 1 );
Teuchos::RCP<Epetra_CrsMatrix> serialMatrix;
Teuchos::RCP<Epetra_Map> serialMap;
if( oldRowMap.DistributedGlobal() && matProc == -1)
{
// The matrix is distributed and needs to be moved to processor zero.
// Set the zero processor as the master.
matProc = 0;
serialMap = Teuchos::rcp( new Epetra_Map( Epetra_Util::Create_Root_Map( OldMatrix_->RowMap(), matProc ) ) );
Epetra_Import serialImporter( *serialMap, OldMatrix_->RowMap() );
serialMatrix = Teuchos::rcp( new Epetra_CrsMatrix( Copy, *serialMap, 0 ) );
serialMatrix->Import( *OldMatrix_, serialImporter, Insert );
serialMatrix->FillComplete();
}
else {
// The old matrix has already been moved to one processor (matProc).
serialMatrix = OldMatrix_;
}
if( debug_ )
{
cout << "Original (serial) Matrix:\n";
cout << *serialMatrix << endl;
}
// Obtain the current row and column orderings
std::vector<int> origGlobalRows(nGlobal), origGlobalCols(nGlobal);
serialMatrix->RowMap().MyGlobalElements( &origGlobalRows[0] );
serialMatrix->ColMap().MyGlobalElements( &origGlobalCols[0] );
// Perform reindexing on the full serial matrix (needed for BTF).
Epetra_Map reIdxMap( serialMatrix->RowMap().NumGlobalElements(), serialMatrix->RowMap().NumMyElements(), 0, serialMatrix->Comm() );
Teuchos::RCP<EpetraExt::ViewTransform<Epetra_CrsMatrix> > reIdxTrans =
Teuchos::rcp( new EpetraExt::CrsMatrix_Reindex( reIdxMap ) );
Epetra_CrsMatrix newSerialMatrix = (*reIdxTrans)( *serialMatrix );
reIdxTrans->fwd();
// Compute and apply BTF to the serial CrsMatrix and has been filtered by the threshold
EpetraExt::AmesosBTF_CrsMatrix BTFTrans( threshold_, upperTri_, verbose_, debug_ );
Epetra_CrsMatrix newSerialMatrixBTF = BTFTrans( newSerialMatrix );
rowPerm_ = BTFTrans.RowPerm();
colPerm_ = BTFTrans.ColPerm();
blockPtr_ = BTFTrans.BlockPtr();
numBlocks_ = BTFTrans.NumBlocks();
if (myPID == matProc && verbose_) {
bool isSym = true;
for (int i=0; i<nGlobal; ++i) {
if (rowPerm_[i] != colPerm_[i]) {
isSym = false;
break;
}
}
std::cout << "The BTF permutation symmetry (0=false,1=true) is : " << isSym << std::endl;
}
// Compute the permutation w.r.t. the original row and column GIDs.
std::vector<int> origGlobalRowsPerm(nGlobal), origGlobalColsPerm(nGlobal);
if (myPID == matProc) {
for (int i=0; i<nGlobal; ++i) {
origGlobalRowsPerm[i] = origGlobalRows[ rowPerm_[i] ];
origGlobalColsPerm[i] = origGlobalCols[ colPerm_[i] ];
}
}
OldMatrix_->Comm().Broadcast( &origGlobalRowsPerm[0], nGlobal, matProc );
OldMatrix_->Comm().Broadcast( &origGlobalColsPerm[0], nGlobal, matProc );
// Generate the full serial matrix that imports according to the previously computed BTF.
Epetra_CrsMatrix newSerialMatrixT( Copy, newSerialMatrixBTF.RowMap(), 0 );
newSerialMatrixT.Import( newSerialMatrix, *(BTFTrans.Importer()), Insert );
newSerialMatrixT.FillComplete();
if( debug_ )
{
cout << "Original (serial) Matrix permuted via BTF:\n";
cout << newSerialMatrixT << endl;
}
// Perform reindexing on the full serial matrix (needed for balancing).
Epetra_Map reIdxMap2( newSerialMatrixT.RowMap().NumGlobalElements(), newSerialMatrixT.RowMap().NumMyElements(), 0, newSerialMatrixT.Comm() );
Teuchos::RCP<EpetraExt::ViewTransform<Epetra_CrsMatrix> > reIdxTrans2 =
Teuchos::rcp( new EpetraExt::CrsMatrix_Reindex( reIdxMap2 ) );
Epetra_CrsMatrix tNewSerialMatrixT = (*reIdxTrans2)( newSerialMatrixT );
reIdxTrans2->fwd();
Teuchos::RCP<Epetra_Map> balancedMap;
if (balance_ == "linear") {
// Distribute block somewhat evenly across processors
std::vector<int> rowDist(numProcs+1,0);
int balRows = nGlobal / numProcs + 1;
int numRows = balRows, currProc = 1;
for ( int i=0; i<numBlocks_ || currProc < numProcs; ++i ) {
if (blockPtr_[i] > numRows) {
rowDist[currProc++] = blockPtr_[i-1];
numRows = blockPtr_[i-1] + balRows;
}
}
rowDist[numProcs] = nGlobal;
// Create new Map based on this linear distribution.
int numMyBalancedRows = rowDist[myPID+1]-rowDist[myPID];
NewRowMap_ = Teuchos::rcp( new Epetra_Map( nGlobal, numMyBalancedRows, &origGlobalRowsPerm[ rowDist[myPID] ], 0, OldMatrix_->Comm() ) );
// Right now we do not explicitly build the column map and assume the BTF permutation is symmetric!
//NewColMap_ = Teuchos::rcp( new Epetra_Map( nGlobal, nGlobal, &colPerm_[0], 0, OldMatrix_->Comm() ) );
if ( verbose_ )
std::cout << "Processor " << myPID << " has " << numMyBalancedRows << " rows." << std::endl;
//balancedMap = Teuchos::rcp( new Epetra_Map( nGlobal, numMyBalancedRows, 0, serialMatrix->Comm() ) );
}
else if (balance_ == "isorropia") {
// Compute block adjacency graph for partitioning.
std::vector<double> weight;
Teuchos::RCP<Epetra_CrsGraph> blkGraph;
EpetraExt::BlockAdjacencyGraph adjGraph;
blkGraph = adjGraph.compute( const_cast<Epetra_CrsGraph&>(tNewSerialMatrixT.Graph()),
numBlocks_, blockPtr_, weight, verbose_);
Epetra_Vector rowWeights( View, blkGraph->Map(), &weight[0] );
// Call Isorropia to rebalance this graph.
Teuchos::RCP<Epetra_CrsGraph> balancedGraph =
Isorropia::Epetra::create_balanced_copy( *blkGraph, rowWeights );
int myNumBlkRows = balancedGraph->NumMyRows();
//std::vector<int> myGlobalElements(nGlobal);
std::vector<int> newRangeElements(nGlobal), newDomainElements(nGlobal);
int grid = 0, myElements = 0;
for (int i=0; i<myNumBlkRows; ++i) {
grid = balancedGraph->GRID( i );
for (int j=blockPtr_[grid]; j<blockPtr_[grid+1]; ++j) {
newRangeElements[myElements++] = origGlobalRowsPerm[j];
//myGlobalElements[myElements++] = j;
}
}
NewRowMap_ = Teuchos::rcp( new Epetra_Map( nGlobal, myElements, &newRangeElements[0], 0, OldMatrix_->Comm() ) );
// Right now we do not explicitly build the column map and assume the BTF permutation is symmetric!
//NewColMap_ = Teuchos::rcp( new Epetra_Map( nGlobal, nGlobal, &colPerm_[0], 0, OldMatrix_->Comm() ) );
//balancedMap = Teuchos::rcp( new Epetra_Map( nGlobal, myElements, &myGlobalElements[0], 0, serialMatrix->Comm() ) );
if ( verbose_ )
std::cout << "Processor " << myPID << " has " << myElements << " rows." << std::endl;
}
// Use New Domain and Range Maps to Generate Importer
//for now, assume they start out as identical
Epetra_Map OldRowMap = OldMatrix_->RowMap();
Epetra_Map OldColMap = OldMatrix_->ColMap();
if( debug_ )
{
cout << "New Row Map\n";
cout << *NewRowMap_ << endl;
//cout << "New Col Map\n";
//cout << *NewColMap_ << endl;
}
// Generate New Graph
// NOTE: Right now we are creating the graph, assuming that the permutation is symmetric!
// NewGraph_ = Teuchos::rcp( new Epetra_CrsGraph( Copy, *NewRowMap_, *NewColMap_, 0 ) );
NewGraph_ = Teuchos::rcp( new Epetra_CrsGraph( Copy, *NewRowMap_, 0 ) );
Importer_ = Teuchos::rcp( new Epetra_Import( *NewRowMap_, OldRowMap ) );
Importer2_ = Teuchos::rcp( new Epetra_Import( OldRowMap, *NewRowMap_ ) );
NewGraph_->Import( OldMatrix_->Graph(), *Importer_, Insert );
NewGraph_->FillComplete();
if( debug_ )
{
cout << "NewGraph\n";
cout << *NewGraph_;
}
// Create new linear problem and import information from old linear problem
NewMatrix_ = Teuchos::rcp( new Epetra_CrsMatrix( Copy, *NewGraph_ ) );
NewMatrix_->Import( *OldMatrix_, *Importer_, Insert );
NewMatrix_->FillComplete();
NewLHS_ = Teuchos::rcp( new Epetra_MultiVector( *NewRowMap_, OldLHS_->NumVectors() ) );
NewLHS_->Import( *OldLHS_, *Importer_, Insert );
NewRHS_ = Teuchos::rcp( new Epetra_MultiVector( *NewRowMap_, OldRHS_->NumVectors() ) );
NewRHS_->Import( *OldRHS_, *Importer_, Insert );
if( debug_ )
{
cout << "New Matrix\n";
cout << *NewMatrix_ << endl;
}
newObj_ = new Epetra_LinearProblem( &*NewMatrix_, &*NewLHS_, &*NewRHS_ );
return *newObj_;
}
