int powerMethodTests(Epetra_RowMatrix & A, Epetra_RowMatrix & JadA, Epetra_Map & Map,
Epetra_Vector & q, Epetra_Vector & z, Epetra_Vector & resid, bool verbose) {
// variable needed for iteration
double lambda = 0.0;
// int niters = 10000;
int niters = 300;
double tolerance = 1.0e-2;
int ierr = 0;
/////////////////////////////////////////////////////////////////////////////////////////////////
// Iterate
Epetra_Time timer(Map.Comm());
double startTime = timer.ElapsedTime();
EPETRA_TEST_ERR(power_method(false, A, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
double elapsed_time = timer.ElapsedTime() - startTime;
double total_flops = q.Flops();
double MFLOPs = total_flops/elapsed_time/1000000.0;
double lambdaref = lambda;
double flopsref = total_flops;
if (verbose)
cout << "\n\nTotal MFLOPs for reference first solve = " << MFLOPs << endl
<< "Total FLOPS = " <<total_flops <<endl<<endl;
lambda = 0.0;
startTime = timer.ElapsedTime();
EPETRA_TEST_ERR(power_method(false, JadA, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
elapsed_time = timer.ElapsedTime() - startTime;
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose)
cout << "\n\nTotal MFLOPs for candidate first solve = " << MFLOPs << endl
<< "Total FLOPS = " <<total_flops <<endl<<endl;
EPETRA_TEST_ERR(checkValues(lambda,lambdaref," No-transpose Power Method result", verbose),ierr);
EPETRA_TEST_ERR(checkValues(total_flops,flopsref," No-transpose Power Method flop count", verbose),ierr);
/////////////////////////////////////////////////////////////////////////////////////////////////
// Solve transpose problem
if (verbose) cout << "\n\nUsing transpose of matrix and solving again (should give same result).\n\n"
<< endl;
// Iterate
lambda = 0.0;
startTime = timer.ElapsedTime();
EPETRA_TEST_ERR(power_method(true, A, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
elapsed_time = timer.ElapsedTime() - startTime;
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
lambdaref = lambda;
flopsref = total_flops;
if (verbose)
cout << "\n\nTotal MFLOPs for reference transpose solve = " << MFLOPs << endl
<< "Total FLOPS = " <<total_flops <<endl<<endl;
lambda = 0.0;
startTime = timer.ElapsedTime();
EPETRA_TEST_ERR(power_method(true, JadA, q, z, resid, &lambda, niters, tolerance, verbose),ierr);
elapsed_time = timer.ElapsedTime() - startTime;
total_flops = q.Flops();
MFLOPs = total_flops/elapsed_time/1000000.0;
if (verbose)
cout << "\n\nTotal MFLOPs for candidate transpose solve = " << MFLOPs << endl
<< "Total FLOPS = " <<total_flops <<endl<<endl;
EPETRA_TEST_ERR(checkValues(lambda,lambdaref,"Transpose Power Method result", verbose),ierr);
EPETRA_TEST_ERR(checkValues(total_flops,flopsref,"Transpose Power Method flop count", verbose),ierr);
EPETRA_TEST_ERR(check(A, JadA, verbose),ierr);
return(0);
}
int runTests(Epetra_Map & map, Epetra_CrsMatrix & A, Epetra_Vector & x, Epetra_Vector & b, Epetra_Vector & xexact, bool verbose) {
int ierr = 0;
// Create MultiVectors and put x, b, xexact in both columns of X, B, and Xexact, respectively.
Epetra_MultiVector X( map, 2, false );
Epetra_MultiVector B( map, 2, false );
Epetra_MultiVector Xexact( map, 2, false );
for (int i=0; i<X.NumVectors(); ++i) {
*X(i) = x;
*B(i) = b;
*Xexact(i) = xexact;
}
double residual;
std::vector<double> residualmv(2);
residual = A.NormInf(); double rAInf = residual;
if (verbose) std::cout << "Inf Norm of A = " << residual << std::endl;
residual = A.NormOne(); double rAOne = residual;
if (verbose) std::cout << "One Norm of A = " << residual << std::endl;
xexact.Norm2(&residual); double rxx = residual;
Xexact.Norm2(&residualmv[0]); std::vector<double> rXX( residualmv );
if (verbose) std::cout << "Norm of xexact = " << residual << std::endl;
if (verbose) std::cout << "Norm of Xexact = (" << residualmv[0] << ", " <<residualmv[1] <<")"<< std::endl;
Epetra_Vector tmp1(map);
Epetra_MultiVector tmp1mv(map,2,false);
A.Multiply(false, xexact, tmp1);
A.Multiply(false, Xexact, tmp1mv);
tmp1.Norm2(&residual); double rAx = residual;
tmp1mv.Norm2(&residualmv[0]); std::vector<double> rAX( residualmv );
if (verbose) std::cout << "Norm of Ax = " << residual << std::endl;
if (verbose) std::cout << "Norm of AX = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
b.Norm2(&residual); double rb = residual;
B.Norm2(&residualmv[0]); std::vector<double> rB( residualmv );
if (verbose) std::cout << "Norm of b (should equal norm of Ax) = " << residual << std::endl;
if (verbose) std::cout << "Norm of B (should equal norm of AX) = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
tmp1.Update(1.0, b, -1.0);
tmp1mv.Update(1.0, B, -1.0);
tmp1.Norm2(&residual);
tmp1mv.Norm2(&residualmv[0]);
if (verbose) std::cout << "Norm of difference between compute Ax and Ax from file = " << residual << std::endl;
if (verbose) std::cout << "Norm of difference between compute AX and AX from file = (" << residualmv[0] << ", " << residualmv[1] <<")"<< std::endl;
map.Comm().Barrier();
EPETRA_CHK_ERR(EpetraExt::BlockMapToMatrixMarketFile("Test_map.mm", map, "Official EpetraExt test map",
"This is the official EpetraExt test map generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::RowMatrixToMatrixMarketFile("Test_A.mm", A, "Official EpetraExt test matrix",
"This is the official EpetraExt test matrix generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_x.mm", x, "Official EpetraExt test initial guess",
"This is the official EpetraExt test initial guess generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvX.mm", X, "Official EpetraExt test initial guess",
"This is the official EpetraExt test initial guess generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_xexact.mm", xexact, "Official EpetraExt test exact solution",
"This is the official EpetraExt test exact solution generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvXexact.mm", Xexact, "Official EpetraExt test exact solution",
"This is the official EpetraExt test exact solution generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::VectorToMatrixMarketFile("Test_b.mm", b, "Official EpetraExt test right hand side",
"This is the official EpetraExt test right hand side generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatrixMarketFile("Test_mvB.mm", B, "Official EpetraExt test right hand side",
"This is the official EpetraExt test right hand side generated by the EpetraExt regression tests"));
EPETRA_CHK_ERR(EpetraExt::MultiVectorToMatlabFile("Test_mvB.mat", B));
EPETRA_CHK_ERR(EpetraExt::RowMatrixToMatlabFile("Test_A.dat", A));
Epetra_Map * map1;
Epetra_CrsMatrix * A1;
Epetra_CrsMatrix * A2;
Epetra_CrsMatrix * A3;
Epetra_Vector * x1;
Epetra_Vector * b1;
Epetra_Vector * xexact1;
Epetra_MultiVector * X1;
Epetra_MultiVector * B1;
Epetra_MultiVector * Xexact1;
EpetraExt::MatrixMarketFileToMap("Test_map.mm", map.Comm(), map1);
if (map.SameAs(*map1)) {
if (verbose) std::cout << "Maps are equal. In/Out works." << std::endl;
}
else {
if (verbose) std::cout << "Maps are not equal. In/Out fails." << std::endl;
ierr += 1;
}
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToCrsMatrix("Test_A.mm", *map1, A1));
// If map is zero-based, then we can compare to the convenient reading versions
if (map1->IndexBase()==0) EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToCrsMatrix("Test_A.mm", map1->Comm(), A2));
if (map1->IndexBase()==0) EPETRA_CHK_ERR(EpetraExt::MatlabFileToCrsMatrix("Test_A.dat", map1->Comm(), A3));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_x.mm", *map1, x1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_xexact.mm", *map1, xexact1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToVector("Test_b.mm", *map1, b1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvX.mm", *map1, X1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvXexact.mm", *map1, Xexact1));
EPETRA_CHK_ERR(EpetraExt::MatrixMarketFileToMultiVector("Test_mvB.mm", *map1, B1));
residual = A1->NormInf(); double rA1Inf = residual;
if (verbose) std::cout << "Inf Norm of A1 = " << residual << std::endl;
ierr += checkValues(rA1Inf,rAInf,"Inf Norm of A", verbose);
residual = A1->NormOne(); double rA1One = residual;
if (verbose) std::cout << "One Norm of A1 = " << residual << std::endl;
ierr += checkValues(rA1One,rAOne,"One Norm of A", verbose);
xexact1->Norm2(&residual); double rxx1 = residual;
if (verbose) std::cout << "Norm of xexact1 = " << residual << std::endl;
ierr += checkValues(rxx1,rxx,"Norm of xexact", verbose);
Xexact1->Norm2(&residualmv[0]); std::vector<double> rXX1(residualmv);
if (verbose) std::cout << "Norm of Xexact1 = (" << residualmv[0] <<", " <<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rXX1[0],rXX[0],"Norm of Xexact", verbose);
ierr += checkValues(rXX1[1],rXX[1],"Norm of Xexact", verbose);
Epetra_Vector tmp11(*map1);
A1->Multiply(false, *xexact1, tmp11);
Epetra_MultiVector tmp11mv(*map1,2,false);
A1->Multiply(false, *Xexact1, tmp11mv);
tmp11.Norm2(&residual); double rAx1 = residual;
if (verbose) std::cout << "Norm of A1*x1 = " << residual << std::endl;
ierr += checkValues(rAx1,rAx,"Norm of A1*x", verbose);
tmp11mv.Norm2(&residualmv[0]); std::vector<double> rAX1(residualmv);
if (verbose) std::cout << "Norm of A1*X1 = (" << residualmv[0] <<", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rAX1[0],rAX[0],"Norm of A1*X", verbose);
ierr += checkValues(rAX1[1],rAX[1],"Norm of A1*X", verbose);
if (map1->IndexBase()==0) {
Epetra_Vector tmp12(*map1);
A2->Multiply(false, *xexact1, tmp12);
tmp12.Norm2(&residual); double rAx2 = residual;
if (verbose) std::cout << "Norm of A2*x1 = " << residual << std::endl;
ierr += checkValues(rAx2,rAx,"Norm of A2*x", verbose);
Epetra_Vector tmp13(*map1);
A3->Multiply(false, *xexact1, tmp13);
tmp13.Norm2(&residual); double rAx3 = residual;
if (verbose) std::cout << "Norm of A3*x1 = " << residual << std::endl;
ierr += checkValues(rAx3,rAx,"Norm of A3*x", verbose);
}
b1->Norm2(&residual); double rb1 = residual;
if (verbose) std::cout << "Norm of b1 (should equal norm of Ax) = " << residual << std::endl;
ierr += checkValues(rb1,rb,"Norm of b", verbose);
B1->Norm2(&residualmv[0]); std::vector<double> rB1(residualmv);
if (verbose) std::cout << "Norm of B1 (should equal norm of AX) = (" << residualmv[0] <<", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(rB1[0],rB[0],"Norm of B", verbose);
ierr += checkValues(rB1[1],rB[1],"Norm of B", verbose);
tmp11.Update(1.0, *b1, -1.0);
tmp11.Norm2(&residual);
if (verbose) std::cout << "Norm of difference between computed A1x1 and A1x1 from file = " << residual << std::endl;
ierr += checkValues(residual,0.0,"Norm of difference between computed A1x1 and A1x1 from file", verbose);
tmp11mv.Update(1.0, *B1, -1.0);
tmp11mv.Norm2(&residualmv[0]);
if (verbose) std::cout << "Norm of difference between computed A1X1 and A1X1 from file = (" << residualmv[0] << ", "<<residualmv[1]<<")"<< std::endl;
ierr += checkValues(residualmv[0],0.0,"Norm of difference between computed A1X1 and A1X1 from file", verbose);
ierr += checkValues(residualmv[1],0.0,"Norm of difference between computed A1X1 and A1X1 from file", verbose);
if (map1->IndexBase()==0) {delete A2; delete A3;}
delete A1;
delete x1;
delete b1;
delete xexact1;
delete X1;
delete B1;
delete Xexact1;
delete map1;
return(ierr);
}
int DoCopyRowMatrix(mxArray* matlabA, int& valueCount, const Epetra_RowMatrix& A) {
//cout << "doing DoCopyRowMatrix\n";
int ierr = 0;
int numRows = A.NumGlobalRows();
//cout << "numRows: " << numRows << "\n";
Epetra_Map rowMap = A.RowMatrixRowMap();
Epetra_Map colMap = A.RowMatrixColMap();
int minAllGID = rowMap.MinAllGID();
const Epetra_Comm & comm = rowMap.Comm();
//cout << "did global setup\n";
if (comm.MyPID()!=0) {
if (A.NumMyRows()!=0) ierr = -1;
if (A.NumMyCols()!=0) ierr = -1;
}
else {
// declare and get initial values of all matlabA pointers
double* matlabAvaluesPtr = mxGetPr(matlabA);
int* matlabAcolumnIndicesPtr = mxGetJc(matlabA);
int* matlabArowIndicesPtr = mxGetIr(matlabA);
// set all matlabA pointers to the proper offset
matlabAvaluesPtr += valueCount;
matlabArowIndicesPtr += valueCount;
if (numRows!=A.NumMyRows()) ierr = -1;
Epetra_SerialDenseVector values(A.MaxNumEntries());
Epetra_IntSerialDenseVector indices(A.MaxNumEntries());
//cout << "did proc0 setup\n";
for (int i=0; i<numRows; i++) {
//cout << "extracting a row\n";
int I = rowMap.GID(i);
int numEntries = 0;
if (A.ExtractMyRowCopy(i, values.Length(), numEntries,
values.Values(), indices.Values())) return(-1);
matlabAcolumnIndicesPtr[I - minAllGID] = valueCount; // set the starting index of column I
double* serialValuesPtr = values.Values();
for (int j=0; j<numEntries; j++) {
int J = colMap.GID(indices[j]);
*matlabAvaluesPtr = *serialValuesPtr++;
*matlabArowIndicesPtr = J;
// increment matlabA pointers
matlabAvaluesPtr++;
matlabArowIndicesPtr++;
valueCount++;
}
}
//cout << "proc0 row extraction for this chunck is done\n";
}
/*
if (comm.MyPID() == 0) {
cout << "printing matlabA pointers\n";
double* matlabAvaluesPtr = mxGetPr(matlabA);
int* matlabAcolumnIndicesPtr = mxGetJc(matlabA);
int* matlabArowIndicesPtr = mxGetIr(matlabA);
for(int i=0; i < numRows; i++) {
for(int j=0; j < A.MaxNumEntries(); j++) {
cout << "*matlabAvaluesPtr: " << *matlabAvaluesPtr++ << " *matlabAcolumnIndicesPtr: " << *matlabAcolumnIndicesPtr++ << " *matlabArowIndicesPtr" << *matlabArowIndicesPtr++ << "\n";
}
}
cout << "done printing matlabA pointers\n";
}
*/
int ierrGlobal;
comm.MinAll(&ierr, &ierrGlobal, 1); // If any processor has -1, all return -1
return(ierrGlobal);
}
int CopyRowMatrix(mxArray* matlabA, const Epetra_RowMatrix& A) {
int valueCount = 0;
//int* valueCount = &temp;
Epetra_Map map = A.RowMatrixRowMap();
const Epetra_Comm & comm = map.Comm();
int numProc = comm.NumProc();
if (numProc==1)
DoCopyRowMatrix(matlabA, valueCount, A);
else {
int numRows = map.NumMyElements();
//cout << "creating allGidsMap\n";
Epetra_Map allGidsMap(-1, numRows, 0,comm);
//cout << "done creating allGidsMap\n";
Epetra_IntVector allGids(allGidsMap);
for (int i=0; i<numRows; i++) allGids[i] = map.GID(i);
// Now construct a RowMatrix on PE 0 by strip-mining the rows of the input matrix A.
int numChunks = numProc;
int stripSize = allGids.GlobalLength()/numChunks;
int remainder = allGids.GlobalLength()%numChunks;
int curStart = 0;
int curStripSize = 0;
Epetra_IntSerialDenseVector importGidList;
int numImportGids = 0;
if (comm.MyPID()==0)
importGidList.Size(stripSize+1); // Set size of vector to max needed
for (int i=0; i<numChunks; i++) {
if (comm.MyPID()==0) { // Only PE 0 does this part
curStripSize = stripSize;
if (i<remainder) curStripSize++; // handle leftovers
for (int j=0; j<curStripSize; j++) importGidList[j] = j + curStart;
curStart += curStripSize;
}
// The following import map will be non-trivial only on PE 0.
//cout << "creating importGidMap\n";
Epetra_Map importGidMap(-1, curStripSize, importGidList.Values(), 0, comm);
//cout << "done creating importGidMap\n";
Epetra_Import gidImporter(importGidMap, allGidsMap);
Epetra_IntVector importGids(importGidMap);
if (importGids.Import(allGids, gidImporter, Insert)) return(-1);
// importGids now has a list of GIDs for the current strip of matrix rows.
// Use these values to build another importer that will get rows of the matrix.
// The following import map will be non-trivial only on PE 0.
//cout << "creating importMap\n";
//cout << "A.RowMatrixRowMap().MinAllGID: " << A.RowMatrixRowMap().MinAllGID() << "\n";
Epetra_Map importMap(-1, importGids.MyLength(), importGids.Values(), A.RowMatrixRowMap().MinAllGID(), comm);
//cout << "done creating importMap\n";
Epetra_Import importer(importMap, map);
Epetra_CrsMatrix importA(Copy, importMap, 0);
if (importA.Import(A, importer, Insert)) return(-1);
if (importA.FillComplete()) return(-1);
// Finally we are ready to write this strip of the matrix to ostream
if (DoCopyRowMatrix(matlabA, valueCount, importA)) return(-1);
}
}
if (A.RowMatrixRowMap().Comm().MyPID() == 0) {
// set max cap
int* matlabAcolumnIndicesPtr = mxGetJc(matlabA);
matlabAcolumnIndicesPtr[A.NumGlobalRows()] = valueCount;
}
return(0);
}
int checkmap(Epetra_Map & Map, int NumGlobalElements, int NumMyElements,
int *MyGlobalElements, int IndexBase, Epetra_Comm& Comm,
bool DistributedGlobal)
{
int i, ierr=0, forierr = 0;
EPETRA_TEST_ERR(!Map.ConstantElementSize(),ierr);
EPETRA_TEST_ERR(DistributedGlobal!=Map.DistributedGlobal(),ierr);
EPETRA_TEST_ERR(Map.ElementSize()!=1,ierr);
int *MyElementSizeList = new int[NumMyElements];
EPETRA_TEST_ERR(Map.ElementSizeList(MyElementSizeList)!=0,ierr);
forierr = 0;
for (i=0; i<NumMyElements; i++) forierr += MyElementSizeList[i]!=1;
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyElementSizeList;
const Epetra_Comm & Comm1 = Map.Comm();
EPETRA_TEST_ERR(Comm1.NumProc()!=Comm.NumProc(),ierr);
EPETRA_TEST_ERR(Comm1.MyPID()!=Comm.MyPID(),ierr);
EPETRA_TEST_ERR(Map.IndexBase()!=IndexBase,ierr);
EPETRA_TEST_ERR(!Map.LinearMap() && MyGlobalElements==0,ierr);
EPETRA_TEST_ERR(Map.LinearMap() && MyGlobalElements!=0,ierr);
EPETRA_TEST_ERR(Map.MaxAllGID()!=NumGlobalElements-1+IndexBase,ierr);
EPETRA_TEST_ERR(Map.MaxElementSize()!=1,ierr);
int MaxLID = Map.MaxLID();
EPETRA_TEST_ERR(MaxLID!=NumMyElements-1,ierr);
int MaxMyGID = (Comm.MyPID()+1)*NumMyElements-1+IndexBase;
if (Comm.MyPID()>2) MaxMyGID+=3;
if (!DistributedGlobal) MaxMyGID = NumMyElements-1+IndexBase;
EPETRA_TEST_ERR(Map.MaxMyGID()!=MaxMyGID,ierr);
EPETRA_TEST_ERR(Map.MinAllGID()!=IndexBase,ierr);
EPETRA_TEST_ERR(Map.MinElementSize()!=1,ierr);
EPETRA_TEST_ERR(Map.MinLID()!=0,ierr);
int MinMyGID = Comm.MyPID()*NumMyElements+IndexBase;
if (Comm.MyPID()>2) MinMyGID+=3;
if (!DistributedGlobal) MinMyGID = 0;
EPETRA_TEST_ERR(Map.MinMyGID()!=MinMyGID,ierr);
int * MyGlobalElements1 = new int[NumMyElements];
EPETRA_TEST_ERR(Map.MyGlobalElements(MyGlobalElements1)!=0,ierr);
forierr = 0;
if (MyGlobalElements==0)
{
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements1[i]!=MinMyGID+i;
EPETRA_TEST_ERR(forierr,ierr);
}
else {
for (i=0; i<NumMyElements; i++)
forierr += MyGlobalElements[i]!=MyGlobalElements1[i];
EPETRA_TEST_ERR(forierr,ierr);
}
EPETRA_TEST_ERR(Map.NumGlobalElements()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumGlobalPoints()!=NumGlobalElements,ierr);
EPETRA_TEST_ERR(Map.NumMyElements()!=NumMyElements,ierr);
EPETRA_TEST_ERR(Map.NumMyPoints()!=NumMyElements,ierr);
int MaxMyGID2 = Map.GID(Map.LID(MaxMyGID));
EPETRA_TEST_ERR(MaxMyGID2 != MaxMyGID,ierr);
int MaxLID2 = Map.LID(Map.GID(MaxLID));
EPETRA_TEST_ERR(MaxLID2 != MaxLID,ierr);
EPETRA_TEST_ERR(Map.GID(MaxLID+1) != IndexBase-1,ierr);// MaxLID+1 doesn't exist
EPETRA_TEST_ERR(Map.LID(MaxMyGID+1) != -1,ierr);// MaxMyGID+1 doesn't exist or is on a different processor
EPETRA_TEST_ERR(!Map.MyGID(MaxMyGID),ierr);
EPETRA_TEST_ERR(Map.MyGID(MaxMyGID+1),ierr);
EPETRA_TEST_ERR(!Map.MyLID(MaxLID),ierr);
EPETRA_TEST_ERR(Map.MyLID(MaxLID+1),ierr);
EPETRA_TEST_ERR(!Map.MyGID(Map.GID(MaxLID)),ierr);
EPETRA_TEST_ERR(Map.MyGID(Map.GID(MaxLID+1)),ierr);
EPETRA_TEST_ERR(!Map.MyLID(Map.LID(MaxMyGID)),ierr);
EPETRA_TEST_ERR(Map.MyLID(Map.LID(MaxMyGID+1)),ierr);
// Check RemoteIDList function
// Get some GIDs off of each processor to test
int TotalNumEle, NumElePerProc, NumProc = Comm.NumProc();
int MinNumEleOnProc;
int NumMyEle=Map.NumMyElements();
Comm.MinAll(&NumMyEle,&MinNumEleOnProc,1);
if (MinNumEleOnProc > 5) NumElePerProc = 6;
else NumElePerProc = MinNumEleOnProc;
if (NumElePerProc > 0) {
TotalNumEle = NumElePerProc*NumProc;
int * MyGIDlist = new int[NumElePerProc];
int * GIDlist = new int[TotalNumEle];
int * PIDlist = new int[TotalNumEle];
int * LIDlist = new int[TotalNumEle];
for (i=0; i<NumElePerProc; i++)
MyGIDlist[i] = MyGlobalElements1[i];
Comm.GatherAll(MyGIDlist,GIDlist,NumElePerProc);// Get a few values from each proc
Map.RemoteIDList(TotalNumEle, GIDlist, PIDlist, LIDlist);
int MyPID= Comm.MyPID();
forierr = 0;
for (i=0; i<TotalNumEle; i++) {
if (Map.MyGID(GIDlist[i])) {
forierr += PIDlist[i] != MyPID;
forierr += !Map.MyLID(Map.LID(GIDlist[i])) || Map.LID(GIDlist[i]) != LIDlist[i] || Map.GID(LIDlist[i]) != GIDlist[i];
}
else {
forierr += PIDlist[i] == MyPID; // If MyGID comes back false, the PID listed should be that of another proc
}
}
EPETRA_TEST_ERR(forierr,ierr);
delete [] MyGIDlist;
delete [] GIDlist;
delete [] PIDlist;
delete [] LIDlist;
}
delete [] MyGlobalElements1;
// Check RemoteIDList function (assumes all maps are linear, even if not stored that way)
if (Map.LinearMap()) {
int * GIDList = new int[3];
int * PIDList = new int[3];
int * LIDList = new int[3];
int MyPID = Map.Comm().MyPID();
int NumIDs = 0;
//GIDList[NumIDs++] = Map.MaxAllGID()+1; // Should return -1 for both PID and LID
if (Map.MinMyGID()-1>=Map.MinAllGID()) GIDList[NumIDs++] = Map.MinMyGID()-1;
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) GIDList[NumIDs++] = Map.MaxMyGID()+1;
Map.RemoteIDList(NumIDs, GIDList, PIDList, LIDList);
NumIDs = 0;
//EPETRA_TEST_ERR(!(PIDList[NumIDs]==-1),ierr);
//EPETRA_TEST_ERR(!(LIDList[NumIDs++]==-1),ierr);
if (Map.MinMyGID()-1>=Map.MinAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs++]==MyPID-1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(PIDList[NumIDs]==MyPID+1),ierr);
if (Map.MaxMyGID()+1<=Map.MaxAllGID()) EPETRA_TEST_ERR(!(LIDList[NumIDs++]==0),ierr);
delete [] GIDList;
delete [] PIDList;
delete [] LIDList;
}
return (ierr);
}
void
LOCA::Epetra::AugmentedOp::buildExtendedMap(const Epetra_BlockMap& uMap,
Epetra_Map*& eMapPtr,
bool buildImporter,
bool haveParam)
{
Epetra_BlockMap& nonconstUnderlyingMap = const_cast<Epetra_BlockMap&>(uMap);
// Convert underlying map to point map if necessary
Epetra_Map* uPointMapPtr =
dynamic_cast<Epetra_Map*>(&nonconstUnderlyingMap);
bool allocatedPointMap = false;
if (uPointMapPtr == NULL) {
allocatedPointMap = true;
blockMap2PointMap(uMap, uPointMapPtr);
}
int max_gid = uPointMapPtr->MaxAllGID();
int num_global_elements = uPointMapPtr->NumGlobalElements();
int num_my_elements = uPointMapPtr->NumMyElements();
int *global_elements = uPointMapPtr->MyGlobalElements();
const Epetra_Comm& comm = uPointMapPtr->Comm();
int index_base = uPointMapPtr->IndexBase();
int ext_num_global_elements;
int ext_num_my_elements;
int *ext_global_elements;
// Compute number of extended global elements
if (buildImporter)
ext_num_global_elements =
num_global_elements + numConstraints*comm.NumProc();
else
ext_num_global_elements = num_global_elements + numConstraints;
// Compute number of extended local elements
if (buildImporter || haveParam)
ext_num_my_elements = num_my_elements + numConstraints;
else
ext_num_my_elements = num_my_elements;
// Allocate extended global elements array
ext_global_elements = new int[ext_num_my_elements];
// Set extended global elements
for (int i=0; i<num_my_elements; i++) {
ext_global_elements[i] = global_elements[i];
}
if (buildImporter || haveParam)
for (int i=0; i<numConstraints; i++)
ext_global_elements[num_my_elements+i] = max_gid + 1 + i;
// Create extended point map
eMapPtr = new Epetra_Map(ext_num_global_elements, ext_num_my_elements,
ext_global_elements, index_base, comm);
// Free global elements array
delete [] ext_global_elements;
if (allocatedPointMap)
delete uPointMapPtr;
}
int MultiVectorTests(const Epetra_Map & Map, int NumVectors, bool verbose)
{
const Epetra_Comm & Comm = Map.Comm();
int ierr = 0, i, j;
/* get number of processors and the name of this processor */
int MyPID = Comm.MyPID();
// Construct FEVbrMatrix
if (verbose && MyPID==0) cout << "constructing Epetra_FEVbrMatrix" << endl;
//
//we'll set up a tri-diagonal matrix.
//
int numGlobalRows = Map.NumGlobalElements();
int minLocalRow = Map.MinMyGID();
int rowLengths = 3;
Epetra_FEVbrMatrix A(Copy, Map, rowLengths);
if (verbose && MyPID==0) {
cout << "calling A.InsertGlobalValues with 1-D data array"<<endl;
}
int numCols = 3;
int* ptIndices = new int[numCols];
for(int k=0; k<numCols; ++k) {
ptIndices[k] = minLocalRow+k;
}
double* values_1d = new double[numCols*numCols];
for(j=0; j<numCols*numCols; ++j) {
values_1d[j] = 3.0;
}
//For an extreme test, we'll have all processors sum into all rows.
int minGID = Map.MinAllGID();
//For now we're going to assume that there's just one point associated with
//each GID (element).
double* ptCoefs = new double[3];
{for(i=0; i<numGlobalRows; ++i) {
if (i>0 && i<numGlobalRows-1) {
ptIndices[0] = minGID+i-1;
ptIndices[1] = minGID+i;
ptIndices[2] = minGID+i+1;
ptCoefs[0] = -1.0;
ptCoefs[1] = 2.0;
ptCoefs[2] = -1.0;
numCols = 3;
}
else if (i == 0) {
ptIndices[0] = minGID+i;
ptIndices[1] = minGID+i+1;
ptIndices[2] = minGID+i+2;
ptCoefs[0] = 2.0;
ptCoefs[1] = -1.0;
ptCoefs[2] = -1.0;
numCols = 3;
}
else {
ptIndices[0] = minGID+i-2;
ptIndices[1] = minGID+i-1;
ptIndices[2] = minGID+i;
ptCoefs[0] = -1.0;
ptCoefs[1] = -1.0;
ptCoefs[2] = 2.0;
numCols = 3;
}
int row = minGID+i;
EPETRA_TEST_ERR( A.BeginInsertGlobalValues(row, rowLengths, ptIndices), ierr);
for(j=0; j<rowLengths; ++j) {
EPETRA_TEST_ERR( A.SubmitBlockEntry(&(ptCoefs[j]), 1, 1, 1), ierr);
}
EPETRA_TEST_ERR( A.EndSubmitEntries(), ierr);
}}
if (verbose&&MyPID==0) {
cout << "calling A.GlobalAssemble()" << endl;
}
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
if (verbose&&MyPID==0) {
cout << "after globalAssemble"<<endl;
}
if (verbose) {
A.Print(cout);
}
delete [] values_1d;
delete [] ptIndices;
delete [] ptCoefs;
return(ierr);
}
int quad1(const Epetra_Map& map, bool verbose)
{
const Epetra_Comm & Comm = map.Comm();
int numProcs = Comm.NumProc();
int localProc = Comm.MyPID();
Comm.Barrier();
if (verbose && localProc == 0) {
cout << "====================== quad1 =============================="<<endl;
}
//Set up a simple finite-element mesh containing 2-D quad elements, 1 per proc.
//
// *-----*-----*-----*
// 0| 2| 4| 6|
// | 0 | 1 | np-1|
// | | | |
// *-----*-----*-----*
// 1 3 5 7
//
//In the above drawing, 'np' means num-procs. node-numbers are to the
//lower-left of each node (*).
//
//Each processor owns element 'localProc', and each processor owns
//nodes 'localProc*2' and 'localProc*2+1' except for the last processor,
//which also owns the last two nodes.
//
//There will be 3 degrees-of-freedom per node, so each element-matrix is
//of size 12x12. (4 nodes per element, X 3 dof per node)
//
int myFirstNode = localProc*2;
int myLastNode = localProc*2+1;
if (localProc == numProcs-1) {
myLastNode += 2;
}
int numMyNodes = myLastNode - myFirstNode + 1;
int* myNodes = new int[numMyNodes];
int i, j, ierr;
for(i=0; i<numMyNodes; ++i) {
myNodes[i] = myFirstNode + i;
}
int dofPerNode = 3; //degrees-of-freedom per node
int indexBase = 0;
Epetra_BlockMap blkMap(-1, numMyNodes, myNodes, dofPerNode,
indexBase, Comm);
int rowLengths = 3; //each element-matrix will have 4 block-columns.
//the rows of the assembled matrix will be longer than
//this, but we don't need to worry about that because the
//VbrMatrix will add memory as needed. For a real
//application where efficiency is a concern, better
//performance would be obtained by giving more accurate
//row-lengths here.
Epetra_FEVbrMatrix A(Copy, blkMap, rowLengths);
int nodesPerElem = 4;
int* elemNodes = new int[nodesPerElem];
for(i=0; i<nodesPerElem; ++i) elemNodes[i] = myFirstNode+i;
int elemMatrixDim = nodesPerElem*dofPerNode;
int len = elemMatrixDim*elemMatrixDim;
double* elemMatrix = new double[len];
//In an actual finite-element problem, we would calculate and fill
//meaningful element stiffness matrices. But for this simple matrix assembly
//test, we're just going to fill our element matrix with 1.0's. This will
//make it easy to see whether the matrix is correct after it's assembled.
for(i=0; i<len; ++i) elemMatrix[i] = 1.0;
//For filling in the matrix block-entries, we would ordinarily have to
//carefully copy, or set pointers to, appropriate sections of the
//element-matrix. But for this simple case we know that the element-matrix
//is all 1's, so we'll just set our block-entry pointer to point to the
//beginning of the element-matrix and leave it at that.
//Note that the matrix class will refer to dofPerNode X dofPerNode (==9)
//positions in the memory pointed to by 'blockEntry'.
double* blockEntry = elemMatrix;
//The element-matrix is a 4x4 (nodesPerElem X nodesPerElem) matrix of
//3x3 block-entries. We'll now load our element-matrix into the global
//matrix by looping over it and loading block-entries individually.
for(i=0; i<nodesPerElem; ++i) {
int blkrow = myFirstNode+i;
EPETRA_TEST_ERR( A.BeginInsertGlobalValues(blkrow, nodesPerElem, elemNodes),
ierr);
for(j=0; j<nodesPerElem; ++j) {
for(int ii=0; ii<dofPerNode*dofPerNode; ii++) {
blockEntry[ii] = blkrow+elemNodes[j];
}
EPETRA_TEST_ERR( A.SubmitBlockEntry( blockEntry, dofPerNode,
dofPerNode, dofPerNode), ierr);
}
int err = A.EndSubmitEntries();
if (err < 0) {
cout << "quad1: error in A.EndSubmitEntries: "<<err<<endl;
return(-1);
}
}
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr);
if (verbose && localProc==0) {
cout << "after globalAssemble"<<endl;
}
if (verbose) {
A.Print(cout);
}
int numMyRows = A.NumMyRows();
int correct_numMyRows = dofPerNode*numMyNodes;
if (numMyRows != correct_numMyRows) {
cout << "proc " << localProc << ", numMyRows("<<numMyRows<<") doesn't match"
<< " correct_numMyRows("<<correct_numMyRows<<")."<<endl;
return(-1);
}
int numMyNonzeros = A.NumMyNonzeros();
int correct_numMyNonzeros = nodesPerElem*nodesPerElem*dofPerNode*dofPerNode;
if (numProcs > 1) {
if (localProc == numProcs-1) {
correct_numMyNonzeros += dofPerNode*dofPerNode*4;
}
else if (localProc > 0) {
correct_numMyNonzeros -= dofPerNode*dofPerNode*4;
}
else {
//localProc==0 && numProcs > 1
correct_numMyNonzeros -= dofPerNode*dofPerNode*8;
}
}
if (numMyNonzeros != correct_numMyNonzeros) {
cout << "proc " << localProc << ", numMyNonzeros(" << numMyNonzeros
<<") != correct_numMyNonzeros("<<correct_numMyNonzeros<<")"<<endl;
return(-1);
}
delete [] elemMatrix;
delete [] myNodes;
delete [] elemNodes;
Comm.Barrier();
return(0);
}
int Drumm1(const Epetra_Map& map, bool verbose)
{
//Simple 2-element problem (element as in "finite-element") from
//Clif Drumm. Two triangular elements, one per processor, as shown
//here:
//
// *----*
// 3|\ 2|
// | \ |
// | 0\1|
// | \|
// *----*
// 0 1
//
//Element 0 on processor 0, element 1 on processor 1.
//Processor 0 will own nodes 0,1 and processor 1 will own nodes 2,3.
//Each processor will pass a 3x3 element-connectivity-matrix to
//Epetra_FECrsGraph.
//After GlobalAssemble(), the graph should be as follows:
//
// row 0: 2 1 0 1
//proc 0 row 1: 1 4 1 2
//----------------------------------
// row 2: 0 1 2 1
//proc 1 row 3: 1 2 1 4
//
int numProcs = map.Comm().NumProc();
int localProc = map.Comm().MyPID();
if (numProcs != 2) return(0);
int indexBase = 0, ierr = 0;
int numMyNodes = 2;
long long* myNodes = new long long[numMyNodes];
if (localProc == 0) {
myNodes[0] = 0;
myNodes[1] = 1;
}
else {
myNodes[0] = 2;
myNodes[1] = 3;
}
Epetra_Map Map((long long) -1, numMyNodes, myNodes, indexBase, map.Comm());
delete [] myNodes;
numMyNodes = 3;
myNodes = new long long[numMyNodes];
if (localProc == 0) {
myNodes[0] = 0;
myNodes[1] = 1;
myNodes[2] = 3;
}
else {
myNodes[0] = 1;
myNodes[1] = 2;
myNodes[2] = 3;
}
int rowLengths = 3;
Epetra_FECrsGraph A(Copy, Map, rowLengths);
EPETRA_TEST_ERR( A.InsertGlobalIndices(numMyNodes, myNodes,
numMyNodes, myNodes),ierr);
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
if (verbose) {
A.Print(std::cout);
}
delete [] myNodes;
return(0);
}
int quad2(const Epetra_Map& map, bool verbose)
{
const Epetra_Comm & Comm = map.Comm();
int numProcs = Comm.NumProc();
int localProc = Comm.MyPID();
Comm.Barrier();
if (verbose && localProc == 0) {
cout << "====================== quad2 =============================="<<endl;
}
//Set up a simple finite-element mesh containing 2-D quad elements, 2 per proc.
//(This test is similar to quad1() above, except for having 2 elements-per-proc
// rather than 1.)
//
// *-----*-----*-----*-------*
// 0| 2| 4| 6| 8|
// | 0 | 1 | 2 | 2*np-1|
// | | | | |
// *-----*-----*-----*-------*
// 1 3 5 7 9
//
//In the above drawing, 'np' means num-procs. node-numbers are to the
//lower-left of each node (*).
//
//Each processor owns element 'localProc' and 'localProc+1', and each processor
//owns nodes 'localProc*4' through 'localProc*4+3' except for the last
//processor, which also owns the last two nodes in the mesh.
//
//There will be 3 degrees-of-freedom per node, so each element-matrix is
//of size 12x12.
//
int myFirstNode = localProc*4;
int myLastNode = localProc*4+3;
if (localProc == numProcs-1) {
myLastNode += 2;
}
int numMyElems = 2;
int numMyNodes = myLastNode - myFirstNode + 1;
int* myNodes = new int[numMyNodes];
int i, j, ierr;
for(i=0; i<numMyNodes; ++i) {
myNodes[i] = myFirstNode + i;
}
int dofPerNode = 3; //degrees-of-freedom per node
int indexBase = 0;
Epetra_BlockMap blkMap(-1, numMyNodes, myNodes, dofPerNode,
indexBase, Comm);
int rowLengths = 4; //each element-matrix will have 4 block-columns.
//the rows of the assembled matrix will be longer than
//this, but we don't need to worry about that because the
//VbrMatrix will add memory as needed. For a real
//application where efficiency is a concern, better
//performance would be obtained by giving a more accurate
//row-length here.
Epetra_FEVbrMatrix A(Copy, blkMap, rowLengths);
int nodesPerElem = 4;
int* elemNodes = new int[nodesPerElem];
int elemMatrixDim = nodesPerElem*dofPerNode;
int len = elemMatrixDim*elemMatrixDim;
double* elemMatrix = new double[len];
//In an actual finite-element problem, we would calculate and fill
//meaningful element stiffness matrices. But for this simple matrix assembly
//test, we're just going to fill our element matrix with 1.0's. This will
//make it easy to see whether the matrix is correct after it's assembled.
for(i=0; i<len; ++i) elemMatrix[i] = 1.0;
//For filling in the matrix block-entries, we would ordinarily have to
//carefully copy, or set pointers to, appropriate sections of the
//element-matrix. But for this simple case we know that the element-matrix
//is all 1's, so we'll just set our block-entry pointer to point to the
//beginning of the element-matrix and leave it at that.
//Note that the matrix class will refer to dofPerNode X dofPerNode (==9)
//positions in the memory pointed to by 'blockEntry'.
double* blockEntry = elemMatrix;
//Each element-matrix is a 4x4 (nodesPerElem X nodesPerElem) matrix of
//3x3 block-entries. We'll now load our element-matrices into the global
//matrix by looping over them and loading block-entries individually.
int firstNode = myFirstNode;
for(int el=0; el<numMyElems; ++el) {
for(i=0; i<nodesPerElem; ++i) {
for(int n=0; n<nodesPerElem; ++n) elemNodes[n] = firstNode+n;
int blkrow = firstNode+i;
EPETRA_TEST_ERR( A.BeginInsertGlobalValues(blkrow, nodesPerElem, elemNodes),
ierr);
for(j=0; j<nodesPerElem; ++j) {
EPETRA_TEST_ERR( A.SubmitBlockEntry( blockEntry, dofPerNode,
dofPerNode, dofPerNode), ierr);
}
int this_err = A.EndSubmitEntries();
if (this_err < 0) {
cerr << "error in quad2, A.EndSubmitEntries(): " << this_err << endl;
return(this_err);
}
}
firstNode += 2;
}
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr);
if (verbose && localProc==0) {
cout << "after globalAssemble"<<endl;
}
if (verbose) {
A.Print(cout);
}
//now let's make sure that we can perform a matvec...
Epetra_FEVector x(blkMap, 1), y(blkMap, 1);
x.PutScalar(1.0);
EPETRA_TEST_ERR( A.Multiply(false, x, y), ierr);
if (verbose && localProc==0) {
cout << "quad2, y:"<<endl;
}
if (verbose) {
y.Print(cout);
}
delete [] elemMatrix;
delete [] myNodes;
delete [] elemNodes;
return(0);
}
// FIXME long long
Epetra_Map
Epetra_Util::Create_Root_Map(const Epetra_Map& usermap,
int root)
{
int numProc = usermap.Comm().NumProc();
if (numProc==1) {
Epetra_Map newmap(usermap);
return(newmap);
}
const Epetra_Comm & comm = usermap.Comm();
bool isRoot = usermap.Comm().MyPID()==root;
//if usermap is already completely owned by root then we'll just return a copy of it.
int quickreturn = 0;
int globalquickreturn = 0;
if (isRoot) {
if (usermap.NumMyElements()==usermap.NumGlobalElements64()) quickreturn = 1;
}
else {
if (usermap.NumMyElements()==0) quickreturn = 1;
}
usermap.Comm().MinAll(&quickreturn, &globalquickreturn, 1);
if (globalquickreturn==1) {
Epetra_Map newmap(usermap);
return(newmap);
}
// Linear map: Simple case, just put all GIDs linearly on root processor
if (usermap.LinearMap() && root!=-1) {
int numMyElements = 0;
if (isRoot) numMyElements = usermap.MaxAllGID64()+1; // FIXME long long
Epetra_Map newmap(-1, numMyElements, usermap.IndexBase(), comm);
return(newmap);
}
if (!usermap.UniqueGIDs())
throw usermap.ReportError("usermap must have unique GIDs",-1);
// General map
// Build IntVector of the GIDs, then ship them to root processor
int numMyElements = usermap.NumMyElements();
Epetra_Map allGidsMap(-1, numMyElements, 0, comm);
Epetra_IntVector allGids(allGidsMap);
for (int i=0; i<numMyElements; i++) allGids[i] = usermap.GID64(i);
int numGlobalElements = usermap.NumGlobalElements64();
if (root!=-1) {
int n1 = 0; if (isRoot) n1 = numGlobalElements;
Epetra_Map allGidsOnRootMap(-1, n1, 0, comm);
Epetra_Import importer(allGidsOnRootMap, allGidsMap);
Epetra_IntVector allGidsOnRoot(allGidsOnRootMap);
allGidsOnRoot.Import(allGids, importer, Insert);
Epetra_Map rootMap(-1, allGidsOnRoot.MyLength(), allGidsOnRoot.Values(), usermap.IndexBase(), comm);
return(rootMap);
}
else {
int n1 = numGlobalElements;
Epetra_LocalMap allGidsOnRootMap(n1, 0, comm);
Epetra_Import importer(allGidsOnRootMap, allGidsMap);
Epetra_IntVector allGidsOnRoot(allGidsOnRootMap);
allGidsOnRoot.Import(allGids, importer, Insert);
Epetra_Map rootMap(-1, allGidsOnRoot.MyLength(), allGidsOnRoot.Values(), usermap.IndexBase(), comm);
return(rootMap);
}
}
int TLowCommunicationMakeColMapAndReindex(int N, const int * rowptr, int * colind_LID, const int_type *colind_GID, const Epetra_Map& domainMap, const int * owningPIDs, bool SortGhostsAssociatedWithEachProcessor, std::vector<int>& RemotePIDs, MapType1 & NewColMap)
{
int i,j;
// Sanity checks
bool UseLL;
if(domainMap.GlobalIndicesLongLong()) UseLL=true;
else if(domainMap.GlobalIndicesInt()) UseLL=false;
else throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot detect int type.");
// Scan all column indices and sort into two groups:
// Local: those whose GID matches a GID of the domain map on this processor and
// Remote: All others.
int numDomainElements = domainMap.NumMyElements();
bool * LocalGIDs = 0;
if (numDomainElements>0) LocalGIDs = new bool[numDomainElements];
for (i=0; i<numDomainElements; i++) LocalGIDs[i] = false; // Assume domain GIDs are not local
bool DoSizes = !domainMap.ConstantElementSize(); // If not constant element size, then error
if(DoSizes) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: cannot handle non-constant sized domainMap.");
// In principle it is good to have RemoteGIDs and RemotGIDList be as long as the number of remote GIDs
// on this processor, but this would require two passes through the column IDs, so we make it the max of 100
// and the number of block rows.
const int numMyBlockRows = N;
int hashsize = numMyBlockRows; if (hashsize < 100) hashsize = 100;
Epetra_HashTable<int_type> RemoteGIDs(hashsize);
std::vector<int_type> RemoteGIDList; RemoteGIDList.reserve(hashsize);
std::vector<int> PIDList; PIDList.reserve(hashsize);
// Here we start using the *int* colind array. If int_type==int this clobbers the GIDs, if
// int_type==long long, then this is the first use of the colind array.
// For *local* GID's set colind with with their LID in the domainMap. For *remote* GIDs,
// we set colind with (numDomainElements+NumRemoteColGIDs) before the increment of
// the remote count. These numberings will be separate because no local LID is greater
// than numDomainElements.
int NumLocalColGIDs = 0;
int NumRemoteColGIDs = 0;
for(i = 0; i < numMyBlockRows; i++) {
for(j = rowptr[i]; j < rowptr[i+1]; j++) {
int_type GID = colind_GID[j];
// Check if GID matches a row GID
int LID = domainMap.LID(GID);
if(LID != -1) {
bool alreadyFound = LocalGIDs[LID];
if (!alreadyFound) {
LocalGIDs[LID] = true; // There is a column in the graph associated with this domain map GID
NumLocalColGIDs++;
}
colind_LID[j] = LID;
}
else {
int_type hash_value=RemoteGIDs.Get(GID);
if(hash_value == -1) { // This means its a new remote GID
int PID = owningPIDs[j];
if(PID==-1) throw std::runtime_error("LowCommunicationMakeColMapAndReindex: Cannot figure out if PID is owned.");
colind_LID[j] = numDomainElements + NumRemoteColGIDs;
RemoteGIDs.Add(GID, NumRemoteColGIDs);
RemoteGIDList.push_back(GID);
PIDList.push_back(PID);
NumRemoteColGIDs++;
}
else
colind_LID[j] = numDomainElements + hash_value;
}
}
}
// Possible short-circuit: If all domain map GIDs are present as column indices, then set ColMap=domainMap and quit
if (domainMap.Comm().NumProc()==1) {
if (NumRemoteColGIDs!=0) {
throw std::runtime_error("Some column IDs are not in domainMap. If matrix is rectangular, you must pass in a domainMap");
// Sanity test: When one processor,there can be no remoteGIDs
}
if (NumLocalColGIDs==numDomainElements) {
if (LocalGIDs!=0) delete [] LocalGIDs;
// In this case, we just use the domainMap's indices, which is, not coincidently, what we clobbered colind with up above anyway.
// No further reindexing is needed.
NewColMap = domainMap;
return 0;
}
}
// Now build integer array containing column GIDs
// Build back end, containing remote GIDs, first
int numMyBlockCols = NumLocalColGIDs + NumRemoteColGIDs;
std::vector<int_type> ColIndices;
int_type * RemoteColIndices=0;
if(numMyBlockCols > 0) {
ColIndices.resize(numMyBlockCols);
if(NumLocalColGIDs!=numMyBlockCols) RemoteColIndices = &ColIndices[NumLocalColGIDs]; // Points to back end of ColIndices
else RemoteColIndices=0;
}
for(i = 0; i < NumRemoteColGIDs; i++)
RemoteColIndices[i] = RemoteGIDList[i];
// Build permute array for *remote* reindexing.
std::vector<int> RemotePermuteIDs(NumRemoteColGIDs);
for(i=0; i<NumRemoteColGIDs; i++) RemotePermuteIDs[i]=i;
// Sort External column indices so that all columns coming from a given remote processor are contiguous
int NumListsInt=0;
int NumListsLL =0;
int * IntSortLists[2];
long long * LLSortLists[2];
int * RemotePermuteIDs_ptr = RemotePermuteIDs.size() ? &RemotePermuteIDs[0] : 0;
if(!UseLL) {
// int version
IntSortLists[0] = (int*) RemoteColIndices;
IntSortLists[1] = RemotePermuteIDs_ptr;
NumListsInt=2;
}
else {
//LL version
LLSortLists[0] = (long long*) RemoteColIndices;
IntSortLists[0] = RemotePermuteIDs_ptr;
NumListsInt = NumListsLL = 1;
}
int * PIDList_ptr = PIDList.size() ? &PIDList[0] : 0;
Epetra_Util::Sort(true, NumRemoteColGIDs, PIDList_ptr, 0, 0, NumListsInt, IntSortLists,NumListsLL,LLSortLists);
// Stash the RemotePIDs
PIDList.resize(NumRemoteColGIDs);
RemotePIDs = PIDList;
if (SortGhostsAssociatedWithEachProcessor) {
// Sort external column indices so that columns from a given remote processor are not only contiguous
// but also in ascending order. NOTE: I don't know if the number of externals associated
// with a given remote processor is known at this point ... so I count them here.
// NTS: Only sort the RemoteColIndices this time...
int StartCurrent, StartNext;
StartCurrent = 0; StartNext = 1;
while ( StartNext < NumRemoteColGIDs ) {
if (PIDList[StartNext]==PIDList[StartNext-1]) StartNext++;
else {
IntSortLists[0] = &RemotePermuteIDs[StartCurrent];
Epetra_Util::Sort(true,StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]),0,0,1,IntSortLists,0,0);
StartCurrent = StartNext; StartNext++;
}
}
IntSortLists[0] = &RemotePermuteIDs[StartCurrent];
Epetra_Util::Sort(true, StartNext-StartCurrent, &(RemoteColIndices[StartCurrent]), 0, 0, 1,IntSortLists,0,0);
}
// Reverse the permutation to get the information we actually care about
std::vector<int> ReverseRemotePermuteIDs(NumRemoteColGIDs);
for(i=0; i<NumRemoteColGIDs; i++) ReverseRemotePermuteIDs[RemotePermuteIDs[i]]=i;
// Build permute array for *local* reindexing.
bool use_local_permute=false;
std::vector<int> LocalPermuteIDs(numDomainElements);
// Now fill front end. Two cases:
// (1) If the number of Local column GIDs is the same as the number of Local domain GIDs, we
// can simply read the domain GIDs into the front part of ColIndices, otherwise
// (2) We step through the GIDs of the domainMap, checking to see if each domain GID is a column GID.
// we want to do this to maintain a consistent ordering of GIDs between the columns and the domain.
if(NumLocalColGIDs == domainMap.NumMyElements()) {
if(NumLocalColGIDs > 0) {
domainMap.MyGlobalElements(&ColIndices[0]); // Load Global Indices into first numMyBlockCols elements column GID list
}
}
else {
int_type* MyGlobalElements = 0;
domainMap.MyGlobalElementsPtr(MyGlobalElements);
int* ElementSizeList = 0;
if(DoSizes)
ElementSizeList = domainMap.ElementSizeList();
int NumLocalAgain = 0;
use_local_permute = true;
for(i = 0; i < numDomainElements; i++) {
if(LocalGIDs[i]) {
LocalPermuteIDs[i] = NumLocalAgain;
ColIndices[NumLocalAgain++] = MyGlobalElements[i];
}
}
assert(NumLocalAgain==NumLocalColGIDs); // Sanity test
}
// Done with this array
if (LocalGIDs!=0) delete [] LocalGIDs;
// Make Column map with same element sizes as Domain map
int_type * ColIndices_ptr = ColIndices.size() ? &ColIndices[0] : 0;
MapType2 temp((int_type)(-1), numMyBlockCols, ColIndices_ptr, (int_type)domainMap.IndexBase64(), domainMap.Comm());
NewColMap = temp;
// Low-cost reindex of the matrix
for(i=0; i<numMyBlockRows; i++){
for(j=rowptr[i]; j<rowptr[i+1]; j++){
int ID=colind_LID[j];
if(ID < numDomainElements){
if(use_local_permute) colind_LID[j] = LocalPermuteIDs[colind_LID[j]];
// In the case where use_local_permute==false, we just copy the DomainMap's ordering, which it so happens
// is what we put in colind to begin with.
}
else
colind_LID[j] = NumLocalColGIDs + ReverseRemotePermuteIDs[colind_LID[j]-numDomainElements];
}
}
return 0;
}
int Drumm1(const Epetra_Map& map, bool verbose)
{
(void)verbose;
//Simple 2-element problem (element as in "finite-element") from
//Clif Drumm. Two triangular elements, one per processor, as shown
//here:
//
// *----*
// 3|\ 2|
// | \ |
// | 0\1|
// | \|
// *----*
// 0 1
//
//Element 0 on processor 0, element 1 on processor 1.
//Processor 0 will own nodes 0,1 and processor 1 will own nodes 2,3.
//Each processor will pass a 3x3 element-matrix to Epetra_FECrsMatrix.
//After GlobalAssemble(), the matrix should be as follows:
//
// row 0: 2 1 0 1
//proc 0 row 1: 1 4 1 2
//----------------------------------
// row 2: 0 1 2 1
//proc 1 row 3: 1 2 1 4
//
int numProcs = map.Comm().NumProc();
int localProc = map.Comm().MyPID();
if (numProcs != 2) return(0);
//so first we'll set up a epetra_test::matrix_data object with
//contents that match the above-described matrix. (but the
//matrix_data object will have all 4 rows on each processor)
int i;
int rowlengths[4];
rowlengths[0] = 3;
rowlengths[1] = 4;
rowlengths[2] = 3;
rowlengths[3] = 4;
epetra_test::matrix_data matdata(4, rowlengths);
for(i=0; i<4; ++i) {
for(int j=0; j<matdata.rowlengths()[i]; ++j) {
matdata.colindices()[i][j] = j;
}
}
matdata.colindices()[0][2] = 3;
matdata.colindices()[2][0] = 1;
matdata.colindices()[2][1] = 2;
matdata.colindices()[2][2] = 3;
double** coefs = matdata.coefs();
coefs[0][0] = 2.0; coefs[0][1] = 1.0; coefs[0][2] = 1.0;
coefs[1][0] = 1.0; coefs[1][1] = 4.0; coefs[1][2] = 1.0; coefs[1][3] = 2.0;
coefs[2][0] = 1.0; coefs[2][1] = 2.0; coefs[2][2] = 1.0;
coefs[3][0] = 1.0; coefs[3][1] = 2.0; coefs[3][2] = 1.0; coefs[3][3] = 4.0;
//now we'll load a Epetra_FECrsMatrix with data that matches the
//above-described finite-element problem.
int indexBase = 0, ierr = 0;
int myNodes[4];
double values[9];
values[0] = 2.0;
values[1] = 1.0;
values[2] = 1.0;
values[3] = 1.0;
values[4] = 2.0;
values[5] = 1.0;
values[6] = 1.0;
values[7] = 1.0;
values[8] = 2.0;
int numMyNodes = 2;
if (localProc == 0) {
myNodes[0] = 0;
myNodes[1] = 1;
}
else {
myNodes[0] = 2;
myNodes[1] = 3;
}
Epetra_Map Map(-1, numMyNodes, myNodes, indexBase, map.Comm());
numMyNodes = 3;
if (localProc == 0) {
myNodes[0] = 0;
myNodes[1] = 1;
myNodes[2] = 3;
}
else {
myNodes[0] = 1;
myNodes[1] = 2;
myNodes[2] = 3;
}
int rowLengths = 3;
Epetra_FECrsMatrix A(Copy, Map, rowLengths);
EPETRA_TEST_ERR( A.InsertGlobalValues(numMyNodes, myNodes,
numMyNodes, myNodes, values,
Epetra_FECrsMatrix::ROW_MAJOR),ierr);
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
//now the test is to check whether the FECrsMatrix data matches the
//epetra_test::matrix_data object...
bool the_same = matdata.compare_local_data(A);
if (!the_same) {
return(-1);
}
return(0);
}
int Drumm3(const Epetra_Map& map, bool verbose)
{
const Epetra_Comm & Comm = map.Comm();
/* get number of processors and the name of this processor */
int Numprocs = Comm.NumProc();
int MyPID = Comm.MyPID();
if (Numprocs != 2) return(0);
int NumGlobalRows = 4;
int IndexBase = 0;
Epetra_Map Map(NumGlobalRows, IndexBase, Comm);
// Construct FECrsMatrix
int NumEntriesPerRow = 3;
Epetra_FECrsMatrix A(Copy, Map, NumEntriesPerRow);
double ElementArea = 0.5;
int NumCols = 3;
int* Indices = new int[NumCols];
if(MyPID==0) // indices corresponding to element 0 on processor 0
{
Indices[0] = 0;
Indices[1] = 1;
Indices[2] = 3;
}
else if(MyPID==1) // indices corresponding to element 1 on processor 1
{
Indices[0] = 1;
Indices[1] = 2;
Indices[2] = 3;
}
double* Values = new double[NumCols*NumCols];
// removal term
Values[0] = 2*ElementArea/12.;
Values[1] = 1*ElementArea/12.;
Values[2] = 1*ElementArea/12.;
Values[3] = 1*ElementArea/12.;
Values[4] = 2*ElementArea/12.;
Values[5] = 1*ElementArea/12.;
Values[6] = 1*ElementArea/12.;
Values[7] = 1*ElementArea/12.;
Values[8] = 2*ElementArea/12.;
A.InsertGlobalValues(NumCols, Indices,
Values,
Epetra_FECrsMatrix::ROW_MAJOR);
A.GlobalAssemble();
A.GlobalAssemble();
// A.Print(cout);
// Create vectors for CG algorithm
Epetra_FEVector* bptr = new Epetra_FEVector(A.RowMap(), 1);
Epetra_FEVector* x0ptr = new Epetra_FEVector(A.RowMap(), 1);
Epetra_FEVector& b = *bptr;
Epetra_FEVector& x0 = *x0ptr;
// source terms
NumCols = 2;
if(MyPID==0) // indices corresponding to element 0 on processor 0
{
Indices[0] = 0;
Indices[1] = 3;
Values[0] = 1./2.;
Values[1] = 1./2.;
}
else
{
Indices[0] = 1;
Indices[1] = 2;
Values[0] = 0;
Values[1] = 0;
}
b.SumIntoGlobalValues(NumCols, Indices, Values);
b.GlobalAssemble();
if (verbose&&MyPID==0) cout << "b:" << endl;
if (verbose) {
b.Print(cout);
}
x0 = b;
if (verbose&&MyPID==0) {
cout << "x:"<<endl;
}
if (verbose) {
x0.Print(cout);
}
delete [] Values;
delete [] Indices;
delete bptr;
delete x0ptr;
return(0);
}
int Drumm2(const Epetra_Map& map, bool verbose)
{
//Simple 2-element problem (element as in "finite-element") from
//Clif Drumm. Two triangular elements, one per processor, as shown
//here:
//
// *----*
// 3|\ 2|
// | \ |
// | 0\1|
// | \|
// *----*
// 0 1
//
//Element 0 on processor 0, element 1 on processor 1.
//Processor 0 will own nodes 0,1,3 and processor 1 will own node 2.
//Each processor will pass a 3x3 element-matrix to Epetra_FECrsMatrix.
//After GlobalAssemble(), the matrix should be as follows:
//
// row 0: 2 1 0 1
//proc 0 row 1: 1 4 1 2
// row 2: 0 1 2 1
//----------------------------------
//proc 1 row 3: 1 2 1 4
//
int numProcs = map.Comm().NumProc();
int localProc = map.Comm().MyPID();
if (numProcs != 2) return(0);
int indexBase = 0, ierr = 0;
double* values = new double[9];
values[0] = 2.0;
values[1] = 1.0;
values[2] = 1.0;
values[3] = 1.0;
values[4] = 2.0;
values[5] = 1.0;
values[6] = 1.0;
values[7] = 1.0;
values[8] = 2.0;
if (localProc == 0) {
int numMyNodes = 3;
int* myNodes = new int[numMyNodes];
myNodes[0] = 0;
myNodes[1] = 1;
myNodes[2] = 3;
Epetra_Map Map(-1, numMyNodes, myNodes, indexBase, map.Comm());
int rowLengths = 3;
Epetra_FECrsMatrix A(Copy, Map, rowLengths);
EPETRA_TEST_ERR( A.InsertGlobalValues(numMyNodes, myNodes,
numMyNodes, myNodes,
values, Epetra_FECrsMatrix::ROW_MAJOR),ierr);
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
if (verbose) {
A.Print(cout);
}
//now let's make sure we can do a matvec with this matrix.
Epetra_Vector x(Map), y(Map);
x.PutScalar(1.0);
EPETRA_TEST_ERR( A.Multiply(false, x, y), ierr);
if (verbose&&localProc==0) {
cout << "y = A*x, x=1.0's"<<endl;
}
if (verbose) {
y.Print(cout);
}
delete [] myNodes;
delete [] values;
}
else {
int numMyNodes = 1;
int* myNodes = new int[numMyNodes];
myNodes[0] = 2;
Epetra_Map Map(-1, numMyNodes, myNodes, indexBase, map.Comm());
int rowLengths = 3;
Epetra_FECrsMatrix A(Copy, Map, rowLengths);
delete [] myNodes;
numMyNodes = 3;
myNodes = new int[numMyNodes];
myNodes[0] = 1;
myNodes[1] = 2;
myNodes[2] = 3;
EPETRA_TEST_ERR( A.InsertGlobalValues(numMyNodes, myNodes,
numMyNodes, myNodes,
values, Epetra_FECrsMatrix::ROW_MAJOR),ierr);
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
EPETRA_TEST_ERR( A.GlobalAssemble(), ierr );
if (verbose) {
A.Print(cout);
}
//now let's make sure we can do a matvec with this matrix.
Epetra_Vector x(Map), y(Map);
x.PutScalar(1.0);
EPETRA_TEST_ERR( A.Multiply(false, x, y), ierr);
if (verbose) {
y.Print(cout);
}
delete [] myNodes;
delete [] values;
}
return(0);
}
