void
LOCA::Epetra::CompactWYOp::applyCompactWY(const Epetra_MultiVector& x,
Epetra_MultiVector& result_x,
Epetra_MultiVector& result_p) const
{
// Compute Y_x^T*x
result_p.Multiply('T', 'N', 1.0, *Y_x, x, 0.0);
// Compute T*(Y_x^T*x)
dblas.TRMM(Teuchos::LEFT_SIDE, Teuchos::UPPER_TRI, Teuchos::NO_TRANS,
Teuchos::NON_UNIT_DIAG, result_p.MyLength(),
result_p.NumVectors(), 1.0, T.Values(), T.MyLength(),
result_p.Values(), result_p.MyLength());
// Compute x = x + Y_x*T*(Y_x^T*x)
result_x = x;
result_x.Multiply('N', 'N', 1.0, *Y_x, result_p, 1.0);
// Compute result_p = Y_p*T*(Y_x^T*x)
dblas.TRMM(Teuchos::LEFT_SIDE, Teuchos::LOWER_TRI, Teuchos::NO_TRANS,
Teuchos::UNIT_DIAG, result_p.MyLength(),
result_p.NumVectors(), 1.0, Y_p.Values(), Y_p.MyLength(),
result_p.Values(), result_p.MyLength());
}
Epetra_OskiMultiVector::Epetra_OskiMultiVector(const Epetra_MultiVector& Source)
: Epetra_MultiVector(Source),
Epetra_View_(&Source),
Copy_Created_(false) {
double* A;
double** Aptr;
int LDA;
int* LDAptr;
LDAptr = new int[1];
Aptr = new double*[1];
if(Source.ConstantStride() || (Source.NumVectors() == 1)) {
if(Source.ExtractView(Aptr, LDAptr))
std::cerr << "Extract view failed\n";
else
Oski_View_ = oski_CreateMultiVecView(*Aptr, Source.MyLength(), Source.NumVectors(), LAYOUT_COLMAJ, *LDAptr);
}
else {
Copy_Created_ = true;
LDA = Source.MyLength();
A = new double[LDA*Source.NumVectors()];
if(Source.ExtractCopy(A, LDA))
std::cerr << "Extract copy failed\n";
else
Oski_View_ = oski_CreateMultiVecView(A, Source.MyLength(), Source.NumVectors(), LAYOUT_COLMAJ, LDA);
}
delete [] LDAptr;
delete [] Aptr;
}
int writeMultiVector(FILE * handle, const Epetra_MultiVector & A, bool mmFormat) {
int ierr = 0;
int length = A.GlobalLength();
int numVectors = A.NumVectors();
const Epetra_Comm & comm = A.Map().Comm();
if (comm.MyPID()!=0) {
if (A.MyLength()!=0) ierr = -1;
}
else {
if (length!=A.MyLength()) ierr = -1;
for (int j=0; j<numVectors; j++) {
for (int i=0; i<length; i++) {
double val = A[j][i];
if (mmFormat)
fprintf(handle, "%22.16e\n", val);
else
fprintf(handle, "%22.16e ", val);
}
if (!mmFormat) fprintf(handle, "%s", "\n");
}
}
int ierrGlobal;
comm.MinAll(&ierr, &ierrGlobal, 1); // If any processor has -1, all return -1
return(ierrGlobal);
}
//=======================================================
int EpetraExt_HypreIJMatrix::Multiply(bool TransA,
const Epetra_MultiVector& X,
Epetra_MultiVector& Y) const
{
//printf("Proc[%d], Row start: %d, Row End: %d\n", Comm().MyPID(), MyRowStart_, MyRowEnd_);
bool SameVectors = false;
int NumVectors = X.NumVectors();
if (NumVectors != Y.NumVectors()) return -1; // X and Y must have same number of vectors
if(X.Pointers() == Y.Pointers()){
SameVectors = true;
}
for(int VecNum = 0; VecNum < NumVectors; VecNum++) {
//Get values for current vector in multivector.
double * x_values;
double * y_values;
EPETRA_CHK_ERR((*X(VecNum)).ExtractView(&x_values));
double *x_temp = x_local->data;
double *y_temp = y_local->data;
if(!SameVectors){
EPETRA_CHK_ERR((*Y(VecNum)).ExtractView(&y_values));
} else {
y_values = new double[X.MyLength()];
}
y_local->data = y_values;
EPETRA_CHK_ERR(HYPRE_ParVectorSetConstantValues(par_y,0.0));
// Temporarily make a pointer to data in Hypre for end
// Replace data in Hypre vectors with epetra values
x_local->data = x_values;
// Do actual computation.
if(TransA) {
// Use transpose of A in multiply
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixMatvecT(1.0, ParMatrix_, par_x, 1.0, par_y));
} else {
EPETRA_CHK_ERR(HYPRE_ParCSRMatrixMatvec(1.0, ParMatrix_, par_x, 1.0, par_y));
}
if(SameVectors){
int NumEntries = Y.MyLength();
std::vector<double> new_values; new_values.resize(NumEntries);
std::vector<int> new_indices; new_indices.resize(NumEntries);
for(int i = 0; i < NumEntries; i++){
new_values[i] = y_values[i];
new_indices[i] = i;
}
EPETRA_CHK_ERR((*Y(VecNum)).ReplaceMyValues(NumEntries, &new_values[0], &new_indices[0]));
delete[] y_values;
}
x_local->data = x_temp;
y_local->data = y_temp;
}
double flops = (double) NumVectors * (double) NumGlobalNonzeros();
UpdateFlops(flops);
return 0;
} //Multiply()
//==============================================================================
int Ifpack_Hypre::Multiply(bool TransA, const Epetra_MultiVector& X, Epetra_MultiVector& Y) const{
if(IsInitialized() == false){
IFPACK_CHK_ERR(-1);
}
bool SameVectors = false;
int NumVectors = X.NumVectors();
if (NumVectors != Y.NumVectors()) IFPACK_CHK_ERR(-1); // X and Y must have same number of vectors
if(X.Pointers() == Y.Pointers()){
SameVectors = true;
}
for(int VecNum = 0; VecNum < NumVectors; VecNum++) {
//Get values for current vector in multivector.
double * XValues;
double * YValues;
IFPACK_CHK_ERR((*X(VecNum)).ExtractView(&XValues));
double *XTemp = XLocal_->data;
double *YTemp = YLocal_->data;
if(!SameVectors){
IFPACK_CHK_ERR((*Y(VecNum)).ExtractView(&YValues));
} else {
YValues = new double[X.MyLength()];
}
YLocal_->data = YValues;
IFPACK_CHK_ERR(HYPRE_ParVectorSetConstantValues(ParY_,0.0));
// Temporarily make a pointer to data in Hypre for end
// Replace data in Hypre vectors with epetra values
XLocal_->data = XValues;
// Do actual computation.
if(TransA) {
// Use transpose of A in multiply
IFPACK_CHK_ERR(HYPRE_ParCSRMatrixMatvecT(1.0, ParMatrix_, ParX_, 1.0, ParY_));
} else {
IFPACK_CHK_ERR(HYPRE_ParCSRMatrixMatvec(1.0, ParMatrix_, ParX_, 1.0, ParY_));
}
if(SameVectors){
int NumEntries = Y.MyLength();
std::vector<double> new_values; new_values.resize(NumEntries);
std::vector<int> new_indices; new_indices.resize(NumEntries);
for(int i = 0; i < NumEntries; i++){
new_values[i] = YValues[i];
new_indices[i] = i;
}
IFPACK_CHK_ERR((*Y(VecNum)).ReplaceMyValues(NumEntries, &new_values[0], &new_indices[0]));
delete[] YValues;
}
XLocal_->data = XTemp;
YLocal_->data = YTemp;
}
return 0;
} //Multiply()
int Epetra_PETScAIJMatrix::Multiply(bool TransA,
const Epetra_MultiVector& X,
Epetra_MultiVector& Y) const
{
(void)TransA;
int NumVectors = X.NumVectors();
if (NumVectors!=Y.NumVectors()) EPETRA_CHK_ERR(-1); // X and Y must have same number of vectors
double ** xptrs;
double ** yptrs;
X.ExtractView(&xptrs);
Y.ExtractView(&yptrs);
if (RowMatrixImporter()!=0) {
if (ImportVector_!=0) {
if (ImportVector_->NumVectors()!=NumVectors) { delete ImportVector_; ImportVector_= 0;}
}
if (ImportVector_==0) ImportVector_ = new Epetra_MultiVector(RowMatrixColMap(),NumVectors);
ImportVector_->Import(X, *RowMatrixImporter(), Insert);
ImportVector_->ExtractView(&xptrs);
}
double *vals=0;
int length;
Vec petscX, petscY;
int ierr;
for (int i=0; i<NumVectors; i++) {
# ifdef HAVE_MPI
ierr=VecCreateMPIWithArray(Comm_->Comm(),X.MyLength(),X.GlobalLength(),xptrs[i],&petscX); CHKERRQ(ierr);
ierr=VecCreateMPIWithArray(Comm_->Comm(),Y.MyLength(),Y.GlobalLength(),yptrs[i],&petscY); CHKERRQ(ierr);
# else //FIXME untested
ierr=VecCreateSeqWithArray(Comm_->Comm(),X.MyLength(),X.GlobalLength(),xptrs[i],&petscX); CHKERRQ(ierr);
ierr=VecCreateSeqWithArray(Comm_->Comm(),Y.MyLength(),Y.GlobalLength(),yptrs[i],&petscY); CHKERRQ(ierr);
# endif
ierr = MatMult(Amat_,petscX,petscY);CHKERRQ(ierr);
ierr = VecGetArray(petscY,&vals);CHKERRQ(ierr);
ierr = VecGetLocalSize(petscY,&length);CHKERRQ(ierr);
for (int j=0; j<length; j++) yptrs[i][j] = vals[j];
ierr = VecRestoreArray(petscY,&vals);CHKERRQ(ierr);
}
VecDestroy(petscX); VecDestroy(petscY);
double flops = NumGlobalNonzeros();
flops *= 2.0;
flops *= (double) NumVectors;
UpdateFlops(flops);
return(0);
} //Multiply()
void operator () (const Epetra_MultiVector &x, Epetra_MultiVector &y)
{
int myCols = y.MyLength();
for (int j=0; j < x.NumVectors(); ++j) {
for (int i=0; i < myCols; ++i) (*y(j))[i] = (i+1)*v*(*x(j))[i]; // NOTE: square operator!
}
};
// application of the tridiagonal operator
int Apply( const Epetra_MultiVector & X,
Epetra_MultiVector & Y ) const
{
int Length = X.MyLength();
// maybe some error checks on MultiVector Lenghts
// for the future...
for( int vec=0 ; vec<X.NumVectors() ; ++vec ) {
// one-dimensional problems here
if( Length == 1 ) {
Y[vec][0] = diag_ * X[vec][0];
break;
}
// more general case (Lenght >= 2)
// first row
Y[vec][0] = diag_ * X[vec][0] + diag_plus_one_ * X[vec][1];
// intermediate rows
for( int i=1 ; i<Length-1 ; ++i ) {
Y[vec][i] = diag_ * X[vec][i] + diag_plus_one_ * X[vec][i+1]
+ diag_minus_one_ * X[vec][i-1];
}
// final row
Y[vec][Length-1] = diag_ * X[vec][Length-1]
+ diag_minus_one_ * X[vec][Length-2];
}
return true;
}
static void UpdateComponent( spaceT const& Xh, Epetra_MultiVector& sol, Epetra_MultiVector& comp )
{
Epetra_Map componentMap ( epetraMap( Xh->template functionSpace<index>()->map() ) );
Epetra_Map globalMap ( epetraMap( Xh->map() ) );
int shift = Xh->nDofStart( index );
int Length = comp.MyLength();
for ( int i=0; i < Length; i++ )
{
int compGlobalID = componentMap.GID( i );
if ( compGlobalID >= 0 )
{
int compLocalID = componentMap.LID( compGlobalID );
int localID = globalMap.LID( compGlobalID+shift );
// int globalID = globalMap.GID(localID);
DVLOG(2) << "Copy entry component[" << compLocalID << "] to sol[" << localID << "]="
<< sol[0][localID]
<< "]\n";
sol[0][localID] = comp[0][compLocalID] ;
DVLOG(2) << comp[0][compLocalID] << "\n";
}
}
}
int EpetraSamplingOperator::Apply(const Epetra_MultiVector &X, Epetra_MultiVector &Y) const
{
TEUCHOS_ASSERT(map_.PointSameAs(X.Map()) && map_.PointSameAs(Y.Map()));
TEUCHOS_ASSERT(X.NumVectors() == Y.NumVectors());
Y.PutScalar(0.0);
for (int iVec = 0; iVec < X.NumVectors(); ++iVec) {
const ArrayView<const double> sourceVec(X[iVec], X.MyLength());
const ArrayView<double> targetVec(Y[iVec], Y.MyLength());
for (Array<GlobalIndex>::const_iterator it = sampleLIDs_.begin(), it_end = sampleLIDs_.end(); it != it_end; ++it) {
targetVec[*it] = sourceVec[*it];
}
}
return 0;
}
int DoCopyMultiVector(double** matlabApr, const Epetra_MultiVector& A) {
int ierr = 0;
int length = A.GlobalLength();
int numVectors = A.NumVectors();
const Epetra_Comm & comm = A.Map().Comm();
if (comm.MyPID()!=0) {
if (A.MyLength()!=0) ierr = -1;
}
else {
if (length!=A.MyLength()) ierr = -1;
double* matlabAvalues = *matlabApr;
double* Aptr = A.Values();
memcpy((void *)matlabAvalues, (void *)Aptr, sizeof(*Aptr) * length * numVectors);
*matlabApr += length;
}
int ierrGlobal;
comm.MinAll(&ierr, &ierrGlobal, 1); // If any processor has -1, all return -1
return(ierrGlobal);
}
void
Stokhos::EpetraMultiVectorOrthogPoly::
computeStandardDeviation(Epetra_MultiVector& v) const
{
const Teuchos::Array<double>& nrm2 = this->basis_->norm_squared();
v.PutScalar(0.0);
for (int i=1; i<this->size(); i++)
v.Multiply(nrm2[i], *coeff_[i], *coeff_[i], 1.0);
for (int j=0; j<v.NumVectors(); j++)
for (int i=0; i<v.MyLength(); i++)
v[j][i] = std::sqrt(v[j][i]);
}
// ============================================================================
int Ifpack_DiagPreconditioner::ApplyInverse(const Epetra_MultiVector& X, Epetra_MultiVector& Y) const
{
if (X.NumVectors() != Y.NumVectors())
IFPACK_CHK_ERR(-1);
for (int v = 0; v < X.NumVectors(); ++v)
for (int i = 0; i < X.MyLength(); ++i)
Y[v][i] = diag_[i] * X[v][i];
///Y.ReciprocalMultiply(1.0, diag_, X, 0.0);
return(0);
}
// Convert a Epetra_MultiVector with assumed block structure dictated by the
// vector space into a Thyra::MultiVectorBase object.
// const Teuchos::RCP<const Thyra::MultiVectorBase<double> > blockEpetraToThyra(const Epetra_MultiVector & e,const Teuchos::RCP<const Thyra::VectorSpaceBase<double> > & vs)
void blockEpetraToThyra(const Epetra_MultiVector & epetraX,const Teuchos::Ptr<Thyra::MultiVectorBase<double> > & thyraX)
{
TEUCHOS_ASSERT(thyraX->range()->dim()==epetraX.GlobalLength());
// extract local information from the Epetra_MultiVector
int leadingDim=0,numVectors=0,localDim=0;
double * epetraData=0;
epetraX.ExtractView(&epetraData,&leadingDim);
numVectors = epetraX.NumVectors();
blockEpetraToThyra(numVectors,epetraData,leadingDim,thyraX.ptr(),localDim);
TEUCHOS_ASSERT(localDim==epetraX.MyLength());
}
int InitMVValues( const Epetra_MultiVector& newb, Epetra_MultiVector* b )
{
int length = newb.MyLength();
int numVecs = newb.NumVectors();
const Epetra_Vector *tempnewvec;
Epetra_Vector *tempvec = 0;
for (int i=0; i<numVecs; ++i) {
tempnewvec = newb(i);
tempvec = (*b)(i);
for (int j=0; j<length; ++j)
(*tempvec)[j] = (*tempnewvec)[j];
}
return 0;
}
int checkMultiVectors( Epetra_MultiVector & X, Epetra_MultiVector & Y, string message = "", bool verbose = false) {
int numVectors = X.NumVectors();
int length = Y.MyLength();
int badvalue = 0;
int globalbadvalue = 0;
for (int j=0; j<numVectors; j++)
for (int i=0; i< length; i++)
if (checkValues(X[j][i], Y[j][i])==1) badvalue = 1;
X.Map().Comm().MaxAll(&badvalue, &globalbadvalue, 1);
if (verbose) {
if (globalbadvalue==0) cout << message << " check OK." << endl;
else cout << "********* " << message << " check failed.********** " << endl;
}
return(globalbadvalue);
}
//==============================================================================
int Komplex_LinearProblem::ExtractSolution(Epetra_MultiVector & Xr, Epetra_MultiVector & Xi) {
int NumMyRows = Xr.MyLength();
// Process X and B values
for (int j=0; j<Xr.NumVectors(); j++) {
double *localKX = &((*KomplexLHS_)[j][0]);
double *localXr = &(Xr[j][0]);
double *localXi = &(Xi[j][0]);
for (int i=0; i< NumMyRows; i++) {
localXr[i] = localKX[2*i];
localXi[i] = localKX[2*i+1];
}
}
return(0);
}
// ============================================================================
void EpetraExt::XMLWriter::
Write(const std::string& Label, const Epetra_MultiVector& MultiVector)
{
TEUCHOS_TEST_FOR_EXCEPTION(IsOpen_ == false, std::logic_error,
"No file has been opened");
int Length = MultiVector.GlobalLength();
int NumVectors = MultiVector.NumVectors();
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "<MultiVector Label=\"" << Label
<< "\" Length=\"" << Length << '"'
<< " NumVectors=\"" << NumVectors << '"'
<< " Type=\"double\">" << std::endl;
}
for (int iproc = 0; iproc < Comm_.NumProc(); iproc++)
{
if (iproc == Comm_.MyPID())
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of.precision(15);
for (int i = 0; i < MultiVector.MyLength(); ++i)
{
for (int j = 0; j < NumVectors; ++j)
of << std::setiosflags(std::ios::scientific) << MultiVector[j][i] << " ";
of << std::endl;
}
of.close();
}
Comm_.Barrier();
}
if (Comm_.MyPID() == 0)
{
std::ofstream of(FileName_.c_str(), std::ios::app);
of << "</MultiVector>" << std::endl;
of.close();
}
}
int ARPACKm3::reSolve(int numEigen, Epetra_MultiVector &Q, double *lambda, int startingEV) {
// Computes eigenvalues and the corresponding eigenvectors
// of the generalized eigenvalue problem
//
// K X = M X Lambda
//
// using ARPACK (mode 3).
//
// The convergence test is provided by ARPACK.
//
// Note that if M is not specified, then K X = X Lambda is solved.
// (using the mode for generalized eigenvalue problem).
//
// Input variables:
//
// numEigen (integer) = Number of eigenmodes requested
//
// Q (Epetra_MultiVector) = Initial search space
// The number of columns of Q defines the size of search space (=NCV).
// The rows of X are distributed across processors.
// As a rule of thumb in ARPACK User's guide, NCV >= 2*numEigen.
// At exit, the first numEigen locations contain the eigenvectors requested.
//
// lambda (array of doubles) = Converged eigenvalues
// The length of this array is equal to the number of columns in Q.
// At exit, the first numEigen locations contain the eigenvalues requested.
//
// startingEV (integer) = Number of eigenmodes already stored in Q
// A linear combination of these vectors is made to define the starting
// vector, placed in resid.
//
// Return information on status of computation
//
// info >= 0 >> Number of converged eigenpairs at the end of computation
//
// // Failure due to input arguments
//
// info = - 1 >> The stiffness matrix K has not been specified.
// info = - 2 >> The maps for the matrix K and the matrix M differ.
// info = - 3 >> The maps for the matrix K and the preconditioner P differ.
// info = - 4 >> The maps for the vectors and the matrix K differ.
// info = - 5 >> Q is too small for the number of eigenvalues requested.
// info = - 6 >> Q is too small for the computation parameters.
//
// info = - 8 >> numEigen must be smaller than the dimension of the matrix.
//
// info = - 30 >> MEMORY
//
// See ARPACK documentation for the meaning of INFO
if (numEigen <= startingEV) {
return numEigen;
}
int info = myVerify.inputArguments(numEigen, K, M, 0, Q, minimumSpaceDimension(numEigen));
if (info < 0)
return info;
int myPid = MyComm.MyPID();
int localSize = Q.MyLength();
int NCV = Q.NumVectors();
int knownEV = 0;
if (NCV > Q.GlobalLength()) {
if (numEigen >= Q.GlobalLength()) {
cerr << endl;
cerr << " !! The number of requested eigenvalues must be smaller than the dimension";
cerr << " of the matrix !!\n";
cerr << endl;
return -8;
}
NCV = Q.GlobalLength();
}
int localVerbose = verbose*(myPid == 0);
// Define data for ARPACK
highMem = (highMem > currentSize()) ? highMem : currentSize();
int ido = 0;
int lwI = 22 + NCV;
int *wI = new (nothrow) int[lwI];
if (wI == 0) {
return -30;
}
memRequested += sizeof(int)*lwI/(1024.0*1024.0);
int *iparam = wI;
int *ipntr = wI + 11;
int *select = wI + 22;
int lworkl = NCV*(NCV+8);
int lwD = lworkl + 4*localSize;
double *wD = new (nothrow) double[lwD];
if (wD == 0) {
delete[] wI;
return -30;
}
memRequested += sizeof(double)*(4*localSize+lworkl)/(1024.0*1024.0);
double *pointer = wD;
double *workl = pointer;
pointer = pointer + lworkl;
double *resid = pointer;
pointer = pointer + localSize;
double *workd = pointer;
double *v = Q.Values();
highMem = (highMem > currentSize()) ? highMem : currentSize();
double sigma = 0.0;
if (startingEV > 0) {
// Define the initial starting vector
memset(resid, 0, localSize*sizeof(double));
for (int jj = 0; jj < startingEV; ++jj)
for (int ii = 0; ii < localSize; ++ii)
resid[ii] += v[ii + jj*localSize];
info = 1;
}
iparam[1-1] = 1;
iparam[3-1] = maxIterEigenSolve;
iparam[7-1] = 3;
// The fourth parameter forces to use the convergence test provided by ARPACK.
// This requires a customization of ARPACK (provided by R. Lehoucq).
iparam[4-1] = 0;
Epetra_Vector v1(View, Q.Map(), workd);
Epetra_Vector v2(View, Q.Map(), workd + localSize);
Epetra_Vector v3(View, Q.Map(), workd + 2*localSize);
double *vTmp = new (nothrow) double[localSize];
if (vTmp == 0) {
delete[] wI;
delete[] wD;
return -30;
}
memRequested += sizeof(double)*localSize/(1024.0*1024.0);
highMem = (highMem > currentSize()) ? highMem : currentSize();
if (localVerbose > 0) {
cout << endl;
cout << " *|* Problem: ";
if (M)
cout << "K*Q = M*Q D ";
else
cout << "K*Q = Q D ";
cout << endl;
cout << " *|* Algorithm = ARPACK (mode 3)" << endl;
cout << " *|* Number of requested eigenvalues = " << numEigen << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
cout << " *|* Tolerance for convergence = " << tolEigenSolve << endl;
if (startingEV > 0)
cout << " *|* User-defined starting vector (Combination of " << startingEV << " vectors)\n";
cout << "\n -- Start iterations -- \n";
}
#ifdef EPETRA_MPI
Epetra_MpiComm *MPIComm = dynamic_cast<Epetra_MpiComm *>(const_cast<Epetra_Comm*>(&MyComm));
#endif
timeOuterLoop -= MyWatch.WallTime();
while (ido != 99) {
highMem = (highMem > currentSize()) ? highMem : currentSize();
#ifdef EPETRA_MPI
if (MPIComm)
callFortran.PSAUPD(MPIComm->Comm(), &ido, 'G', localSize, which, numEigen, tolEigenSolve,
resid, NCV, v, localSize, iparam, ipntr, workd, workl, lworkl, &info, localVerbose);
else
callFortran.SAUPD(&ido, 'G', localSize, which, numEigen, tolEigenSolve, resid, NCV, v,
localSize, iparam, ipntr, workd, workl, lworkl, &info, localVerbose);
#else
callFortran.SAUPD(&ido, 'G', localSize, which, numEigen, tolEigenSolve, resid, NCV, v,
localSize, iparam, ipntr, workd, workl, lworkl, &info, localVerbose);
#endif
if (ido == -1) {
// Apply the mass matrix
v3.ResetView(workd + ipntr[0] - 1);
v1.ResetView(vTmp);
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(v3, v1);
else
memcpy(v1.Values(), v3.Values(), localSize*sizeof(double));
timeMassOp += MyWatch.WallTime();
massOp += 1;
// Solve the stiffness problem
v2.ResetView(workd + ipntr[1] - 1);
timeStifOp -= MyWatch.WallTime();
K->ApplyInverse(v1, v2);
timeStifOp += MyWatch.WallTime();
stifOp += 1;
continue;
} // if (ido == -1)
if (ido == 1) {
// Solve the stiffness problem
v1.ResetView(workd + ipntr[2] - 1);
v2.ResetView(workd + ipntr[1] - 1);
timeStifOp -= MyWatch.WallTime();
K->ApplyInverse(v1, v2);
timeStifOp += MyWatch.WallTime();
stifOp += 1;
continue;
} // if (ido == 1)
if (ido == 2) {
// Apply the mass matrix
v1.ResetView(workd + ipntr[0] - 1);
v2.ResetView(workd + ipntr[1] - 1);
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(v1, v2);
else
memcpy(v2.Values(), v1.Values(), localSize*sizeof(double));
timeMassOp += MyWatch.WallTime();
massOp += 1;
continue;
} // if (ido == 2)
} // while (ido != 99)
timeOuterLoop += MyWatch.WallTime();
highMem = (highMem > currentSize()) ? highMem : currentSize();
if (info < 0) {
if (myPid == 0) {
cerr << endl;
cerr << " Error with DSAUPD, info = " << info << endl;
cerr << endl;
}
}
else {
// Compute the eigenvectors
timePostProce -= MyWatch.WallTime();
#ifdef EPETRA_MPI
if (MPIComm)
callFortran.PSEUPD(MPIComm->Comm(), 1, 'A', select, lambda, v, localSize, sigma, 'G',
localSize, which, numEigen, tolEigenSolve, resid, NCV, v, localSize, iparam, ipntr,
workd, workl, lworkl, &info);
else
callFortran.SEUPD(1, 'A', select, lambda, v, localSize, sigma, 'G', localSize, which,
numEigen, tolEigenSolve, resid, NCV, v, localSize, iparam, ipntr, workd, workl,
lworkl, &info);
#else
callFortran.SEUPD(1, 'A', select, lambda, v, localSize, sigma, 'G', localSize, which,
numEigen, tolEigenSolve, resid, NCV, v, localSize, iparam, ipntr, workd, workl,
lworkl, &info);
#endif
timePostProce += MyWatch.WallTime();
highMem = (highMem > currentSize()) ? highMem : currentSize();
// Treat the error
if (info != 0) {
if (myPid == 0) {
cerr << endl;
cerr << " Error with DSEUPD, info = " << info << endl;
cerr << endl;
}
}
} // if (info < 0)
if (info == 0) {
outerIter = iparam[3-1];
knownEV = iparam[5-1];
orthoOp = iparam[11-1];
}
delete[] wI;
delete[] wD;
delete[] vTmp;
return (info == 0) ? knownEV : info;
}
int BuildMultiVectorTests (Epetra_MultiVector & C, const double alpha,
Epetra_MultiVector& A,
Epetra_MultiVector& sqrtA,
Epetra_MultiVector& B,
Epetra_MultiVector& C_alphaA,
Epetra_MultiVector& C_alphaAplusB,
Epetra_MultiVector& C_plusB,
double* const dotvec_AB,
double* const norm1_A,
double* const norm2_sqrtA,
double* const norminf_A,
double* const normw_A,
Epetra_MultiVector& Weights,
double* const minval_A,
double* const maxval_A,
double* const meanval_A ) {
// For given values alpha and a (possibly zero) filled
// Epetra_MultiVector (the this object), allocated double * arguments dotvec_AB,
// norm1_A, and norm2_A, and allocated Epetra_MultiVectors A, sqrtA,
// B, C_alpha, C_alphaAplusB and C_plusB, this method will generate values for
// Epetra_MultiVectors A, B and all of the additional arguments on
// the list above such that, if A, B and (this) are used with the methods in
// this class, the results should match the results generated by this routine.
// Specifically, the results in dotvec_AB should match those from a call to
// A.dotProd (B,dotvec). Similarly for other routines.
int i,j;
double fi, fj; // Used for casting loop variables to floats
// Define some useful constants
int A_nrows = A.MyLength();
int A_ncols = A.NumVectors();
int sqrtA_nrows = sqrtA.MyLength();
int sqrtA_ncols = sqrtA.NumVectors();
int B_nrows = B.MyLength();
int B_ncols = B.NumVectors();
double **Ap = 0;
double **sqrtAp = 0;
double **Bp = 0;
double **Cp = 0;
double **C_alphaAp = 0;
double **C_alphaAplusBp = 0;
double **C_plusBp = 0;
double **Weightsp = 0;
A.ExtractView(&Ap);
sqrtA.ExtractView(&sqrtAp);
B.ExtractView(&Bp);
C.ExtractView(&Cp);
C_alphaA.ExtractView(&C_alphaAp);
C_alphaAplusB.ExtractView(&C_alphaAplusBp);
C_plusB.ExtractView(&C_plusBp);
Weights.ExtractView(&Weightsp);
bool A_is_local = (A.MyLength() == A.GlobalLength());
bool B_is_local = (B.MyLength() == B.GlobalLength());
bool C_is_local = (C.MyLength() == C.GlobalLength());
int A_IndexBase = A.Map().IndexBase();
int B_IndexBase = B.Map().IndexBase();
// Build two new maps that we can use for defining global equation indices below
Epetra_Map * A_Map = new Epetra_Map(-1, A_nrows, A_IndexBase, A.Map().Comm());
Epetra_Map * B_Map = new Epetra_Map(-1, B_nrows, B_IndexBase, B.Map().Comm());
int* A_MyGlobalElements = new int[A_nrows];
A_Map->MyGlobalElements(A_MyGlobalElements);
int* B_MyGlobalElements = new int[B_nrows];
B_Map->MyGlobalElements(B_MyGlobalElements);
// Check for compatible dimensions
if (C.MyLength() != A_nrows ||
A_nrows != B_nrows ||
C.NumVectors() != A_ncols ||
A_ncols != B_ncols ||
sqrtA_nrows != A_nrows ||
sqrtA_ncols != A_ncols ||
C.MyLength() != C_alphaA.MyLength() ||
C.NumVectors() != C_alphaA.NumVectors() ||
C.MyLength() != C_alphaAplusB.MyLength() ||
C.NumVectors() != C_alphaAplusB.NumVectors() ||
C.MyLength() != C_plusB.MyLength() ||
C.NumVectors() != C_plusB.NumVectors() ) return(-2); // Return error
bool Case1 = ( A_is_local && B_is_local && C_is_local); // Case 1
bool Case2 = (!A_is_local && !B_is_local && !C_is_local);// Case 2
// Test for meaningful cases
if (!(Case1 || Case2)) return(-3); // Meaningless case
/* Fill A and B with values as follows:
If A_is_local is false:
A(i,j) = A_MyGlobalElements[i]*j, i=1,...,numLocalEquations, j=1,...,NumVectors
else
A(i,j) = i*j, i=1,...,numLocalEquations, j=1,...,NumVectors
If B_is_local is false:
B(i,j) = 1/(A_MyGlobalElements[i]*j), i=1,...,numLocalEquations, j=1,...,NumVectors
else
B(i,j) = 1/(i*j), i=1,...,numLocalEquations, j=1,...,NumVectors
In addition, scale each entry by GlobalLength for A and
1/GlobalLength for B--keeps the magnitude of entries in check
*/
//Define scale factor
double sf = A.GlobalLength();
double sfinv = 1.0/sf;
// Define A
if (A_is_local)
{
for (j = 0; j <A_ncols ; j++)
{
for (i = 0; i<A_nrows; i++)
{
fi = i+1; // Get float version of i and j, offset by 1.
fj = j+1;
Ap[j][i] = (fi*sfinv)*fj;
sqrtAp[j][i] = std::sqrt(Ap[j][i]);
}
}
}
else
{
for (j = 0; j <A_ncols ; j++)
{
for (i = 0; i<A_nrows; i++)
{
fi = A_MyGlobalElements[i]+1; // Get float version of i and j, offset by 1.
fj = j+1;
Ap[j][i] = (fi*sfinv)*fj;
sqrtAp[j][i] = std::sqrt(Ap[j][i]);
}
}
}
// Define B depending on TransB and B_is_local
if (B_is_local)
{
for (j = 0; j <B_ncols ; j++)
{
for (i = 0; i<B_nrows; i++)
{
fi = i+1; // Get float version of i and j, offset by 1.
fj = j+1;
Bp[j][i] = 1.0/((fi*sfinv)*fj);
}
}
}
else
{
for (j = 0; j <B_ncols ; j++)
{
for (i = 0; i<B_nrows; i++)
{
fi = B_MyGlobalElements[i]+1; // Get float version of i and j, offset by 1.
fj = j+1;
Bp[j][i] = 1.0/((fi*sfinv)*fj);
}
}
}
// Generate C_alphaA = alpha * A
for (j = 0; j <A_ncols ; j++)
for (i = 0; i<A_nrows; i++)
C_alphaAp[j][i] = alpha * Ap[j][i];
// Generate C_alphaA = alpha * A + B
for (j = 0; j <A_ncols ; j++)
for (i = 0; i<A_nrows; i++)
C_alphaAplusBp[j][i] = alpha * Ap[j][i] + Bp[j][i];
// Generate C_plusB = this + B
for (j = 0; j <A_ncols ; j++)
for (i = 0; i<A_nrows; i++)
C_plusBp[j][i] = Cp[j][i] + Bp[j][i];
// Generate dotvec_AB. Because B(i,j) = 1/A(i,j), dotvec[i] = C.GlobalLength()
for (i=0; i< A.NumVectors(); i++) dotvec_AB[i] = C.GlobalLength();
// For the next two results we want to be careful how we do arithmetic
// to avoid very large numbers.
// We are computing sfinv*(C.GlobalLength()*(C.GlobalLength()+1)/2)
double result = C.GlobalLength();
result *= sfinv;
result /= 2.0;
result *= (double)(C.GlobalLength()+1);
// Generate norm1_A. Can use formula for sum of first n integers.
for (i=0; i< A.NumVectors(); i++)
// m1_A[i] = (i+1)*C.GlobalLength()*(C.GlobalLength()+1)/2;
norm1_A[i] = result * ((double) (i+1));
// Generate norm2_sqrtA. Can use formula for sum of first n integers.
for (i=0; i< A.NumVectors(); i++)
// norm2_sqrtA[i] = std::sqrt((double) ((i+1)*C.GlobalLength()*(C.GlobalLength()+1)/2));
norm2_sqrtA[i] = std::sqrt(result * ((double) (i+1)));
// Generate norminf_A, minval_A, maxval_A, meanval_A.
for (i=0; i< A.NumVectors(); i++)
{
norminf_A[i] = (double) (i+1);
minval_A[i] = (double) (i+1)/ (double) A.GlobalLength();
maxval_A[i] = (double) (i+1);
meanval_A[i] = norm1_A[i]/((double) (A.GlobalLength()));
}
// Define weights and expected weighted norm
for (i=0; i< A.NumVectors(); i++)
{
double ip1 = (double) i+1;
normw_A[i] = ip1;
for (j=0; j<A_nrows; j++) Weightsp[i][j] = Ap[i][j]/ip1;
}
delete A_Map;
delete B_Map;
delete [] A_MyGlobalElements;
delete [] B_MyGlobalElements;
return(0);
}
int BuildMatrixTests (Epetra_MultiVector & C,
const char TransA, const char TransB,
const double alpha,
Epetra_MultiVector& A,
Epetra_MultiVector& B,
const double beta,
Epetra_MultiVector& C_GEMM ) {
// For given values of TransA, TransB, alpha and beta, a (possibly
// zero) filled Epetra_MultiVector C, and allocated
// Epetra_MultiVectors A, B and C_GEMM this routine will generate values for
// Epetra_MultiVectors A, B and C_GEMM such that, if A, B and (this) are
// used with GEMM in this class, the results should match the results
// generated by this routine.
// Test for Strided multivectors (required for GEMM ops)
if (!A.ConstantStride() ||
!B.ConstantStride() ||
!C_GEMM.ConstantStride() ||
!C.ConstantStride()) return(-1); // Error
int i, j;
double fi, fj; // Used for casting loop variables to floats
// Get a view of the MultiVectors
double *Ap = 0;
double *Bp = 0;
double *Cp = 0;
double *C_GEMMp = 0;
int A_nrows = A.MyLength();
int A_ncols = A.NumVectors();
int B_nrows = B.MyLength();
int B_ncols = B.NumVectors();
int C_nrows = C.MyLength();
int C_ncols = C.NumVectors();
int A_Stride = 0;
int B_Stride = 0;
int C_Stride = 0;
int C_GEMM_Stride = 0;
A.ExtractView(&Ap, &A_Stride);
B.ExtractView(&Bp, &B_Stride);
C.ExtractView(&Cp, &C_Stride);
C_GEMM.ExtractView(&C_GEMMp, &C_GEMM_Stride);
// Define some useful constants
int opA_ncols = (TransA=='N') ? A.NumVectors() : A.MyLength();
int opB_nrows = (TransB=='N') ? B.MyLength() : B.NumVectors();
int C_global_inner_dim = (TransA=='N') ? A.NumVectors() : A.GlobalLength();
bool A_is_local = (!A.DistributedGlobal());
bool B_is_local = (!B.DistributedGlobal());
bool C_is_local = (!C.DistributedGlobal());
int A_IndexBase = A.Map().IndexBase();
int B_IndexBase = B.Map().IndexBase();
// Build two new maps that we can use for defining global equation indices below
Epetra_Map * A_Map = new Epetra_Map(-1, A_nrows, A_IndexBase, A.Map().Comm());
Epetra_Map * B_Map = new Epetra_Map(-1, B_nrows, B_IndexBase, B.Map().Comm());
int* A_MyGlobalElements = new int[A_nrows];
A_Map->MyGlobalElements(A_MyGlobalElements);
int* B_MyGlobalElements = new int[B_nrows];
B_Map->MyGlobalElements(B_MyGlobalElements);
// Check for compatible dimensions
if (C.MyLength() != C_nrows ||
opA_ncols != opB_nrows ||
C.NumVectors() != C_ncols ||
C.MyLength() != C_GEMM.MyLength() ||
C.NumVectors() != C_GEMM.NumVectors() ) {
delete A_Map;
delete B_Map;
delete [] A_MyGlobalElements;
delete [] B_MyGlobalElements;
return(-2); // Return error
}
bool Case1 = ( A_is_local && B_is_local && C_is_local); // Case 1 above
bool Case2 = (!A_is_local && !B_is_local && C_is_local && TransA=='T' );// Case 2
bool Case3 = (!A_is_local && B_is_local && !C_is_local && TransA=='N');// Case 3
// Test for meaningful cases
if (!(Case1 || Case2 || Case3)) {
delete A_Map;
delete B_Map;
delete [] A_MyGlobalElements;
delete [] B_MyGlobalElements;
return(-3); // Meaningless case
}
/* Fill A, B and C with values as follows:
If A_is_local is false:
A(i,j) = A_MyGlobalElements[i]*j, i=1,...,numLocalEquations, j=1,...,NumVectors
else
A(i,j) = i*j, i=1,...,numLocalEquations, j=1,...,NumVectors
If B_is_local is false:
B(i,j) = 1/(A_MyGlobalElements[i]*j), i=1,...,numLocalEquations, j=1,...,NumVectors
else
B(i,j) = 1/(i*j), i=1,...,numLocalEquations, j=1,...,NumVectors
In addition, scale each entry by GlobalLength for A and
1/GlobalLength for B--keeps the magnitude of entries in check
C_GEMM will depend on A_is_local and B_is_local. Three cases:
1) A_is_local true and B_is_local true:
C_GEMM will be local replicated and equal to A*B = i*NumVectors/j
2) A_is_local false and B_is_local false
C_GEMM will be local replicated = A(trans)*B(i,j) = i*numGlobalEquations/j
3) A_is_local false B_is_local true
C_GEMM will distributed global and equals A*B = A_MyGlobalElements[i]*NumVectors/j
*/
// Define a scalar to keep magnitude of entries reasonable
double sf = C_global_inner_dim;
double sfinv = 1.0/sf;
// Define A depending on A_is_local
if (A_is_local)
{
for (j = 0; j <A_ncols ; j++)
for (i = 0; i<A_nrows; i++)
{
fi = i+1; // Get float version of i and j, offset by 1.
fj = j+1;
Ap[i + A_Stride*j] = (fi*sfinv)*fj;
}
}
else
{
for (j = 0; j <A_ncols ; j++)
for (i = 0; i<A_nrows; i++)
{
fi = A_MyGlobalElements[i]+1; // Get float version of i and j, offset by 1.
fj = j+1;
Ap[i + A_Stride*j] = (fi*sfinv)*fj;
}
}
// Define B depending on TransB and B_is_local
if (B_is_local)
{
for (j = 0; j <B_ncols ; j++)
for (i = 0; i<B_nrows; i++)
{
fi = i+1; // Get float version of i and j, offset by 1.
fj = j+1;
Bp[i + B_Stride*j] = 1.0/((fi*sfinv)*fj);
}
}
else
{
for (j = 0; j <B_ncols ; j++)
for (i = 0; i<B_nrows; i++)
{
fi = B_MyGlobalElements[i]+1; // Get float version of i and j, offset by 1.
fj = j+1;
Bp[i + B_Stride*j] = 1.0/((fi*sfinv)*fj);
}
}
// Define C_GEMM depending on A_is_local and B_is_local. C_GEMM is also a
// function of alpha, beta, TransA, TransB:
// C_GEMM = alpha*A(TransA)*B(TransB) + beta*C_GEMM
if (Case1)
{
for (j = 0; j <C_ncols ; j++)
for (i = 0; i<C_nrows; i++)
{
// Get float version of i and j, offset by 1.
fi = (i+1)*C_global_inner_dim;
fj = j+1;
C_GEMMp[i + C_GEMM_Stride*j] = alpha * (fi/fj)
+ beta * Cp[i + C_Stride*j];
}
}
else if (Case2)
{
for (j = 0; j <C_ncols ; j++)
for (i = 0; i<C_nrows; i++)
{
// Get float version of i and j, offset by 1.
fi = (i+1)*C_global_inner_dim;
fj = j+1;
C_GEMMp[i + C_GEMM_Stride*j] = alpha * (fi/fj)
+ beta * Cp[i + C_Stride*j];
}
}
else
{
for (j = 0; j <C_ncols ; j++)
for (i = 0; i<C_nrows; i++)
{
// Get float version of i and j.
fi = (A_MyGlobalElements[i]+1)*C_global_inner_dim;
fj = j+1;
C_GEMMp[i + C_GEMM_Stride*j] = alpha * (fi/fj)
+ beta * Cp[i + C_Stride*j];
}
}
delete A_Map;
delete B_Map;
delete [] A_MyGlobalElements;
delete [] B_MyGlobalElements;
return(0);
}
//==============================================================================
int Ifpack_Hypre::ApplyInverse(const Epetra_MultiVector& X, Epetra_MultiVector& Y) const{
if(IsComputed() == false){
IFPACK_CHK_ERR(-1);
}
// These are hypre requirements
// hypre needs A, X, and Y to have the same contiguous distribution
// NOTE: Maps are only considered to be contiguous if they were generated using a
// particular constructor. Otherwise, LinearMap() will not detect whether they are
// actually contiguous.
if(!X.Map().LinearMap() || !Y.Map().LinearMap()) {
std::cerr << "ERROR: X and Y must have contiguous maps.\n";
IFPACK_CHK_ERR(-1);
}
if(!X.Map().PointSameAs(*MySimpleMap_) ||
!Y.Map().PointSameAs(*MySimpleMap_)) {
std::cerr << "ERROR: X, Y, and A must have the same distribution.\n";
IFPACK_CHK_ERR(-1);
}
Time_.ResetStartTime();
bool SameVectors = false;
int NumVectors = X.NumVectors();
if (NumVectors != Y.NumVectors()) IFPACK_CHK_ERR(-1); // X and Y must have same number of vectors
if(X.Pointers() == Y.Pointers()){
SameVectors = true;
}
for(int VecNum = 0; VecNum < NumVectors; VecNum++) {
//Get values for current vector in multivector.
// FIXME amk Nov 23, 2015: This will not work for funky data layouts
double * XValues;
IFPACK_CHK_ERR((*X(VecNum)).ExtractView(&XValues));
double * YValues;
if(!SameVectors){
IFPACK_CHK_ERR((*Y(VecNum)).ExtractView(&YValues));
} else {
YValues = new double[X.MyLength()];
}
// Temporarily make a pointer to data in Hypre for end
double *XTemp = XLocal_->data;
// Replace data in Hypre vectors with epetra values
XLocal_->data = XValues;
double *YTemp = YLocal_->data;
YLocal_->data = YValues;
IFPACK_CHK_ERR(HYPRE_ParVectorSetConstantValues(ParY_, 0.0));
if(SolveOrPrec_ == Solver){
// Use the solver methods
IFPACK_CHK_ERR(SolverSolvePtr_(Solver_, ParMatrix_, ParX_, ParY_));
} else {
// Apply the preconditioner
IFPACK_CHK_ERR(PrecondSolvePtr_(Preconditioner_, ParMatrix_, ParX_, ParY_));
}
if(SameVectors){
int NumEntries = Y.MyLength();
std::vector<double> new_values; new_values.resize(NumEntries);
std::vector<int> new_indices; new_indices.resize(NumEntries);
for(int i = 0; i < NumEntries; i++){
new_values[i] = YValues[i];
new_indices[i] = i;
}
IFPACK_CHK_ERR((*Y(VecNum)).ReplaceMyValues(NumEntries, &new_values[0], &new_indices[0]));
delete[] YValues;
}
XLocal_->data = XTemp;
YLocal_->data = YTemp;
}
NumApplyInverse_ = NumApplyInverse_ + 1;
ApplyInverseTime_ = ApplyInverseTime_ + Time_.ElapsedTime();
return 0;
} //ApplyInverse()
int ModifiedARPACKm3::reSolve(int numEigen, Epetra_MultiVector &Q, double *lambda,
int startingEV, const Epetra_MultiVector *orthoVec) {
// Computes the smallest eigenvalues and the corresponding eigenvectors
// of the generalized eigenvalue problem
//
// K X = M X Lambda
//
// using ModifiedARPACK (mode 3).
//
// The convergence test is performed outisde of ARPACK
//
// || Kx - Mx lambda || < tol*lambda
//
// The norm ||.|| can be specified by the user through the array normWeight.
// By default, the L2 Euclidean norm is used.
//
// Note that if M is not specified, then K X = X Lambda is solved.
// (using the mode for generalized eigenvalue problem).
//
// Input variables:
//
// numEigen (integer) = Number of eigenmodes requested
//
// Q (Epetra_MultiVector) = Initial search space
// The number of columns of Q defines the size of search space (=NCV).
// The rows of X are distributed across processors.
// As a rule of thumb in ARPACK User's guide, NCV >= 2*numEigen.
// At exit, the first numEigen locations contain the eigenvectors requested.
//
// lambda (array of doubles) = Converged eigenvalues
// The length of this array is equal to the number of columns in Q.
// At exit, the first numEigen locations contain the eigenvalues requested.
//
// startingEV (integer) = Number of eigenmodes already stored in Q
// A linear combination of these vectors is made to define the starting
// vector, placed in resid.
//
// orthoVec (Pointer to Epetra_MultiVector) = Space to be orthogonal to
// The computation is performed in the orthogonal of the space spanned
// by the columns vectors in orthoVec.
//
// Return information on status of computation
//
// info >= 0 >> Number of converged eigenpairs at the end of computation
//
// // Failure due to input arguments
//
// info = - 1 >> The stiffness matrix K has not been specified.
// info = - 2 >> The maps for the matrix K and the matrix M differ.
// info = - 3 >> The maps for the matrix K and the preconditioner P differ.
// info = - 4 >> The maps for the vectors and the matrix K differ.
// info = - 5 >> Q is too small for the number of eigenvalues requested.
// info = - 6 >> Q is too small for the computation parameters.
//
// info = - 8 >> numEigen must be smaller than the dimension of the matrix.
//
// info = - 30 >> MEMORY
//
// See ARPACK documentation for the meaning of INFO
if (numEigen <= startingEV) {
return numEigen;
}
int info = myVerify.inputArguments(numEigen, K, M, 0, Q, minimumSpaceDimension(numEigen));
if (info < 0)
return info;
int myPid = MyComm.MyPID();
int localSize = Q.MyLength();
int NCV = Q.NumVectors();
int knownEV = 0;
if (NCV > Q.GlobalLength()) {
if (numEigen >= Q.GlobalLength()) {
cerr << endl;
cerr << " !! The number of requested eigenvalues must be smaller than the dimension";
cerr << " of the matrix !!\n";
cerr << endl;
return -8;
}
NCV = Q.GlobalLength();
}
// Get the weight for approximating the M-inverse norm
Epetra_Vector *vectWeight = 0;
if (normWeight) {
vectWeight = new Epetra_Vector(View, Q.Map(), normWeight);
}
int localVerbose = verbose*(myPid == 0);
// Define data for ARPACK
//
// UH (10/17/03) Note that workl is also used
// * to store the eigenvectors of the tridiagonal matrix
// * as a workspace for DSTEQR
// * as a workspace for recovering the global eigenvectors
highMem = (highMem > currentSize()) ? highMem : currentSize();
int ido = 0;
int lwI = 22;
int *wI = new (nothrow) int[lwI];
if (wI == 0) {
if (vectWeight)
delete vectWeight;
return -30;
}
memRequested += sizeof(int)*lwI/(1024.0*1024.0);
int *iparam = wI;
int *ipntr = wI + 11;
int lworkl = NCV*(NCV+8);
int lwD = lworkl + 4*localSize;
double *wD = new (nothrow) double[lwD];
if (wD == 0) {
if (vectWeight)
delete vectWeight;
delete[] wI;
return -30;
}
memRequested += sizeof(double)*(4*localSize+lworkl)/(1024.0*1024.0);
double *pointer = wD;
double *workl = pointer;
pointer = pointer + lworkl;
double *resid = pointer;
pointer = pointer + localSize;
double *workd = pointer;
double *v = Q.Values();
highMem = (highMem > currentSize()) ? highMem : currentSize();
if (startingEV > 0) {
// Define the initial starting vector
memset(resid, 0, localSize*sizeof(double));
for (int jj = 0; jj < startingEV; ++jj)
for (int ii = 0; ii < localSize; ++ii)
resid[ii] += v[ii + jj*localSize];
info = 1;
}
iparam[1-1] = 1;
iparam[3-1] = maxIterEigenSolve;
iparam[7-1] = 3;
// The fourth parameter forces to use the convergence test provided by ARPACK.
// This requires a customization of ARPACK (provided by R. Lehoucq).
iparam[4-1] = 1;
Epetra_Vector v1(View, Q.Map(), workd);
Epetra_Vector v2(View, Q.Map(), workd + localSize);
Epetra_Vector v3(View, Q.Map(), workd + 2*localSize);
// Define further storage for the new residual check
// Use a block of vectors to compute the residuals more quickly.
// Note that workd could be used if memory becomes an issue.
int loopZ = (NCV > 10) ? 10 : NCV;
int lwD2 = localSize + 2*NCV-1 + NCV;
lwD2 += (M) ? 3*loopZ*localSize : 2*loopZ*localSize;
double *wD2 = new (nothrow) double[lwD2];
if (wD2 == 0) {
if (vectWeight)
delete vectWeight;
delete[] wI;
delete[] wD;
return -30;
}
memRequested += sizeof(double)*lwD2/(1024.0*1024.0);
pointer = wD2;
// vTmp is used when ido = -1
double *vTmp = pointer;
pointer = pointer + localSize;
// dd and ee stores the tridiagonal matrix.
// Note that DSTEQR destroys the contents of the input arrays.
double *dd = pointer;
pointer = pointer + NCV;
double *ee = pointer;
pointer = pointer + NCV-1;
double *vz = pointer;
pointer = pointer + loopZ*localSize;
Epetra_MultiVector approxEV(View, Q.Map(), vz, localSize, loopZ);
double *kvz = pointer;
pointer = pointer + loopZ*localSize;
Epetra_MultiVector KapproxEV(View, Q.Map(), kvz, localSize, loopZ);
double *mvz = (M) ? pointer : vz;
pointer = (M) ? pointer + loopZ*localSize : pointer;
Epetra_MultiVector MapproxEV(View, Q.Map(), mvz, localSize, loopZ);
double *normR = pointer;
// zz contains the eigenvectors of the tridiagonal matrix.
// workt is a workspace for DSTEQR.
// Note that zz and workt will use parts of workl.
double *zz, *workt;
highMem = (highMem > currentSize()) ? highMem : currentSize();
// Define an array to store the residuals history
if (localVerbose > 2) {
resHistory = new (nothrow) double[maxIterEigenSolve*NCV];
if (resHistory == 0) {
if (vectWeight)
delete vectWeight;
delete[] wI;
delete[] wD;
delete[] wD2;
return -30;
}
historyCount = 0;
}
highMem = (highMem > currentSize()) ? highMem : currentSize();
if (localVerbose > 0) {
cout << endl;
cout << " *|* Problem: ";
if (M)
cout << "K*Q = M*Q D ";
else
cout << "K*Q = Q D ";
cout << endl;
cout << " *|* Algorithm = ARPACK (Mode 3, modified such that user checks convergence)" << endl;
cout << " *|* Number of requested eigenvalues = " << numEigen << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
cout << " *|* Tolerance for convergence = " << tolEigenSolve << endl;
if (startingEV > 0)
cout << " *|* User-defined starting vector (Combination of " << startingEV << " vectors)\n";
cout << " *|* Norm used for convergence: ";
if (normWeight)
cout << "weighted L2-norm with user-provided weights" << endl;
else
cout << "L^2-norm" << endl;
if (orthoVec)
cout << " *|* Size of orthogonal subspace = " << orthoVec->NumVectors() << endl;
cout << "\n -- Start iterations -- \n";
}
#ifdef EPETRA_MPI
Epetra_MpiComm *MPIComm = dynamic_cast<Epetra_MpiComm *>(const_cast<Epetra_Comm*>(&MyComm));
#endif
timeOuterLoop -= MyWatch.WallTime();
while (ido != 99) {
highMem = (highMem > currentSize()) ? highMem : currentSize();
#ifdef EPETRA_MPI
if (MPIComm)
callFortran.PSAUPD(MPIComm->Comm(), &ido, 'G', localSize, "LM", numEigen, tolEigenSolve,
resid, NCV, v, localSize, iparam, ipntr, workd, workl, lworkl, &info, 0);
else
callFortran.SAUPD(&ido, 'G', localSize, "LM", numEigen, tolEigenSolve, resid, NCV, v,
localSize, iparam, ipntr, workd, workl, lworkl, &info, 0);
#else
callFortran.SAUPD(&ido, 'G', localSize, "LM", numEigen, tolEigenSolve, resid, NCV, v,
localSize, iparam, ipntr, workd, workl, lworkl, &info, 0);
#endif
if (ido == -1) {
// Apply the mass matrix
v3.ResetView(workd + ipntr[0] - 1);
v1.ResetView(vTmp);
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(v3, v1);
else
memcpy(v1.Values(), v3.Values(), localSize*sizeof(double));
timeMassOp += MyWatch.WallTime();
massOp += 1;
if ((orthoVec) && (verbose > 3)) {
// Check the orthogonality
double maxDot = myVerify.errorOrthogonality(orthoVec, &v1, 0);
if (myPid == 0) {
cout << " Maximum Euclidean dot product against orthogonal space (Before Solve) = ";
cout << maxDot << endl;
}
}
// Solve the stiffness problem
v2.ResetView(workd + ipntr[1] - 1);
timeStifOp -= MyWatch.WallTime();
K->ApplyInverse(v1, v2);
timeStifOp += MyWatch.WallTime();
stifOp += 1;
// Project the solution vector if needed
// Note: Use mvz as workspace
if (orthoVec) {
Epetra_Vector Mv2(View, v2.Map(), mvz);
if (M)
M->Apply(v2, Mv2);
else
memcpy(Mv2.Values(), v2.Values(), localSize*sizeof(double));
modalTool.massOrthonormalize(v2, Mv2, M, *orthoVec, 1, 1);
}
if ((orthoVec) && (verbose > 3)) {
// Check the orthogonality
double maxDot = myVerify.errorOrthogonality(orthoVec, &v2, M);
if (myPid == 0) {
cout << " Maximum M-dot product against orthogonal space (After Solve) = ";
cout << maxDot << endl;
}
}
continue;
} // if (ido == -1)
if (ido == 1) {
// Solve the stiffness problem
v1.ResetView(workd + ipntr[2] - 1);
v2.ResetView(workd + ipntr[1] - 1);
if ((orthoVec) && (verbose > 3)) {
// Check the orthogonality
double maxDot = myVerify.errorOrthogonality(orthoVec, &v1, 0);
if (myPid == 0) {
cout << " Maximum Euclidean dot product against orthogonal space (Before Solve) = ";
cout << maxDot << endl;
}
}
timeStifOp -= MyWatch.WallTime();
K->ApplyInverse(v1, v2);
timeStifOp += MyWatch.WallTime();
stifOp += 1;
// Project the solution vector if needed
// Note: Use mvz as workspace
if (orthoVec) {
Epetra_Vector Mv2(View, v2.Map(), mvz);
if (M)
M->Apply(v2, Mv2);
else
memcpy(Mv2.Values(), v2.Values(), localSize*sizeof(double));
modalTool.massOrthonormalize(v2, Mv2, M, *orthoVec, 1, 1);
}
if ((orthoVec) && (verbose > 3)) {
// Check the orthogonality
double maxDot = myVerify.errorOrthogonality(orthoVec, &v2, M);
if (myPid == 0) {
cout << " Maximum M-dot product against orthogonal space (After Solve) = ";
cout << maxDot << endl;
}
}
continue;
} // if (ido == 1)
if (ido == 2) {
// Apply the mass matrix
v1.ResetView(workd + ipntr[0] - 1);
v2.ResetView(workd + ipntr[1] - 1);
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(v1, v2);
else
memcpy(v2.Values(), v1.Values(), localSize*sizeof(double));
timeMassOp += MyWatch.WallTime();
massOp += 1;
continue;
} // if (ido == 2)
if (ido == 4) {
timeResidual -= MyWatch.WallTime();
// Copy the main diagonal of T
memcpy(dd, workl + NCV + ipntr[4] - 1, NCV*sizeof(double));
// Copy the lower diagonal of T
memcpy(ee, workl + ipntr[4], (NCV-1)*sizeof(double));
// Compute the eigenpairs of the tridiagonal matrix
zz = workl + 4*NCV;
workt = workl + 4*NCV + NCV*NCV;
callFortran.STEQR('I', NCV, dd, ee, zz, NCV, workt, &info);
if (info != 0) {
if (localVerbose > 0) {
cerr << endl;
cerr << " Error with DSTEQR, info = " << info << endl;
cerr << endl;
}
break;
}
// dd contains the eigenvalues in ascending order
// Check the residual of the proposed eigenvectors of (K, M)
int ii, jz;
iparam[4] = 0;
for (jz = 0; jz < NCV; jz += loopZ) {
int colZ = (jz + loopZ < NCV) ? loopZ : NCV - jz;
callBLAS.GEMM('N', 'N', localSize, colZ, NCV, 1.0, v, localSize,
zz + jz*NCV, NCV, 0.0, vz, localSize);
// Form the residuals
if (M)
M->Apply(approxEV, MapproxEV);
K->Apply(approxEV, KapproxEV);
for (ii = 0; ii < colZ; ++ii) {
callBLAS.AXPY(localSize, -1.0/dd[ii+jz], MapproxEV.Values() + ii*localSize,
KapproxEV.Values() + ii*localSize);
}
// Compute the norms of the residuals
if (vectWeight) {
KapproxEV.NormWeighted(*vectWeight, normR + jz);
}
else {
KapproxEV.Norm2(normR + jz);
}
// Scale the norms of residuals with the eigenvalues
for (ii = 0; ii < colZ; ++ii) {
normR[ii+jz] = normR[ii+jz]*dd[ii+jz];
}
// Put the number of converged pairs in iparam[5-1]
for (ii=0; ii<colZ; ++ii) {
if (normR[ii+jz] < tolEigenSolve)
iparam[4] += 1;
}
}
timeResidual += MyWatch.WallTime();
numResidual += NCV;
outerIter += 1;
if (localVerbose > 0) {
cout << " Iteration " << outerIter;
cout << " - Number of converged eigenvalues " << iparam[4] << endl;
}
if (localVerbose > 2) {
memcpy(resHistory + historyCount, normR, NCV*sizeof(double));
historyCount += NCV;
}
if (localVerbose > 1) {
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
for (ii=0; ii < NCV; ++ii) {
cout << " Iteration " << outerIter;
cout << " - Scaled Norm of Residual " << ii << " = " << normR[ii] << endl;
}
cout << endl;
cout.precision(2);
for (ii = 0; ii < NCV; ++ii) {
cout << " Iteration " << outerIter << " - Ritz eigenvalue " << ii;
cout.setf((fabs(dd[ii]) > 100) ? ios::scientific : ios::fixed, ios::floatfield);
cout << " = " << 1.0/dd[ii] << endl;
}
cout << endl;
}
} // if (ido == 4)
} // while (ido != 99)
timeOuterLoop += MyWatch.WallTime();
highMem = (highMem > currentSize()) ? highMem : currentSize();
if (info < 0) {
if (myPid == 0) {
cerr << endl;
cerr << " Error with DSAUPD, info = " << info << endl;
cerr << endl;
}
}
else {
// Get the eigenvalues
timePostProce -= MyWatch.WallTime();
int ii, jj;
double *pointer = workl + 4*NCV + NCV*NCV;
for (ii=0; ii < localSize; ii += 3) {
int nRow = (ii + 3 < localSize) ? 3 : localSize - ii;
for (jj=0; jj<NCV; ++jj)
memcpy(pointer + jj*nRow, v + ii + jj*localSize, nRow*sizeof(double));
callBLAS.GEMM('N', 'N', nRow, NCV, NCV, 1.0, pointer, nRow, zz, NCV,
0.0, Q.Values() + ii, localSize);
}
// Put the converged eigenpairs at the beginning
knownEV = 0;
for (ii=0; ii < NCV; ++ii) {
if (normR[ii] < tolEigenSolve) {
lambda[knownEV] = 1.0/dd[ii];
memcpy(Q.Values()+knownEV*localSize, Q.Values()+ii*localSize, localSize*sizeof(double));
knownEV += 1;
if (knownEV == Q.NumVectors())
break;
}
}
// Sort the eigenpairs
if (knownEV > 0) {
mySort.sortScalars_Vectors(knownEV, lambda, Q.Values(), localSize);
}
timePostProce += MyWatch.WallTime();
} // if (info < 0)
if (info == 0) {
orthoOp = iparam[11-1];
}
delete[] wI;
delete[] wD;
delete[] wD2;
if (vectWeight)
delete vectWeight;
return (info == 0) ? knownEV : info;
}
int BlockDACG::reSolve(int numEigen, Epetra_MultiVector &Q, double *lambda, int startingEV) {
// Computes the smallest eigenvalues and the corresponding eigenvectors
// of the generalized eigenvalue problem
//
// K X = M X Lambda
//
// using a Block Deflation Accelerated Conjugate Gradient algorithm.
//
// Note that if M is not specified, then K X = X Lambda is solved.
//
// Ref: P. Arbenz & R. Lehoucq, "A comparison of algorithms for modal analysis in the
// absence of a sparse direct method", SNL, Technical Report SAND2003-1028J
// With the notations of this report, the coefficient beta is defined as
// diag( H^T_{k} G_{k} ) / diag( H^T_{k-1} G_{k-1} )
//
// Input variables:
//
// numEigen (integer) = Number of eigenmodes requested
//
// Q (Epetra_MultiVector) = Converged eigenvectors
// The number of columns of Q must be equal to numEigen + blockSize.
// The rows of Q are distributed across processors.
// At exit, the first numEigen columns contain the eigenvectors requested.
//
// lambda (array of doubles) = Converged eigenvalues
// At input, it must be of size numEigen + blockSize.
// At exit, the first numEigen locations contain the eigenvalues requested.
//
// startingEV (integer) = Number of existing converged eigenmodes
//
// Return information on status of computation
//
// info >= 0 >> Number of converged eigenpairs at the end of computation
//
// // Failure due to input arguments
//
// info = - 1 >> The stiffness matrix K has not been specified.
// info = - 2 >> The maps for the matrix K and the matrix M differ.
// info = - 3 >> The maps for the matrix K and the preconditioner P differ.
// info = - 4 >> The maps for the vectors and the matrix K differ.
// info = - 5 >> Q is too small for the number of eigenvalues requested.
// info = - 6 >> Q is too small for the computation parameters.
//
// info = - 10 >> Failure during the mass orthonormalization
//
// info = - 20 >> Error in LAPACK during the local eigensolve
//
// info = - 30 >> MEMORY
//
// Check the input parameters
if (numEigen <= startingEV) {
return startingEV;
}
int info = myVerify.inputArguments(numEigen, K, M, Prec, Q, numEigen + blockSize);
if (info < 0)
return info;
int myPid = MyComm.MyPID();
// Get the weight for approximating the M-inverse norm
Epetra_Vector *vectWeight = 0;
if (normWeight) {
vectWeight = new Epetra_Vector(View, Q.Map(), normWeight);
}
int knownEV = startingEV;
int localVerbose = verbose*(myPid==0);
// Define local block vectors
//
// MX = Working vectors (storing M*X if M is specified, else pointing to X)
// KX = Working vectors (storing K*X)
//
// R = Residuals
//
// H = Preconditioned residuals
//
// P = Search directions
// MP = Working vectors (storing M*P if M is specified, else pointing to P)
// KP = Working vectors (storing K*P)
int xr = Q.MyLength();
Epetra_MultiVector X(View, Q, numEigen, blockSize);
X.Random();
int tmp;
tmp = (M == 0) ? 5*blockSize*xr : 7*blockSize*xr;
double *work1 = new (nothrow) double[tmp];
if (work1 == 0) {
if (vectWeight)
delete vectWeight;
info = -30;
return info;
}
memRequested += sizeof(double)*tmp/(1024.0*1024.0);
highMem = (highMem > currentSize()) ? highMem : currentSize();
double *tmpD = work1;
Epetra_MultiVector KX(View, Q.Map(), tmpD, xr, blockSize);
tmpD = tmpD + xr*blockSize;
Epetra_MultiVector MX(View, Q.Map(), (M) ? tmpD : X.Values(), xr, blockSize);
tmpD = (M) ? tmpD + xr*blockSize : tmpD;
Epetra_MultiVector R(View, Q.Map(), tmpD, xr, blockSize);
tmpD = tmpD + xr*blockSize;
Epetra_MultiVector H(View, Q.Map(), tmpD, xr, blockSize);
tmpD = tmpD + xr*blockSize;
Epetra_MultiVector P(View, Q.Map(), tmpD, xr, blockSize);
tmpD = tmpD + xr*blockSize;
Epetra_MultiVector KP(View, Q.Map(), tmpD, xr, blockSize);
tmpD = tmpD + xr*blockSize;
Epetra_MultiVector MP(View, Q.Map(), (M) ? tmpD : P.Values(), xr, blockSize);
// Define arrays
//
// theta = Store the local eigenvalues (size: 2*blockSize)
// normR = Store the norm of residuals (size: blockSize)
//
// oldHtR = Store the previous H_i^T*R_i (size: blockSize)
// currentHtR = Store the current H_i^T*R_i (size: blockSize)
//
// MM = Local mass matrix (size: 2*blockSize x 2*blockSize)
// KK = Local stiffness matrix (size: 2*blockSize x 2*blockSize)
//
// S = Local eigenvectors (size: 2*blockSize x 2*blockSize)
int lwork2;
lwork2 = 5*blockSize + 12*blockSize*blockSize;
double *work2 = new (nothrow) double[lwork2];
if (work2 == 0) {
if (vectWeight)
delete vectWeight;
delete[] work1;
info = -30;
return info;
}
highMem = (highMem > currentSize()) ? highMem : currentSize();
tmpD = work2;
double *theta = tmpD;
tmpD = tmpD + 2*blockSize;
double *normR = tmpD;
tmpD = tmpD + blockSize;
double *oldHtR = tmpD;
tmpD = tmpD + blockSize;
double *currentHtR = tmpD;
tmpD = tmpD + blockSize;
memset(currentHtR, 0, blockSize*sizeof(double));
double *MM = tmpD;
tmpD = tmpD + 4*blockSize*blockSize;
double *KK = tmpD;
tmpD = tmpD + 4*blockSize*blockSize;
double *S = tmpD;
memRequested += sizeof(double)*lwork2/(1024.0*1024.0);
// Define an array to store the residuals history
if (localVerbose > 2) {
resHistory = new (nothrow) double[maxIterEigenSolve*blockSize];
if (resHistory == 0) {
if (vectWeight)
delete vectWeight;
delete[] work1;
delete[] work2;
info = -30;
return info;
}
historyCount = 0;
}
// Miscellaneous definitions
bool reStart = false;
numRestart = 0;
int localSize;
int twoBlocks = 2*blockSize;
int nFound = blockSize;
int i, j;
if (localVerbose > 0) {
cout << endl;
cout << " *|* Problem: ";
if (M)
cout << "K*Q = M*Q D ";
else
cout << "K*Q = Q D ";
if (Prec)
cout << " with preconditioner";
cout << endl;
cout << " *|* Algorithm = DACG (block version)" << endl;
cout << " *|* Size of blocks = " << blockSize << endl;
cout << " *|* Number of requested eigenvalues = " << numEigen << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
cout << " *|* Tolerance for convergence = " << tolEigenSolve << endl;
cout << " *|* Norm used for convergence: ";
if (normWeight)
cout << "weighted L2-norm with user-provided weights" << endl;
else
cout << "L^2-norm" << endl;
if (startingEV > 0)
cout << " *|* Input converged eigenvectors = " << startingEV << endl;
cout << "\n -- Start iterations -- \n";
}
timeOuterLoop -= MyWatch.WallTime();
for (outerIter = 1; outerIter <= maxIterEigenSolve; ++outerIter) {
highMem = (highMem > currentSize()) ? highMem : currentSize();
if ((outerIter == 1) || (reStart == true)) {
reStart = false;
localSize = blockSize;
if (nFound > 0) {
Epetra_MultiVector X2(View, X, blockSize-nFound, nFound);
Epetra_MultiVector MX2(View, MX, blockSize-nFound, nFound);
Epetra_MultiVector KX2(View, KX, blockSize-nFound, nFound);
// Apply the mass matrix to X
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(X2, MX2);
timeMassOp += MyWatch.WallTime();
massOp += nFound;
if (knownEV > 0) {
// Orthonormalize X against the known eigenvectors with Gram-Schmidt
// Note: Use R as a temporary work space
Epetra_MultiVector copyQ(View, Q, 0, knownEV);
timeOrtho -= MyWatch.WallTime();
info = modalTool.massOrthonormalize(X, MX, M, copyQ, nFound, 0, R.Values());
timeOrtho += MyWatch.WallTime();
// Exit the code if the orthogonalization did not succeed
if (info < 0) {
info = -10;
delete[] work1;
delete[] work2;
if (vectWeight)
delete vectWeight;
return info;
}
}
// Apply the stiffness matrix to X
timeStifOp -= MyWatch.WallTime();
K->Apply(X2, KX2);
timeStifOp += MyWatch.WallTime();
stifOp += nFound;
} // if (nFound > 0)
} // if ((outerIter == 1) || (reStart == true))
else {
// Apply the preconditioner on the residuals
if (Prec != 0) {
timePrecOp -= MyWatch.WallTime();
Prec->ApplyInverse(R, H);
timePrecOp += MyWatch.WallTime();
precOp += blockSize;
}
else {
memcpy(H.Values(), R.Values(), xr*blockSize*sizeof(double));
}
// Compute the product H^T*R
timeSearchP -= MyWatch.WallTime();
memcpy(oldHtR, currentHtR, blockSize*sizeof(double));
H.Dot(R, currentHtR);
// Define the new search directions
if (localSize == blockSize) {
P.Scale(-1.0, H);
localSize = twoBlocks;
} // if (localSize == blockSize)
else {
bool hasZeroDot = false;
for (j = 0; j < blockSize; ++j) {
if (oldHtR[j] == 0.0) {
hasZeroDot = true;
break;
}
callBLAS.SCAL(xr, currentHtR[j]/oldHtR[j], P.Values() + j*xr);
}
if (hasZeroDot == true) {
// Restart the computation when there is a null dot product
if (localVerbose > 0) {
cout << endl;
cout << " !! Null dot product -- Restart the search space !!\n";
cout << endl;
}
if (blockSize == 1) {
X.Random();
nFound = blockSize;
}
else {
Epetra_MultiVector Xinit(View, X, j, blockSize-j);
Xinit.Random();
nFound = blockSize - j;
} // if (blockSize == 1)
reStart = true;
numRestart += 1;
info = 0;
continue;
}
callBLAS.AXPY(xr*blockSize, -1.0, H.Values(), P.Values());
} // if (localSize == blockSize)
timeSearchP += MyWatch.WallTime();
// Apply the mass matrix on P
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(P, MP);
timeMassOp += MyWatch.WallTime();
massOp += blockSize;
if (knownEV > 0) {
// Orthogonalize P against the known eigenvectors
// Note: Use R as a temporary work space
Epetra_MultiVector copyQ(View, Q, 0, knownEV);
timeOrtho -= MyWatch.WallTime();
modalTool.massOrthonormalize(P, MP, M, copyQ, blockSize, 1, R.Values());
timeOrtho += MyWatch.WallTime();
}
// Apply the stiffness matrix to P
timeStifOp -= MyWatch.WallTime();
K->Apply(P, KP);
timeStifOp += MyWatch.WallTime();
stifOp += blockSize;
} // if ((outerIter == 1) || (reStart == true))
// Form "local" mass and stiffness matrices
// Note: Use S as a temporary workspace
timeLocalProj -= MyWatch.WallTime();
modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, KX.Values(), xr,
KK, localSize, S);
modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, MX.Values(), xr,
MM, localSize, S);
if (localSize > blockSize) {
modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, KP.Values(), xr,
KK + blockSize*localSize, localSize, S);
modalTool.localProjection(blockSize, blockSize, xr, P.Values(), xr, KP.Values(), xr,
KK + blockSize*localSize + blockSize, localSize, S);
modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, MP.Values(), xr,
MM + blockSize*localSize, localSize, S);
modalTool.localProjection(blockSize, blockSize, xr, P.Values(), xr, MP.Values(), xr,
MM + blockSize*localSize + blockSize, localSize, S);
} // if (localSize > blockSize)
timeLocalProj += MyWatch.WallTime();
// Perform a spectral decomposition
timeLocalSolve -= MyWatch.WallTime();
int nevLocal = localSize;
info = modalTool.directSolver(localSize, KK, localSize, MM, localSize, nevLocal,
S, localSize, theta, localVerbose,
(blockSize == 1) ? 1: 0);
timeLocalSolve += MyWatch.WallTime();
if (info < 0) {
// Stop when spectral decomposition has a critical failure
break;
}
// Check for restarting
if ((theta[0] < 0.0) || (nevLocal < blockSize)) {
if (localVerbose > 0) {
cout << " Iteration " << outerIter;
cout << "- Failure for spectral decomposition - RESTART with new random search\n";
}
if (blockSize == 1) {
X.Random();
nFound = blockSize;
}
else {
Epetra_MultiVector Xinit(View, X, 1, blockSize-1);
Xinit.Random();
nFound = blockSize - 1;
} // if (blockSize == 1)
reStart = true;
numRestart += 1;
info = 0;
continue;
} // if ((theta[0] < 0.0) || (nevLocal < blockSize))
if ((localSize == twoBlocks) && (nevLocal == blockSize)) {
for (j = 0; j < nevLocal; ++j)
memcpy(S + j*blockSize, S + j*twoBlocks, blockSize*sizeof(double));
localSize = blockSize;
}
// Check the direction of eigenvectors
// Note: This sign check is important for convergence
for (j = 0; j < nevLocal; ++j) {
double coeff = S[j + j*localSize];
if (coeff < 0.0)
callBLAS.SCAL(localSize, -1.0, S + j*localSize);
}
// Compute the residuals
timeResidual -= MyWatch.WallTime();
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, KX.Values(), xr,
S, localSize, 0.0, R.Values(), xr);
if (localSize == twoBlocks) {
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, KP.Values(), xr,
S + blockSize, localSize, 1.0, R.Values(), xr);
}
for (j = 0; j < blockSize; ++j)
callBLAS.SCAL(localSize, theta[j], S + j*localSize);
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, -1.0, MX.Values(), xr,
S, localSize, 1.0, R.Values(), xr);
if (localSize == twoBlocks) {
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, -1.0, MP.Values(), xr,
S + blockSize, localSize, 1.0, R.Values(), xr);
}
for (j = 0; j < blockSize; ++j)
callBLAS.SCAL(localSize, 1.0/theta[j], S + j*localSize);
timeResidual += MyWatch.WallTime();
// Compute the norms of the residuals
timeNorm -= MyWatch.WallTime();
if (vectWeight)
R.NormWeighted(*vectWeight, normR);
else
R.Norm2(normR);
// Scale the norms of residuals with the eigenvalues
// Count the converged eigenvectors
nFound = 0;
for (j = 0; j < blockSize; ++j) {
normR[j] = (theta[j] == 0.0) ? normR[j] : normR[j]/theta[j];
if (normR[j] < tolEigenSolve)
nFound += 1;
}
timeNorm += MyWatch.WallTime();
// Store the residual history
if (localVerbose > 2) {
memcpy(resHistory + historyCount*blockSize, normR, blockSize*sizeof(double));
historyCount += 1;
}
// Print information on current iteration
if (localVerbose > 0) {
cout << " Iteration " << outerIter << " - Number of converged eigenvectors ";
cout << knownEV + nFound << endl;
}
if (localVerbose > 1) {
cout << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
for (i=0; i<blockSize; ++i) {
cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i;
cout << " = " << normR[i] << endl;
}
cout << endl;
cout.precision(2);
for (i=0; i<blockSize; ++i) {
cout << " Iteration " << outerIter << " - Ritz eigenvalue " << i;
cout.setf((fabs(theta[i]) < 0.01) ? ios::scientific : ios::fixed, ios::floatfield);
cout << " = " << theta[i] << endl;
}
cout << endl;
}
if (nFound == 0) {
// Update the spaces
// Note: Use H as a temporary work space
timeLocalUpdate -= MyWatch.WallTime();
memcpy(H.Values(), X.Values(), xr*blockSize*sizeof(double));
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, H.Values(), xr, S, localSize,
0.0, X.Values(), xr);
memcpy(H.Values(), KX.Values(), xr*blockSize*sizeof(double));
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, H.Values(), xr, S, localSize,
0.0, KX.Values(), xr);
if (M) {
memcpy(H.Values(), MX.Values(), xr*blockSize*sizeof(double));
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, H.Values(), xr, S, localSize,
0.0, MX.Values(), xr);
}
if (localSize == twoBlocks) {
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, P.Values(), xr,
S + blockSize, localSize, 1.0, X.Values(), xr);
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, KP.Values(), xr,
S + blockSize, localSize, 1.0, KX.Values(), xr);
if (M) {
callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, MP.Values(), xr,
S + blockSize, localSize, 1.0, MX.Values(), xr);
}
} // if (localSize == twoBlocks)
timeLocalUpdate += MyWatch.WallTime();
// When required, monitor some orthogonalities
if (verbose > 2) {
if (knownEV == 0) {
accuracyCheck(&X, &MX, &R, 0, (localSize>blockSize) ? &P : 0);
}
else {
Epetra_MultiVector copyQ(View, Q, 0, knownEV);
accuracyCheck(&X, &MX, &R, ©Q, (localSize>blockSize) ? &P : 0);
}
} // if (verbose > 2)
continue;
} // if (nFound == 0)
// Order the Ritz eigenvectors by putting the converged vectors at the beginning
int firstIndex = blockSize;
for (j = 0; j < blockSize; ++j) {
if (normR[j] >= tolEigenSolve) {
firstIndex = j;
break;
}
} // for (j = 0; j < blockSize; ++j)
while (firstIndex < nFound) {
for (j = firstIndex; j < blockSize; ++j) {
if (normR[j] < tolEigenSolve) {
// Swap the j-th and firstIndex-th position
callFortran.SWAP(localSize, S + j*localSize, 1, S + firstIndex*localSize, 1);
callFortran.SWAP(1, theta + j, 1, theta + firstIndex, 1);
callFortran.SWAP(1, normR + j, 1, normR + firstIndex, 1);
break;
}
} // for (j = firstIndex; j < blockSize; ++j)
for (j = 0; j < blockSize; ++j) {
if (normR[j] >= tolEigenSolve) {
firstIndex = j;
break;
}
} // for (j = 0; j < blockSize; ++j)
} // while (firstIndex < nFound)
// Copy the converged eigenvalues
memcpy(lambda + knownEV, theta, nFound*sizeof(double));
// Convergence test
if (knownEV + nFound >= numEigen) {
callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, X.Values(), xr,
S, localSize, 0.0, R.Values(), xr);
if (localSize > blockSize) {
callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, P.Values(), xr,
S + blockSize, localSize, 1.0, R.Values(), xr);
}
memcpy(Q.Values() + knownEV*xr, R.Values(), nFound*xr*sizeof(double));
knownEV += nFound;
if (localVerbose == 1) {
cout << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
for (i=0; i<blockSize; ++i) {
cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i;
cout << " = " << normR[i] << endl;
}
cout << endl;
}
break;
}
// Store the converged eigenvalues and eigenvectors
callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, X.Values(), xr,
S, localSize, 0.0, Q.Values() + knownEV*xr, xr);
if (localSize == twoBlocks) {
callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, P.Values(), xr,
S + blockSize, localSize, 1.0, Q.Values() + knownEV*xr, xr);
}
knownEV += nFound;
// Define the restarting vectors
timeRestart -= MyWatch.WallTime();
int leftOver = (nevLocal < blockSize + nFound) ? nevLocal - nFound : blockSize;
double *Snew = S + nFound*localSize;
memcpy(H.Values(), X.Values(), blockSize*xr*sizeof(double));
callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, H.Values(), xr,
Snew, localSize, 0.0, X.Values(), xr);
memcpy(H.Values(), KX.Values(), blockSize*xr*sizeof(double));
callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, H.Values(), xr,
Snew, localSize, 0.0, KX.Values(), xr);
if (M) {
memcpy(H.Values(), MX.Values(), blockSize*xr*sizeof(double));
callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, H.Values(), xr,
Snew, localSize, 0.0, MX.Values(), xr);
}
if (localSize == twoBlocks) {
callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, P.Values(), xr,
Snew+blockSize, localSize, 1.0, X.Values(), xr);
callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, KP.Values(), xr,
Snew+blockSize, localSize, 1.0, KX.Values(), xr);
if (M) {
callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, MP.Values(), xr,
Snew+blockSize, localSize, 1.0, MX.Values(), xr);
}
} // if (localSize == twoBlocks)
if (nevLocal < blockSize + nFound) {
// Put new random vectors at the end of the block
Epetra_MultiVector Xtmp(View, X, leftOver, blockSize - leftOver);
Xtmp.Random();
}
else {
nFound = 0;
} // if (nevLocal < blockSize + nFound)
reStart = true;
timeRestart += MyWatch.WallTime();
} // for (outerIter = 1; outerIter <= maxIterEigenSolve; ++outerIter)
timeOuterLoop += MyWatch.WallTime();
highMem = (highMem > currentSize()) ? highMem : currentSize();
// Clean memory
delete[] work1;
delete[] work2;
if (vectWeight)
delete vectWeight;
// Sort the eigenpairs
timePostProce -= MyWatch.WallTime();
if ((info == 0) && (knownEV > 0)) {
mySort.sortScalars_Vectors(knownEV, lambda, Q.Values(), Q.MyLength());
}
timePostProce += MyWatch.WallTime();
return (info == 0) ? knownEV : info;
}
int BlockPCGSolver::Solve(const Epetra_MultiVector &X, Epetra_MultiVector &Y, int blkSize) const {
int xrow = X.MyLength();
int xcol = X.NumVectors();
int ycol = Y.NumVectors();
int info = 0;
int localVerbose = verbose*(MyComm.MyPID() == 0);
double *valX = X.Values();
int NB = 3 + callLAPACK.ILAENV(1, "hetrd", "u", blkSize);
int lworkD = (blkSize > NB) ? blkSize*blkSize : NB*blkSize;
int wSize = 4*blkSize*xrow + 3*blkSize + 2*blkSize*blkSize + lworkD;
bool useY = true;
if (ycol % blkSize != 0) {
// Allocate an extra block to store the solutions
wSize += blkSize*xrow;
useY = false;
}
if (lWorkSpace < wSize) {
delete[] workSpace;
workSpace = new (std::nothrow) double[wSize];
if (workSpace == 0) {
info = -1;
return info;
}
lWorkSpace = wSize;
} // if (lWorkSpace < wSize)
double *pointer = workSpace;
// Array to store the matrix PtKP
double *PtKP = pointer;
pointer = pointer + blkSize*blkSize;
// Array to store coefficient matrices
double *coeff = pointer;
pointer = pointer + blkSize*blkSize;
// Workspace array
double *workD = pointer;
pointer = pointer + lworkD;
// Array to store the eigenvalues of P^t K P
double *da = pointer;
pointer = pointer + blkSize;
// Array to store the norms of right hand sides
double *initNorm = pointer;
pointer = pointer + blkSize;
// Array to store the norms of residuals
double *resNorm = pointer;
pointer = pointer + blkSize;
// Array to store the residuals
double *valR = pointer;
pointer = pointer + xrow*blkSize;
Epetra_MultiVector R(View, X.Map(), valR, xrow, blkSize);
// Array to store the preconditioned residuals
double *valZ = pointer;
pointer = pointer + xrow*blkSize;
Epetra_MultiVector Z(View, X.Map(), valZ, xrow, blkSize);
// Array to store the search directions
double *valP = pointer;
pointer = pointer + xrow*blkSize;
Epetra_MultiVector P(View, X.Map(), valP, xrow, blkSize);
// Array to store the image of the search directions
double *valKP = pointer;
pointer = pointer + xrow*blkSize;
Epetra_MultiVector KP(View, X.Map(), valKP, xrow, blkSize);
// Pointer to store the solutions
double *valSOL = (useY == true) ? Y.Values() : pointer;
int iRHS;
for (iRHS = 0; iRHS < xcol; iRHS += blkSize) {
int numVec = (iRHS + blkSize < xcol) ? blkSize : xcol - iRHS;
// Set the initial residuals to the right hand sides
if (numVec < blkSize) {
R.Random();
}
memcpy(valR, valX + iRHS*xrow, numVec*xrow*sizeof(double));
// Set the initial guess to zero
valSOL = (useY == true) ? Y.Values() + iRHS*xrow : valSOL;
Epetra_MultiVector SOL(View, X.Map(), valSOL, xrow, blkSize);
SOL.PutScalar(0.0);
int ii = 0;
int iter = 0;
int nFound = 0;
R.Norm2(initNorm);
if (localVerbose > 1) {
std::cout << std::endl;
std::cout << " Vectors " << iRHS << " to " << iRHS + numVec - 1 << std::endl;
if (localVerbose > 2) {
std::fprintf(stderr,"\n");
for (ii = 0; ii < numVec; ++ii) {
std::cout << " ... Initial Residual Norm " << ii << " = " << initNorm[ii] << std::endl;
}
std::cout << std::endl;
}
}
// Iteration loop
for (iter = 1; iter <= iterMax; ++iter) {
// Apply the preconditioner
if (Prec)
Prec->ApplyInverse(R, Z);
else
Z = R;
// Define the new search directions
if (iter == 1) {
P = Z;
}
else {
// Compute P^t K Z
callBLAS.GEMM(Teuchos::TRANS, Teuchos::NO_TRANS, blkSize, blkSize, xrow, 1.0, KP.Values(), xrow, Z.Values(), xrow,
0.0, workD, blkSize);
MyComm.SumAll(workD, coeff, blkSize*blkSize);
// Compute the coefficient (P^t K P)^{-1} P^t K Z
callBLAS.GEMM(Teuchos::TRANS, Teuchos::NO_TRANS, blkSize, blkSize, blkSize, 1.0, PtKP, blkSize, coeff, blkSize,
0.0, workD, blkSize);
for (ii = 0; ii < blkSize; ++ii)
callBLAS.SCAL(blkSize, da[ii], workD + ii, blkSize);
callBLAS.GEMM(Teuchos::NO_TRANS, Teuchos::NO_TRANS, blkSize, blkSize, blkSize, 1.0, PtKP, blkSize, workD, blkSize,
0.0, coeff, blkSize);
// Update the search directions
// Note: Use KP as a workspace
memcpy(KP.Values(), P.Values(), xrow*blkSize*sizeof(double));
callBLAS.GEMM(Teuchos::NO_TRANS, Teuchos::NO_TRANS, xrow, blkSize, blkSize, 1.0, KP.Values(), xrow, coeff, blkSize,
0.0, P.Values(), xrow);
P.Update(1.0, Z, -1.0);
} // if (iter == 1)
K->Apply(P, KP);
// Compute P^t K P
callBLAS.GEMM(Teuchos::TRANS, Teuchos::NO_TRANS, blkSize, blkSize, xrow, 1.0, P.Values(), xrow, KP.Values(), xrow,
0.0, workD, blkSize);
MyComm.SumAll(workD, PtKP, blkSize*blkSize);
// Eigenvalue decomposition of P^t K P
callLAPACK.SYEV('V', 'U', blkSize, PtKP, blkSize, da, workD, lworkD, &info);
if (info) {
// Break the loop as spectral decomposition failed
break;
} // if (info)
// Compute the pseudo-inverse of the eigenvalues
for (ii = 0; ii < blkSize; ++ii) {
TEUCHOS_TEST_FOR_EXCEPTION(da[ii] < 0.0, std::runtime_error, "Negative "
"eigenvalue for P^T K P: da[" << ii << "] = "
<< da[ii] << ".");
da[ii] = (da[ii] == 0.0) ? 0.0 : 1.0/da[ii];
} // for (ii = 0; ii < blkSize; ++ii)
// Compute P^t R
callBLAS.GEMM(Teuchos::TRANS, Teuchos::NO_TRANS, blkSize, blkSize, xrow, 1.0, P.Values(), xrow, R.Values(), xrow,
0.0, workD, blkSize);
MyComm.SumAll(workD, coeff, blkSize*blkSize);
// Compute the coefficient (P^t K P)^{-1} P^t R
callBLAS.GEMM(Teuchos::TRANS, Teuchos::NO_TRANS, blkSize, blkSize, blkSize, 1.0, PtKP, blkSize, coeff, blkSize,
0.0, workD, blkSize);
for (ii = 0; ii < blkSize; ++ii)
callBLAS.SCAL(blkSize, da[ii], workD + ii, blkSize);
callBLAS.GEMM(Teuchos::NO_TRANS, Teuchos::NO_TRANS, blkSize, blkSize, blkSize, 1.0, PtKP, blkSize, workD, blkSize,
0.0, coeff, blkSize);
// Update the solutions
callBLAS.GEMM(Teuchos::NO_TRANS, Teuchos::NO_TRANS, xrow, blkSize, blkSize, 1.0, P.Values(), xrow, coeff, blkSize,
1.0, valSOL, xrow);
// Update the residuals
callBLAS.GEMM(Teuchos::NO_TRANS, Teuchos::NO_TRANS, xrow, blkSize, blkSize, -1.0, KP.Values(), xrow, coeff, blkSize,
1.0, R.Values(), xrow);
// Check convergence
R.Norm2(resNorm);
nFound = 0;
for (ii = 0; ii < numVec; ++ii) {
if (resNorm[ii] <= tolCG*initNorm[ii])
nFound += 1;
}
if (localVerbose > 1) {
std::cout << " Vectors " << iRHS << " to " << iRHS + numVec - 1;
std::cout << " -- Iteration " << iter << " -- " << nFound << " converged vectors\n";
if (localVerbose > 2) {
std::cout << std::endl;
for (ii = 0; ii < numVec; ++ii) {
std::cout << " ... ";
std::cout.width(5);
std::cout << ii << " ... Residual = ";
std::cout.precision(2);
std::cout.setf(std::ios::scientific, std::ios::floatfield);
std::cout << resNorm[ii] << " ... Right Hand Side = " << initNorm[ii] << std::endl;
}
std::cout << std::endl;
}
}
if (nFound == numVec) {
break;
}
} // for (iter = 1; iter <= maxIter; ++iter)
if (useY == false) {
// Copy the solutions back into Y
memcpy(Y.Values() + xrow*iRHS, valSOL, numVec*xrow*sizeof(double));
}
numSolve += nFound;
if (nFound == numVec) {
minIter = (iter < minIter) ? iter : minIter;
maxIter = (iter > maxIter) ? iter : maxIter;
sumIter += iter;
}
} // for (iRHS = 0; iRHS < xcol; iRHS += blkSize)
return info;
}
int BlockPCGSolver::Solve(const Epetra_MultiVector &X, Epetra_MultiVector &Y) const {
int info = 0;
int localVerbose = verbose*(MyComm.MyPID() == 0);
int xr = X.MyLength();
int wSize = 3*xr;
if (lWorkSpace < wSize) {
if (workSpace)
delete[] workSpace;
workSpace = new (std::nothrow) double[wSize];
if (workSpace == 0) {
info = -1;
return info;
}
lWorkSpace = wSize;
} // if (lWorkSpace < wSize)
double *pointer = workSpace;
Epetra_Vector r(View, X.Map(), pointer);
pointer = pointer + xr;
Epetra_Vector p(View, X.Map(), pointer);
pointer = pointer + xr;
// Note: Kp and z uses the same memory space
Epetra_Vector Kp(View, X.Map(), pointer);
Epetra_Vector z(View, X.Map(), pointer);
double tmp;
double initNorm = 0.0, rNorm = 0.0, newRZ = 0.0, oldRZ = 0.0, alpha = 0.0;
double tolSquare = tolCG*tolCG;
memcpy(r.Values(), X.Values(), xr*sizeof(double));
tmp = callBLAS.DOT(xr, r.Values(), 1, r.Values(), 1);
MyComm.SumAll(&tmp, &initNorm, 1);
Y.PutScalar(0.0);
if (localVerbose > 1) {
std::cout << std::endl;
std::cout << " --- PCG Iterations --- " << std::endl;
}
int iter;
for (iter = 1; iter <= iterMax; ++iter) {
if (Prec) {
Prec->ApplyInverse(r, z);
}
else {
memcpy(z.Values(), r.Values(), xr*sizeof(double));
}
if (iter == 1) {
tmp = callBLAS.DOT(xr, r.Values(), 1, z.Values(), 1);
MyComm.SumAll(&tmp, &newRZ, 1);
memcpy(p.Values(), z.Values(), xr*sizeof(double));
}
else {
oldRZ = newRZ;
tmp = callBLAS.DOT(xr, r.Values(), 1, z.Values(), 1);
MyComm.SumAll(&tmp, &newRZ, 1);
p.Update(1.0, z, newRZ/oldRZ);
}
K->Apply(p, Kp);
tmp = callBLAS.DOT(xr, p.Values(), 1, Kp.Values(), 1);
MyComm.SumAll(&tmp, &alpha, 1);
alpha = newRZ/alpha;
TEUCHOS_TEST_FOR_EXCEPTION(alpha <= 0.0, std::runtime_error,
" !!! Non-positive value for p^TKp (" << alpha << ") !!!");
callBLAS.AXPY(xr, alpha, p.Values(), 1, Y.Values(), 1);
alpha *= -1.0;
callBLAS.AXPY(xr, alpha, Kp.Values(), 1, r.Values(), 1);
// Check convergence
tmp = callBLAS.DOT(xr, r.Values(), 1, r.Values(), 1);
MyComm.SumAll(&tmp, &rNorm, 1);
if (localVerbose > 1) {
std::cout << " Iter. " << iter;
std::cout.precision(4);
std::cout.setf(std::ios::scientific, std::ios::floatfield);
std::cout << " Residual reduction " << std::sqrt(rNorm/initNorm) << std::endl;
}
if (rNorm <= tolSquare*initNorm)
break;
} // for (iter = 1; iter <= iterMax; ++iter)
if (localVerbose == 1) {
std::cout << std::endl;
std::cout << " --- End of PCG solve ---" << std::endl;
std::cout << " Iter. " << iter;
std::cout.precision(4);
std::cout.setf(std::ios::scientific, std::ios::floatfield);
std::cout << " Residual reduction " << std::sqrt(rNorm/initNorm) << std::endl;
std::cout << std::endl;
}
if (localVerbose > 1) {
std::cout << std::endl;
}
numSolve += 1;
minIter = (iter < minIter) ? iter : minIter;
maxIter = (iter > maxIter) ? iter : maxIter;
sumIter += iter;
return info;
}
int Davidson::reSolve(int numEigen, Epetra_MultiVector &Q, double *lambda, int startingEV) {
// Computes the smallest eigenvalues and the corresponding eigenvectors
// of the generalized eigenvalue problem
//
// K X = M X Lambda
//
// using a generalized Davidson algorithm
//
// Note that if M is not specified, then K X = X Lambda is solved.
//
// Input variables:
//
// numEigen (integer) = Number of eigenmodes requested
//
// Q (Epetra_MultiVector) = Converged eigenvectors
// The number of columns of Q must be at least numEigen + blockSize.
// The rows of Q are distributed across processors.
// At exit, the first numEigen columns contain the eigenvectors requested.
//
// lambda (array of doubles) = Converged eigenvalues
// At input, it must be of size numEigen + blockSize.
// At exit, the first numEigen locations contain the eigenvalues requested.
//
// startingEV (integer) = Number of existing converged eigenvectors
// We assume that the user has check the eigenvectors and
// their M-orthonormality.
//
// Return information on status of computation
//
// info >= 0 >> Number of converged eigenpairs at the end of computation
//
// // Failure due to input arguments
//
// info = - 1 >> The stiffness matrix K has not been specified.
// info = - 2 >> The maps for the matrix K and the matrix M differ.
// info = - 3 >> The maps for the matrix K and the preconditioner P differ.
// info = - 4 >> The maps for the vectors and the matrix K differ.
// info = - 5 >> Q is too small for the number of eigenvalues requested.
// info = - 6 >> Q is too small for the computation parameters.
//
// info = - 8 >> The number of blocks is too small for the number of eigenvalues.
//
// info = - 10 >> Failure during the mass orthonormalization
//
// info = - 30 >> MEMORY
//
// Check the input parameters
if (numEigen <= startingEV) {
return startingEV;
}
int info = myVerify.inputArguments(numEigen, K, M, Prec, Q, minimumSpaceDimension(numEigen));
if (info < 0)
return info;
int myPid = MyComm.MyPID();
if (numBlock*blockSize < numEigen) {
if (myPid == 0) {
cerr << endl;
cerr << " !!! The space dimension (# of blocks x size of blocks) must be greater than ";
cerr << " the number of eigenvalues !!!\n";
cerr << " Number of blocks = " << numBlock << endl;
cerr << " Size of blocks = " << blockSize << endl;
cerr << " Number of eigenvalues = " << numEigen << endl;
cerr << endl;
}
return -8;
}
// Get the weight for approximating the M-inverse norm
Epetra_Vector *vectWeight = 0;
if (normWeight) {
vectWeight = new Epetra_Vector(View, Q.Map(), normWeight);
}
int knownEV = startingEV;
int localVerbose = verbose*(myPid==0);
// Define local block vectors
//
// MX = Working vectors (storing M*X if M is specified, else pointing to X)
// KX = Working vectors (storing K*X)
//
// R = Residuals
int xr = Q.MyLength();
int dimSearch = blockSize*numBlock;
Epetra_MultiVector X(View, Q, 0, dimSearch + blockSize);
if (knownEV > 0) {
Epetra_MultiVector copyX(View, Q, knownEV, blockSize);
copyX.Random();
}
else {
X.Random();
}
int tmp;
tmp = (M == 0) ? 2*blockSize*xr : 3*blockSize*xr;
double *work1 = new (nothrow) double[tmp];
if (work1 == 0) {
if (vectWeight)
delete vectWeight;
info = -30;
return info;
}
memRequested += sizeof(double)*tmp/(1024.0*1024.0);
highMem = (highMem > currentSize()) ? highMem : currentSize();
double *tmpD = work1;
Epetra_MultiVector KX(View, Q.Map(), tmpD, xr, blockSize);
tmpD = tmpD + xr*blockSize;
Epetra_MultiVector MX(View, Q.Map(), (M) ? tmpD : X.Values(), xr, blockSize);
tmpD = (M) ? tmpD + xr*blockSize : tmpD;
Epetra_MultiVector R(View, Q.Map(), tmpD, xr, blockSize);
// Define arrays
//
// theta = Store the local eigenvalues (size: dimSearch)
// normR = Store the norm of residuals (size: blockSize)
//
// KK = Local stiffness matrix (size: dimSearch x dimSearch)
//
// S = Local eigenvectors (size: dimSearch x dimSearch)
//
// tmpKK = Local workspace (size: blockSize x blockSize)
int lwork2 = blockSize + dimSearch + 2*dimSearch*dimSearch + blockSize*blockSize;
double *work2 = new (nothrow) double[lwork2];
if (work2 == 0) {
if (vectWeight)
delete vectWeight;
delete[] work1;
info = -30;
return info;
}
memRequested += sizeof(double)*lwork2/(1024.0*1024.0);
highMem = (highMem > currentSize()) ? highMem : currentSize();
tmpD = work2;
double *theta = tmpD;
tmpD = tmpD + dimSearch;
double *normR = tmpD;
tmpD = tmpD + blockSize;
double *KK = tmpD;
tmpD = tmpD + dimSearch*dimSearch;
memset(KK, 0, dimSearch*dimSearch*sizeof(double));
double *S = tmpD;
tmpD = tmpD + dimSearch*dimSearch;
double *tmpKK = tmpD;
// Define an array to store the residuals history
if (localVerbose > 2) {
resHistory = new (nothrow) double[maxIterEigenSolve*blockSize];
spaceSizeHistory = new (nothrow) int[maxIterEigenSolve];
if ((resHistory == 0) || (spaceSizeHistory == 0)) {
if (vectWeight)
delete vectWeight;
delete[] work1;
delete[] work2;
info = -30;
return info;
}
historyCount = 0;
}
// Miscellaneous definitions
bool reStart = false;
numRestart = 0;
bool criticalExit = false;
int bStart = 0;
int offSet = 0;
numBlock = (dimSearch/blockSize) - (knownEV/blockSize);
int nFound = blockSize;
int i, j;
if (localVerbose > 0) {
cout << endl;
cout << " *|* Problem: ";
if (M)
cout << "K*Q = M*Q D ";
else
cout << "K*Q = Q D ";
if (Prec)
cout << " with preconditioner";
cout << endl;
cout << " *|* Algorithm = Davidson algorithm (block version)" << endl;
cout << " *|* Size of blocks = " << blockSize << endl;
cout << " *|* Largest size of search space = " << numBlock*blockSize << endl;
cout << " *|* Number of requested eigenvalues = " << numEigen << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
cout << " *|* Tolerance for convergence = " << tolEigenSolve << endl;
cout << " *|* Norm used for convergence: ";
if (vectWeight)
cout << "weighted L2-norm with user-provided weights" << endl;
else
cout << "L^2-norm" << endl;
if (startingEV > 0)
cout << " *|* Input converged eigenvectors = " << startingEV << endl;
cout << "\n -- Start iterations -- \n";
}
int maxBlock = (dimSearch/blockSize) - (knownEV/blockSize);
timeOuterLoop -= MyWatch.WallTime();
outerIter = 0;
while (outerIter <= maxIterEigenSolve) {
highMem = (highMem > currentSize()) ? highMem : currentSize();
int nb;
for (nb = bStart; nb < maxBlock; ++nb) {
outerIter += 1;
if (outerIter > maxIterEigenSolve)
break;
int localSize = nb*blockSize;
Epetra_MultiVector Xcurrent(View, X, localSize + knownEV, blockSize);
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(Xcurrent, MX);
timeMassOp += MyWatch.WallTime();
massOp += blockSize;
// Orthonormalize X against the known eigenvectors and the previous vectors
// Note: Use R as a temporary work space
timeOrtho -= MyWatch.WallTime();
if (nb == bStart) {
if (nFound > 0) {
if (knownEV == 0) {
info = modalTool.massOrthonormalize(Xcurrent, MX, M, Q, nFound, 2, R.Values());
}
else {
Epetra_MultiVector copyQ(View, X, 0, knownEV + localSize);
info = modalTool.massOrthonormalize(Xcurrent, MX, M, copyQ, nFound, 0, R.Values());
}
}
nFound = 0;
}
else {
Epetra_MultiVector copyQ(View, X, 0, knownEV + localSize);
info = modalTool.massOrthonormalize(Xcurrent, MX, M, copyQ, blockSize, 0, R.Values());
}
timeOrtho += MyWatch.WallTime();
// Exit the code when the number of vectors exceeds the space dimension
if (info < 0) {
delete[] work1;
delete[] work2;
if (vectWeight)
delete vectWeight;
return -10;
}
timeStifOp -= MyWatch.WallTime();
K->Apply(Xcurrent, KX);
timeStifOp += MyWatch.WallTime();
stifOp += blockSize;
// Check the orthogonality properties of X
if (verbose > 2) {
if (knownEV + localSize == 0)
accuracyCheck(&Xcurrent, &MX, 0);
else {
Epetra_MultiVector copyQ(View, X, 0, knownEV + localSize);
accuracyCheck(&Xcurrent, &MX, ©Q);
}
if (localVerbose > 0)
cout << endl;
} // if (verbose > 2)
// Define the local stiffness matrix
// Note: S is used as a workspace
timeLocalProj -= MyWatch.WallTime();
for (j = 0; j <= nb; ++j) {
callBLAS.GEMM('T', 'N', blockSize, blockSize, xr,
1.0, X.Values()+(knownEV+j*blockSize)*xr, xr, KX.Values(), xr,
0.0, tmpKK, blockSize);
MyComm.SumAll(tmpKK, S, blockSize*blockSize);
int iC;
for (iC = 0; iC < blockSize; ++iC) {
double *Kpointer = KK + localSize*dimSearch + j*blockSize + iC*dimSearch;
memcpy(Kpointer, S + iC*blockSize, blockSize*sizeof(double));
}
}
timeLocalProj += MyWatch.WallTime();
// Perform a spectral decomposition
timeLocalSolve -= MyWatch.WallTime();
int nevLocal = localSize + blockSize;
info = modalTool.directSolver(localSize+blockSize, KK, dimSearch, 0, 0,
nevLocal, S, dimSearch, theta, localVerbose, 10);
timeLocalSolve += MyWatch.WallTime();
if (info != 0) {
// Stop as spectral decomposition has a critical failure
if (info < 0) {
criticalExit = true;
break;
}
// Restart as spectral decomposition failed
if (localVerbose > 0) {
cout << " Iteration " << outerIter;
cout << "- Failure for spectral decomposition - RESTART with new random search\n";
}
reStart = true;
numRestart += 1;
timeRestart -= MyWatch.WallTime();
Epetra_MultiVector Xinit(View, X, knownEV, blockSize);
Xinit.Random();
timeRestart += MyWatch.WallTime();
nFound = blockSize;
bStart = 0;
break;
} // if (info != 0)
// Update the search space
// Note: Use KX as a workspace
timeLocalUpdate -= MyWatch.WallTime();
callBLAS.GEMM('N', 'N', xr, blockSize, localSize+blockSize, 1.0, X.Values()+knownEV*xr, xr,
S, dimSearch, 0.0, KX.Values(), xr);
timeLocalUpdate += MyWatch.WallTime();
// Apply the mass matrix for the next block
timeMassOp -= MyWatch.WallTime();
if (M)
M->Apply(KX, MX);
timeMassOp += MyWatch.WallTime();
massOp += blockSize;
// Apply the stiffness matrix for the next block
timeStifOp -= MyWatch.WallTime();
K->Apply(KX, R);
timeStifOp += MyWatch.WallTime();
stifOp += blockSize;
// Form the residuals
timeResidual -= MyWatch.WallTime();
if (M) {
for (j = 0; j < blockSize; ++j) {
callBLAS.AXPY(xr, -theta[j], MX.Values() + j*xr, R.Values() + j*xr);
}
}
else {
// Note KX contains the updated block
for (j = 0; j < blockSize; ++j) {
callBLAS.AXPY(xr, -theta[j], KX.Values() + j*xr, R.Values() + j*xr);
}
}
timeResidual += MyWatch.WallTime();
residual += blockSize;
// Compute the norm of residuals
timeNorm -= MyWatch.WallTime();
if (vectWeight) {
R.NormWeighted(*vectWeight, normR);
}
else {
R.Norm2(normR);
}
// Scale the norms of residuals with the eigenvalues
// Count the number of converged eigenvectors
nFound = 0;
for (j = 0; j < blockSize; ++j) {
normR[j] = (theta[j] == 0.0) ? normR[j] : normR[j]/theta[j];
if (normR[j] < tolEigenSolve)
nFound += 1;
} // for (j = 0; j < blockSize; ++j)
timeNorm += MyWatch.WallTime();
// Store the residual history
if (localVerbose > 2) {
memcpy(resHistory + historyCount*blockSize, normR, blockSize*sizeof(double));
spaceSizeHistory[historyCount] = localSize + blockSize;
historyCount += 1;
}
maxSpaceSize = (maxSpaceSize > localSize+blockSize) ? maxSpaceSize : localSize+blockSize;
sumSpaceSize += localSize + blockSize;
// Print information on current iteration
if (localVerbose > 0) {
cout << " Iteration " << outerIter << " - Number of converged eigenvectors ";
cout << knownEV + nFound << endl;
} // if (localVerbose > 0)
if (localVerbose > 1) {
cout << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
for (i=0; i<blockSize; ++i) {
cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i;
cout << " = " << normR[i] << endl;
}
cout << endl;
cout.precision(2);
for (i=0; i<nevLocal; ++i) {
cout << " Iteration " << outerIter << " - Ritz eigenvalue " << i;
cout.setf((fabs(theta[i]) < 0.01) ? ios::scientific : ios::fixed, ios::floatfield);
cout << " = " << theta[i] << endl;
}
cout << endl;
}
// Exit the loop to treat the converged eigenvectors
if (nFound > 0) {
nb += 1;
offSet = 0;
break;
}
// Apply the preconditioner on the residuals
// Note: Use KX as a workspace
if (maxBlock == 1) {
if (Prec) {
timePrecOp -= MyWatch.WallTime();
Prec->ApplyInverse(R, Xcurrent);
timePrecOp += MyWatch.WallTime();
precOp += blockSize;
}
else {
memcpy(Xcurrent.Values(), R.Values(), blockSize*xr*sizeof(double));
}
timeRestart -= MyWatch.WallTime();
Xcurrent.Update(1.0, KX, -1.0);
timeRestart += MyWatch.WallTime();
break;
} // if (maxBlock == 1)
if (nb == maxBlock - 1) {
nb += 1;
break;
}
Epetra_MultiVector Xnext(View, X, knownEV+localSize+blockSize, blockSize);
if (Prec) {
timePrecOp -= MyWatch.WallTime();
Prec->ApplyInverse(R, Xnext);
timePrecOp += MyWatch.WallTime();
precOp += blockSize;
}
else {
memcpy(Xnext.Values(), R.Values(), blockSize*xr*sizeof(double));
}
} // for (nb = bStart; nb < maxBlock; ++nb)
if (outerIter > maxIterEigenSolve)
break;
if (reStart == true) {
reStart = false;
continue;
}
if (criticalExit == true)
break;
// Store the final converged eigenvectors
if (knownEV + nFound >= numEigen) {
for (j = 0; j < blockSize; ++j) {
if (normR[j] < tolEigenSolve) {
memcpy(X.Values() + knownEV*xr, KX.Values() + j*xr, xr*sizeof(double));
lambda[knownEV] = theta[j];
knownEV += 1;
}
}
if (localVerbose == 1) {
cout << endl;
cout.precision(2);
cout.setf(ios::scientific, ios::floatfield);
for (i=0; i<blockSize; ++i) {
cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i;
cout << " = " << normR[i] << endl;
}
cout << endl;
}
break;
} // if (knownEV + nFound >= numEigen)
// Treat the particular case of 1 block
if (maxBlock == 1) {
if (nFound > 0) {
double *Xpointer = X.Values() + (knownEV+nFound)*xr;
nFound = 0;
for (j = 0; j < blockSize; ++j) {
if (normR[j] < tolEigenSolve) {
memcpy(X.Values() + knownEV*xr, KX.Values() + j*xr, xr*sizeof(double));
lambda[knownEV] = theta[j];
knownEV += 1;
nFound += 1;
}
else {
memcpy(Xpointer + (j-nFound)*xr, KX.Values() + j*xr, xr*sizeof(double));
}
}
Epetra_MultiVector Xnext(View, X, knownEV + blockSize - nFound, nFound);
Xnext.Random();
}
else {
nFound = blockSize;
}
continue;
}
// Define the restarting block when maxBlock > 1
if (nFound > 0) {
int firstIndex = blockSize;
for (j = 0; j < blockSize; ++j) {
if (normR[j] >= tolEigenSolve) {
firstIndex = j;
break;
}
} // for (j = 0; j < blockSize; ++j)
while (firstIndex < nFound) {
for (j = firstIndex; j < blockSize; ++j) {
if (normR[j] < tolEigenSolve) {
// Swap the j-th and firstIndex-th position
callFortran.SWAP(nb*blockSize, S + j*dimSearch, 1, S + firstIndex*dimSearch, 1);
callFortran.SWAP(1, theta + j, 1, theta + firstIndex, 1);
callFortran.SWAP(1, normR + j, 1, normR + firstIndex, 1);
break;
}
} // for (j = firstIndex; j < blockSize; ++j)
for (j = 0; j < blockSize; ++j) {
if (normR[j] >= tolEigenSolve) {
firstIndex = j;
break;
}
} // for (j = 0; j < blockSize; ++j)
} // while (firstIndex < nFound)
// Copy the converged eigenvalues
memcpy(lambda + knownEV, theta, nFound*sizeof(double));
} // if (nFound > 0)
// Define the restarting size
bStart = ((nb - offSet) > 2) ? (nb - offSet)/2 : 0;
// Define the restarting space and local stiffness
timeRestart -= MyWatch.WallTime();
memset(KK, 0, nb*blockSize*dimSearch*sizeof(double));
for (j = 0; j < bStart*blockSize; ++j) {
KK[j + j*dimSearch] = theta[j + nFound];
}
// Form the restarting space
int oldCol = nb*blockSize;
int newCol = nFound + (bStart+1)*blockSize;
newCol = (newCol > oldCol) ? oldCol : newCol;
callFortran.GEQRF(oldCol, newCol, S, dimSearch, theta, R.Values(), xr*blockSize, &info);
callFortran.ORMQR('R', 'N', xr, oldCol, newCol, S, dimSearch, theta,
X.Values()+knownEV*xr, xr, R.Values(), blockSize*xr, &info);
timeRestart += MyWatch.WallTime();
if (nFound == 0)
offSet += 1;
knownEV += nFound;
maxBlock = (dimSearch/blockSize) - (knownEV/blockSize);
// Put random vectors if the Rayleigh Ritz vectors are not enough
newCol = nFound + (bStart+1)*blockSize;
if (newCol > oldCol) {
Epetra_MultiVector Xnext(View, X, knownEV+blockSize-nFound, nFound);
Xnext.Random();
continue;
}
nFound = 0;
} // while (outerIter <= maxIterEigenSolve)
timeOuterLoop += MyWatch.WallTime();
highMem = (highMem > currentSize()) ? highMem : currentSize();
// Clean memory
delete[] work1;
delete[] work2;
if (vectWeight)
delete vectWeight;
// Sort the eigenpairs
timePostProce -= MyWatch.WallTime();
if ((info == 0) && (knownEV > 0)) {
mySort.sortScalars_Vectors(knownEV, lambda, Q.Values(), Q.MyLength());
}
timePostProce += MyWatch.WallTime();
return (info == 0) ? knownEV : info;
}
//==============================================================================
int Ifpack_Chebyshev::
ApplyInverse(const Epetra_MultiVector& X, Epetra_MultiVector& Y) const
{
if (!IsComputed())
IFPACK_CHK_ERR(-3);
if (PolyDegree_ == 0)
return 0;
int nVec = X.NumVectors();
int len = X.MyLength();
if (nVec != Y.NumVectors())
IFPACK_CHK_ERR(-2);
Time_->ResetStartTime();
// AztecOO gives X and Y pointing to the same memory location,
// need to create an auxiliary vector, Xcopy
Teuchos::RefCountPtr<const Epetra_MultiVector> Xcopy;
if (X.Pointers()[0] == Y.Pointers()[0])
Xcopy = Teuchos::rcp( new Epetra_MultiVector(X) );
else
Xcopy = Teuchos::rcp( &X, false );
double **xPtr = 0, **yPtr = 0;
Xcopy->ExtractView(&xPtr);
Y.ExtractView(&yPtr);
#ifdef HAVE_IFPACK_EPETRAEXT
EpetraExt_PointToBlockDiagPermute* IBD=0;
if (UseBlockMode_) IBD=&*InvBlockDiagonal_;
#endif
//--- Do a quick solve when the matrix is identity
double *invDiag=0;
if(!UseBlockMode_) invDiag=InvDiagonal_->Values();
if ((LambdaMin_ == 1.0) && (LambdaMax_ == LambdaMin_)) {
#ifdef HAVE_IFPACK_EPETRAEXT
if(UseBlockMode_) IBD->ApplyInverse(*Xcopy,Y);
else
#endif
if (nVec == 1) {
double *yPointer = yPtr[0], *xPointer = xPtr[0];
for (int i = 0; i < len; ++i)
yPointer[i] = xPointer[i]*invDiag[i];
}
else {
int i, k;
for (i = 0; i < len; ++i) {
double coeff = invDiag[i];
for (k = 0; k < nVec; ++k)
yPtr[k][i] = xPtr[k][i] * coeff;
}
} // if (nVec == 1)
return 0;
} // if ((LambdaMin_ == 1.0) && (LambdaMax_ == LambdaMin_))
//--- Initialize coefficients
// Note that delta stores the inverse of ML_Cheby::delta
double alpha = LambdaMax_ / EigRatio_;
double beta = 1.1 * LambdaMax_;
double delta = 2.0 / (beta - alpha);
double theta = 0.5 * (beta + alpha);
double s1 = theta * delta;
//--- Define vectors
// In ML_Cheby, V corresponds to pAux and W to dk
Epetra_MultiVector V(X);
Epetra_MultiVector W(X);
#ifdef HAVE_IFPACK_EPETRAEXT
Epetra_MultiVector Temp(X);
#endif
double *vPointer = V.Values(), *wPointer = W.Values();
double oneOverTheta = 1.0/theta;
int i, j, k;
//--- If solving normal equations, multiply RHS by A^T
if(SolveNormalEquations_){
Apply_Transpose(Operator_,Y,V);
Y=V;
}
// Do the smoothing when block scaling is turned OFF
// --- Treat the initial guess
if (ZeroStartingSolution_ == false) {
Operator_->Apply(Y, V);
// Compute W = invDiag * ( X - V )/ Theta
#ifdef HAVE_IFPACK_EPETRAEXT
if(UseBlockMode_) {
Temp.Update(oneOverTheta,X,-oneOverTheta,V,0.0);
IBD->ApplyInverse(Temp,W);
// Perform additional matvecs for normal equations
// CMS: Testing this only in block mode FOR NOW
if(SolveNormalEquations_){
IBD->ApplyInverse(W,Temp);
Apply_Transpose(Operator_,Temp,W);
}
}
else
#endif
if (nVec == 1) {
double *xPointer = xPtr[0];
for (i = 0; i < len; ++i)
wPointer[i] = invDiag[i] * (xPointer[i] - vPointer[i]) * oneOverTheta;
}
else {
for (i = 0; i < len; ++i) {
double coeff = invDiag[i]*oneOverTheta;
double *wi = wPointer + i, *vi = vPointer + i;
for (k = 0; k < nVec; ++k) {
*wi = (xPtr[k][i] - (*vi)) * coeff;
wi = wi + len; vi = vi + len;
}
}
} // if (nVec == 1)
// Update the vector Y
Y.Update(1.0, W, 1.0);
}
else {
// Compute W = invDiag * X / Theta
#ifdef HAVE_IFPACK_EPETRAEXT
if(UseBlockMode_) {
IBD->ApplyInverse(X,W);
// Perform additional matvecs for normal equations
// CMS: Testing this only in block mode FOR NOW
if(SolveNormalEquations_){
IBD->ApplyInverse(W,Temp);
Apply_Transpose(Operator_,Temp,W);
}
W.Scale(oneOverTheta);
Y.Update(1.0, W, 0.0);
}
else
#endif
if (nVec == 1) {
double *xPointer = xPtr[0];
for (i = 0; i < len; ++i){
wPointer[i] = invDiag[i] * xPointer[i] * oneOverTheta;
}
memcpy(yPtr[0], wPointer, len*sizeof(double));
}
else {
for (i = 0; i < len; ++i) {
double coeff = invDiag[i]*oneOverTheta;
double *wi = wPointer + i;
for (k = 0; k < nVec; ++k) {
*wi = xPtr[k][i] * coeff;
wi = wi + len;
}
}
for (k = 0; k < nVec; ++k)
memcpy(yPtr[k], wPointer + k*len, len*sizeof(double));
} // if (nVec == 1)
} // if (ZeroStartingSolution_ == false)
//--- Apply the polynomial
double rhok = 1.0/s1, rhokp1;
double dtemp1, dtemp2;
int degreeMinusOne = PolyDegree_ - 1;
if (nVec == 1) {
double *xPointer = xPtr[0];
for (k = 0; k < degreeMinusOne; ++k) {
Operator_->Apply(Y, V);
rhokp1 = 1.0 / (2.0*s1 - rhok);
dtemp1 = rhokp1 * rhok;
dtemp2 = 2.0 * rhokp1 * delta;
rhok = rhokp1;
// Compute W = dtemp1 * W
W.Scale(dtemp1);
// Compute W = W + dtemp2 * invDiag * ( X - V )
#ifdef HAVE_IFPACK_EPETRAEXT
if(UseBlockMode_) {
//NTS: We can clobber V since it will be reset in the Apply
V.Update(dtemp2,X,-dtemp2);
IBD->ApplyInverse(V,Temp);
// Perform additional matvecs for normal equations
// CMS: Testing this only in block mode FOR NOW
if(SolveNormalEquations_){
IBD->ApplyInverse(V,Temp);
Apply_Transpose(Operator_,Temp,V);
}
W.Update(1.0,Temp,1.0);
}
else{
#endif
for (i = 0; i < len; ++i)
wPointer[i] += dtemp2* invDiag[i] * (xPointer[i] - vPointer[i]);
#ifdef HAVE_IFPACK_EPETRAEXT
}
#endif
// Update the vector Y
Y.Update(1.0, W, 1.0);
} // for (k = 0; k < degreeMinusOne; ++k)
}
else {
for (k = 0; k < degreeMinusOne; ++k) {
Operator_->Apply(Y, V);
rhokp1 = 1.0 / (2.0*s1 - rhok);
dtemp1 = rhokp1 * rhok;
dtemp2 = 2.0 * rhokp1 * delta;
rhok = rhokp1;
// Compute W = dtemp1 * W
W.Scale(dtemp1);
// Compute W = W + dtemp2 * invDiag * ( X - V )
#ifdef HAVE_IFPACK_EPETRAEXT
if(UseBlockMode_) {
//We can clobber V since it will be reset in the Apply
V.Update(dtemp2,X,-dtemp2);
IBD->ApplyInverse(V,Temp);
// Perform additional matvecs for normal equations
// CMS: Testing this only in block mode FOR NOW
if(SolveNormalEquations_){
IBD->ApplyInverse(V,Temp);
Apply_Transpose(Operator_,Temp,V);
}
W.Update(1.0,Temp,1.0);
}
else{
#endif
for (i = 0; i < len; ++i) {
double coeff = invDiag[i]*dtemp2;
double *wi = wPointer + i, *vi = vPointer + i;
for (j = 0; j < nVec; ++j) {
*wi += (xPtr[j][i] - (*vi)) * coeff;
wi = wi + len; vi = vi + len;
}
}
#ifdef HAVE_IFPACK_EPETRAEXT
}
#endif
// Update the vector Y
Y.Update(1.0, W, 1.0);
} // for (k = 0; k < degreeMinusOne; ++k)
} // if (nVec == 1)
// Flops are updated in each of the following.
++NumApplyInverse_;
ApplyInverseTime_ += Time_->ElapsedTime();
return(0);
}
//=======================================================
int EpetraExt_HypreIJMatrix::Solve(bool Upper, bool transpose, bool UnitDiagonal, const Epetra_MultiVector & X, Epetra_MultiVector & Y) const {
bool SameVectors = false;
int NumVectors = X.NumVectors();
if (NumVectors != Y.NumVectors()) return -1; // X and Y must have same number of vectors
if(X.Pointers() == Y.Pointers()){
SameVectors = true;
}
if(SolveOrPrec_ == Solver){
if(IsSolverSetup_[0] == false){
SetupSolver();
}
} else {
if(IsPrecondSetup_[0] == false){
SetupPrecond();
}
}
for(int VecNum = 0; VecNum < NumVectors; VecNum++) {
//Get values for current vector in multivector.
double * x_values;
EPETRA_CHK_ERR((*X(VecNum)).ExtractView(&x_values));
double * y_values;
if(!SameVectors){
EPETRA_CHK_ERR((*Y(VecNum)).ExtractView(&y_values));
} else {
y_values = new double[X.MyLength()];
}
// Temporarily make a pointer to data in Hypre for end
double *x_temp = x_local->data;
// Replace data in Hypre vectors with epetra values
x_local->data = x_values;
double *y_temp = y_local->data;
y_local->data = y_values;
EPETRA_CHK_ERR(HYPRE_ParVectorSetConstantValues(par_y, 0.0));
if(transpose && !TransposeSolve_){
// User requested a transpose solve, but the solver selected doesn't provide one
EPETRA_CHK_ERR(-1);
}
if(SolveOrPrec_ == Solver){
// Use the solver methods
EPETRA_CHK_ERR(SolverSolvePtr_(Solver_, ParMatrix_, par_x, par_y));
} else {
// Apply the preconditioner
EPETRA_CHK_ERR(PrecondSolvePtr_(Preconditioner_, ParMatrix_, par_x, par_y));
}
if(SameVectors){
int NumEntries = Y.MyLength();
std::vector<double> new_values; new_values.resize(NumEntries);
std::vector<int> new_indices; new_indices.resize(NumEntries);
for(int i = 0; i < NumEntries; i++){
new_values[i] = y_values[i];
new_indices[i] = i;
}
EPETRA_CHK_ERR((*Y(VecNum)).ReplaceMyValues(NumEntries, &new_values[0], &new_indices[0]));
delete[] y_values;
}
x_local->data = x_temp;
y_local->data = y_temp;
}
double flops = (double) NumVectors * (double) NumGlobalNonzeros();
UpdateFlops(flops);
return 0;
} //Solve()
