Task * recordQ(Task * queue, Task * task)// keep track the final queue in order to calculate other data { if(queue==NULL){ queue = copyQ(task); queue->next = NULL; } else{ queue->next = copyQ(task); queue->next->prev = queue; queue = queue->next; queue->next = NULL; } return queue; }
int BlockDACG::reSolve(int numEigen, Epetra_MultiVector &Q, double *lambda, int startingEV) { // Computes the smallest eigenvalues and the corresponding eigenvectors // of the generalized eigenvalue problem // // K X = M X Lambda // // using a Block Deflation Accelerated Conjugate Gradient algorithm. // // Note that if M is not specified, then K X = X Lambda is solved. // // Ref: P. Arbenz & R. Lehoucq, "A comparison of algorithms for modal analysis in the // absence of a sparse direct method", SNL, Technical Report SAND2003-1028J // With the notations of this report, the coefficient beta is defined as // diag( H^T_{k} G_{k} ) / diag( H^T_{k-1} G_{k-1} ) // // Input variables: // // numEigen (integer) = Number of eigenmodes requested // // Q (Epetra_MultiVector) = Converged eigenvectors // The number of columns of Q must be equal to numEigen + blockSize. // The rows of Q are distributed across processors. // At exit, the first numEigen columns contain the eigenvectors requested. // // lambda (array of doubles) = Converged eigenvalues // At input, it must be of size numEigen + blockSize. // At exit, the first numEigen locations contain the eigenvalues requested. // // startingEV (integer) = Number of existing converged eigenmodes // // Return information on status of computation // // info >= 0 >> Number of converged eigenpairs at the end of computation // // // Failure due to input arguments // // info = - 1 >> The stiffness matrix K has not been specified. // info = - 2 >> The maps for the matrix K and the matrix M differ. // info = - 3 >> The maps for the matrix K and the preconditioner P differ. // info = - 4 >> The maps for the vectors and the matrix K differ. // info = - 5 >> Q is too small for the number of eigenvalues requested. // info = - 6 >> Q is too small for the computation parameters. // // info = - 10 >> Failure during the mass orthonormalization // // info = - 20 >> Error in LAPACK during the local eigensolve // // info = - 30 >> MEMORY // // Check the input parameters if (numEigen <= startingEV) { return startingEV; } int info = myVerify.inputArguments(numEigen, K, M, Prec, Q, numEigen + blockSize); if (info < 0) return info; int myPid = MyComm.MyPID(); // Get the weight for approximating the M-inverse norm Epetra_Vector *vectWeight = 0; if (normWeight) { vectWeight = new Epetra_Vector(View, Q.Map(), normWeight); } int knownEV = startingEV; int localVerbose = verbose*(myPid==0); // Define local block vectors // // MX = Working vectors (storing M*X if M is specified, else pointing to X) // KX = Working vectors (storing K*X) // // R = Residuals // // H = Preconditioned residuals // // P = Search directions // MP = Working vectors (storing M*P if M is specified, else pointing to P) // KP = Working vectors (storing K*P) int xr = Q.MyLength(); Epetra_MultiVector X(View, Q, numEigen, blockSize); X.Random(); int tmp; tmp = (M == 0) ? 5*blockSize*xr : 7*blockSize*xr; double *work1 = new (nothrow) double[tmp]; if (work1 == 0) { if (vectWeight) delete vectWeight; info = -30; return info; } memRequested += sizeof(double)*tmp/(1024.0*1024.0); highMem = (highMem > currentSize()) ? highMem : currentSize(); double *tmpD = work1; Epetra_MultiVector KX(View, Q.Map(), tmpD, xr, blockSize); tmpD = tmpD + xr*blockSize; Epetra_MultiVector MX(View, Q.Map(), (M) ? tmpD : X.Values(), xr, blockSize); tmpD = (M) ? tmpD + xr*blockSize : tmpD; Epetra_MultiVector R(View, Q.Map(), tmpD, xr, blockSize); tmpD = tmpD + xr*blockSize; Epetra_MultiVector H(View, Q.Map(), tmpD, xr, blockSize); tmpD = tmpD + xr*blockSize; Epetra_MultiVector P(View, Q.Map(), tmpD, xr, blockSize); tmpD = tmpD + xr*blockSize; Epetra_MultiVector KP(View, Q.Map(), tmpD, xr, blockSize); tmpD = tmpD + xr*blockSize; Epetra_MultiVector MP(View, Q.Map(), (M) ? tmpD : P.Values(), xr, blockSize); // Define arrays // // theta = Store the local eigenvalues (size: 2*blockSize) // normR = Store the norm of residuals (size: blockSize) // // oldHtR = Store the previous H_i^T*R_i (size: blockSize) // currentHtR = Store the current H_i^T*R_i (size: blockSize) // // MM = Local mass matrix (size: 2*blockSize x 2*blockSize) // KK = Local stiffness matrix (size: 2*blockSize x 2*blockSize) // // S = Local eigenvectors (size: 2*blockSize x 2*blockSize) int lwork2; lwork2 = 5*blockSize + 12*blockSize*blockSize; double *work2 = new (nothrow) double[lwork2]; if (work2 == 0) { if (vectWeight) delete vectWeight; delete[] work1; info = -30; return info; } highMem = (highMem > currentSize()) ? highMem : currentSize(); tmpD = work2; double *theta = tmpD; tmpD = tmpD + 2*blockSize; double *normR = tmpD; tmpD = tmpD + blockSize; double *oldHtR = tmpD; tmpD = tmpD + blockSize; double *currentHtR = tmpD; tmpD = tmpD + blockSize; memset(currentHtR, 0, blockSize*sizeof(double)); double *MM = tmpD; tmpD = tmpD + 4*blockSize*blockSize; double *KK = tmpD; tmpD = tmpD + 4*blockSize*blockSize; double *S = tmpD; memRequested += sizeof(double)*lwork2/(1024.0*1024.0); // Define an array to store the residuals history if (localVerbose > 2) { resHistory = new (nothrow) double[maxIterEigenSolve*blockSize]; if (resHistory == 0) { if (vectWeight) delete vectWeight; delete[] work1; delete[] work2; info = -30; return info; } historyCount = 0; } // Miscellaneous definitions bool reStart = false; numRestart = 0; int localSize; int twoBlocks = 2*blockSize; int nFound = blockSize; int i, j; if (localVerbose > 0) { cout << endl; cout << " *|* Problem: "; if (M) cout << "K*Q = M*Q D "; else cout << "K*Q = Q D "; if (Prec) cout << " with preconditioner"; cout << endl; cout << " *|* Algorithm = DACG (block version)" << endl; cout << " *|* Size of blocks = " << blockSize << endl; cout << " *|* Number of requested eigenvalues = " << numEigen << endl; cout.precision(2); cout.setf(ios::scientific, ios::floatfield); cout << " *|* Tolerance for convergence = " << tolEigenSolve << endl; cout << " *|* Norm used for convergence: "; if (normWeight) cout << "weighted L2-norm with user-provided weights" << endl; else cout << "L^2-norm" << endl; if (startingEV > 0) cout << " *|* Input converged eigenvectors = " << startingEV << endl; cout << "\n -- Start iterations -- \n"; } timeOuterLoop -= MyWatch.WallTime(); for (outerIter = 1; outerIter <= maxIterEigenSolve; ++outerIter) { highMem = (highMem > currentSize()) ? highMem : currentSize(); if ((outerIter == 1) || (reStart == true)) { reStart = false; localSize = blockSize; if (nFound > 0) { Epetra_MultiVector X2(View, X, blockSize-nFound, nFound); Epetra_MultiVector MX2(View, MX, blockSize-nFound, nFound); Epetra_MultiVector KX2(View, KX, blockSize-nFound, nFound); // Apply the mass matrix to X timeMassOp -= MyWatch.WallTime(); if (M) M->Apply(X2, MX2); timeMassOp += MyWatch.WallTime(); massOp += nFound; if (knownEV > 0) { // Orthonormalize X against the known eigenvectors with Gram-Schmidt // Note: Use R as a temporary work space Epetra_MultiVector copyQ(View, Q, 0, knownEV); timeOrtho -= MyWatch.WallTime(); info = modalTool.massOrthonormalize(X, MX, M, copyQ, nFound, 0, R.Values()); timeOrtho += MyWatch.WallTime(); // Exit the code if the orthogonalization did not succeed if (info < 0) { info = -10; delete[] work1; delete[] work2; if (vectWeight) delete vectWeight; return info; } } // Apply the stiffness matrix to X timeStifOp -= MyWatch.WallTime(); K->Apply(X2, KX2); timeStifOp += MyWatch.WallTime(); stifOp += nFound; } // if (nFound > 0) } // if ((outerIter == 1) || (reStart == true)) else { // Apply the preconditioner on the residuals if (Prec != 0) { timePrecOp -= MyWatch.WallTime(); Prec->ApplyInverse(R, H); timePrecOp += MyWatch.WallTime(); precOp += blockSize; } else { memcpy(H.Values(), R.Values(), xr*blockSize*sizeof(double)); } // Compute the product H^T*R timeSearchP -= MyWatch.WallTime(); memcpy(oldHtR, currentHtR, blockSize*sizeof(double)); H.Dot(R, currentHtR); // Define the new search directions if (localSize == blockSize) { P.Scale(-1.0, H); localSize = twoBlocks; } // if (localSize == blockSize) else { bool hasZeroDot = false; for (j = 0; j < blockSize; ++j) { if (oldHtR[j] == 0.0) { hasZeroDot = true; break; } callBLAS.SCAL(xr, currentHtR[j]/oldHtR[j], P.Values() + j*xr); } if (hasZeroDot == true) { // Restart the computation when there is a null dot product if (localVerbose > 0) { cout << endl; cout << " !! Null dot product -- Restart the search space !!\n"; cout << endl; } if (blockSize == 1) { X.Random(); nFound = blockSize; } else { Epetra_MultiVector Xinit(View, X, j, blockSize-j); Xinit.Random(); nFound = blockSize - j; } // if (blockSize == 1) reStart = true; numRestart += 1; info = 0; continue; } callBLAS.AXPY(xr*blockSize, -1.0, H.Values(), P.Values()); } // if (localSize == blockSize) timeSearchP += MyWatch.WallTime(); // Apply the mass matrix on P timeMassOp -= MyWatch.WallTime(); if (M) M->Apply(P, MP); timeMassOp += MyWatch.WallTime(); massOp += blockSize; if (knownEV > 0) { // Orthogonalize P against the known eigenvectors // Note: Use R as a temporary work space Epetra_MultiVector copyQ(View, Q, 0, knownEV); timeOrtho -= MyWatch.WallTime(); modalTool.massOrthonormalize(P, MP, M, copyQ, blockSize, 1, R.Values()); timeOrtho += MyWatch.WallTime(); } // Apply the stiffness matrix to P timeStifOp -= MyWatch.WallTime(); K->Apply(P, KP); timeStifOp += MyWatch.WallTime(); stifOp += blockSize; } // if ((outerIter == 1) || (reStart == true)) // Form "local" mass and stiffness matrices // Note: Use S as a temporary workspace timeLocalProj -= MyWatch.WallTime(); modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, KX.Values(), xr, KK, localSize, S); modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, MX.Values(), xr, MM, localSize, S); if (localSize > blockSize) { modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, KP.Values(), xr, KK + blockSize*localSize, localSize, S); modalTool.localProjection(blockSize, blockSize, xr, P.Values(), xr, KP.Values(), xr, KK + blockSize*localSize + blockSize, localSize, S); modalTool.localProjection(blockSize, blockSize, xr, X.Values(), xr, MP.Values(), xr, MM + blockSize*localSize, localSize, S); modalTool.localProjection(blockSize, blockSize, xr, P.Values(), xr, MP.Values(), xr, MM + blockSize*localSize + blockSize, localSize, S); } // if (localSize > blockSize) timeLocalProj += MyWatch.WallTime(); // Perform a spectral decomposition timeLocalSolve -= MyWatch.WallTime(); int nevLocal = localSize; info = modalTool.directSolver(localSize, KK, localSize, MM, localSize, nevLocal, S, localSize, theta, localVerbose, (blockSize == 1) ? 1: 0); timeLocalSolve += MyWatch.WallTime(); if (info < 0) { // Stop when spectral decomposition has a critical failure break; } // Check for restarting if ((theta[0] < 0.0) || (nevLocal < blockSize)) { if (localVerbose > 0) { cout << " Iteration " << outerIter; cout << "- Failure for spectral decomposition - RESTART with new random search\n"; } if (blockSize == 1) { X.Random(); nFound = blockSize; } else { Epetra_MultiVector Xinit(View, X, 1, blockSize-1); Xinit.Random(); nFound = blockSize - 1; } // if (blockSize == 1) reStart = true; numRestart += 1; info = 0; continue; } // if ((theta[0] < 0.0) || (nevLocal < blockSize)) if ((localSize == twoBlocks) && (nevLocal == blockSize)) { for (j = 0; j < nevLocal; ++j) memcpy(S + j*blockSize, S + j*twoBlocks, blockSize*sizeof(double)); localSize = blockSize; } // Check the direction of eigenvectors // Note: This sign check is important for convergence for (j = 0; j < nevLocal; ++j) { double coeff = S[j + j*localSize]; if (coeff < 0.0) callBLAS.SCAL(localSize, -1.0, S + j*localSize); } // Compute the residuals timeResidual -= MyWatch.WallTime(); callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, KX.Values(), xr, S, localSize, 0.0, R.Values(), xr); if (localSize == twoBlocks) { callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, KP.Values(), xr, S + blockSize, localSize, 1.0, R.Values(), xr); } for (j = 0; j < blockSize; ++j) callBLAS.SCAL(localSize, theta[j], S + j*localSize); callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, -1.0, MX.Values(), xr, S, localSize, 1.0, R.Values(), xr); if (localSize == twoBlocks) { callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, -1.0, MP.Values(), xr, S + blockSize, localSize, 1.0, R.Values(), xr); } for (j = 0; j < blockSize; ++j) callBLAS.SCAL(localSize, 1.0/theta[j], S + j*localSize); timeResidual += MyWatch.WallTime(); // Compute the norms of the residuals timeNorm -= MyWatch.WallTime(); if (vectWeight) R.NormWeighted(*vectWeight, normR); else R.Norm2(normR); // Scale the norms of residuals with the eigenvalues // Count the converged eigenvectors nFound = 0; for (j = 0; j < blockSize; ++j) { normR[j] = (theta[j] == 0.0) ? normR[j] : normR[j]/theta[j]; if (normR[j] < tolEigenSolve) nFound += 1; } timeNorm += MyWatch.WallTime(); // Store the residual history if (localVerbose > 2) { memcpy(resHistory + historyCount*blockSize, normR, blockSize*sizeof(double)); historyCount += 1; } // Print information on current iteration if (localVerbose > 0) { cout << " Iteration " << outerIter << " - Number of converged eigenvectors "; cout << knownEV + nFound << endl; } if (localVerbose > 1) { cout << endl; cout.precision(2); cout.setf(ios::scientific, ios::floatfield); for (i=0; i<blockSize; ++i) { cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i; cout << " = " << normR[i] << endl; } cout << endl; cout.precision(2); for (i=0; i<blockSize; ++i) { cout << " Iteration " << outerIter << " - Ritz eigenvalue " << i; cout.setf((fabs(theta[i]) < 0.01) ? ios::scientific : ios::fixed, ios::floatfield); cout << " = " << theta[i] << endl; } cout << endl; } if (nFound == 0) { // Update the spaces // Note: Use H as a temporary work space timeLocalUpdate -= MyWatch.WallTime(); memcpy(H.Values(), X.Values(), xr*blockSize*sizeof(double)); callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, H.Values(), xr, S, localSize, 0.0, X.Values(), xr); memcpy(H.Values(), KX.Values(), xr*blockSize*sizeof(double)); callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, H.Values(), xr, S, localSize, 0.0, KX.Values(), xr); if (M) { memcpy(H.Values(), MX.Values(), xr*blockSize*sizeof(double)); callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, H.Values(), xr, S, localSize, 0.0, MX.Values(), xr); } if (localSize == twoBlocks) { callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, P.Values(), xr, S + blockSize, localSize, 1.0, X.Values(), xr); callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, KP.Values(), xr, S + blockSize, localSize, 1.0, KX.Values(), xr); if (M) { callBLAS.GEMM('N', 'N', xr, blockSize, blockSize, 1.0, MP.Values(), xr, S + blockSize, localSize, 1.0, MX.Values(), xr); } } // if (localSize == twoBlocks) timeLocalUpdate += MyWatch.WallTime(); // When required, monitor some orthogonalities if (verbose > 2) { if (knownEV == 0) { accuracyCheck(&X, &MX, &R, 0, (localSize>blockSize) ? &P : 0); } else { Epetra_MultiVector copyQ(View, Q, 0, knownEV); accuracyCheck(&X, &MX, &R, ©Q, (localSize>blockSize) ? &P : 0); } } // if (verbose > 2) continue; } // if (nFound == 0) // Order the Ritz eigenvectors by putting the converged vectors at the beginning int firstIndex = blockSize; for (j = 0; j < blockSize; ++j) { if (normR[j] >= tolEigenSolve) { firstIndex = j; break; } } // for (j = 0; j < blockSize; ++j) while (firstIndex < nFound) { for (j = firstIndex; j < blockSize; ++j) { if (normR[j] < tolEigenSolve) { // Swap the j-th and firstIndex-th position callFortran.SWAP(localSize, S + j*localSize, 1, S + firstIndex*localSize, 1); callFortran.SWAP(1, theta + j, 1, theta + firstIndex, 1); callFortran.SWAP(1, normR + j, 1, normR + firstIndex, 1); break; } } // for (j = firstIndex; j < blockSize; ++j) for (j = 0; j < blockSize; ++j) { if (normR[j] >= tolEigenSolve) { firstIndex = j; break; } } // for (j = 0; j < blockSize; ++j) } // while (firstIndex < nFound) // Copy the converged eigenvalues memcpy(lambda + knownEV, theta, nFound*sizeof(double)); // Convergence test if (knownEV + nFound >= numEigen) { callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, X.Values(), xr, S, localSize, 0.0, R.Values(), xr); if (localSize > blockSize) { callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, P.Values(), xr, S + blockSize, localSize, 1.0, R.Values(), xr); } memcpy(Q.Values() + knownEV*xr, R.Values(), nFound*xr*sizeof(double)); knownEV += nFound; if (localVerbose == 1) { cout << endl; cout.precision(2); cout.setf(ios::scientific, ios::floatfield); for (i=0; i<blockSize; ++i) { cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i; cout << " = " << normR[i] << endl; } cout << endl; } break; } // Store the converged eigenvalues and eigenvectors callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, X.Values(), xr, S, localSize, 0.0, Q.Values() + knownEV*xr, xr); if (localSize == twoBlocks) { callBLAS.GEMM('N', 'N', xr, nFound, blockSize, 1.0, P.Values(), xr, S + blockSize, localSize, 1.0, Q.Values() + knownEV*xr, xr); } knownEV += nFound; // Define the restarting vectors timeRestart -= MyWatch.WallTime(); int leftOver = (nevLocal < blockSize + nFound) ? nevLocal - nFound : blockSize; double *Snew = S + nFound*localSize; memcpy(H.Values(), X.Values(), blockSize*xr*sizeof(double)); callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, H.Values(), xr, Snew, localSize, 0.0, X.Values(), xr); memcpy(H.Values(), KX.Values(), blockSize*xr*sizeof(double)); callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, H.Values(), xr, Snew, localSize, 0.0, KX.Values(), xr); if (M) { memcpy(H.Values(), MX.Values(), blockSize*xr*sizeof(double)); callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, H.Values(), xr, Snew, localSize, 0.0, MX.Values(), xr); } if (localSize == twoBlocks) { callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, P.Values(), xr, Snew+blockSize, localSize, 1.0, X.Values(), xr); callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, KP.Values(), xr, Snew+blockSize, localSize, 1.0, KX.Values(), xr); if (M) { callBLAS.GEMM('N', 'N', xr, leftOver, blockSize, 1.0, MP.Values(), xr, Snew+blockSize, localSize, 1.0, MX.Values(), xr); } } // if (localSize == twoBlocks) if (nevLocal < blockSize + nFound) { // Put new random vectors at the end of the block Epetra_MultiVector Xtmp(View, X, leftOver, blockSize - leftOver); Xtmp.Random(); } else { nFound = 0; } // if (nevLocal < blockSize + nFound) reStart = true; timeRestart += MyWatch.WallTime(); } // for (outerIter = 1; outerIter <= maxIterEigenSolve; ++outerIter) timeOuterLoop += MyWatch.WallTime(); highMem = (highMem > currentSize()) ? highMem : currentSize(); // Clean memory delete[] work1; delete[] work2; if (vectWeight) delete vectWeight; // Sort the eigenpairs timePostProce -= MyWatch.WallTime(); if ((info == 0) && (knownEV > 0)) { mySort.sortScalars_Vectors(knownEV, lambda, Q.Values(), Q.MyLength()); } timePostProce += MyWatch.WallTime(); return (info == 0) ? knownEV : info; }
Task * queueup(Task * fel,Server * servers) // keep track of the task that gets in to the server and form one final queue { float currentT = 0; int remS = 64; Task * newQ = NULL; Task * currentQ0 = createdq(-1); Task * currentQ1 = createdq(-1); Task * queue = NULL; Task * temp = NULL; while(fel != NULL) { currentT = fel->artime; if(fel -> type == 0)//this is a dq event { remS++; servers = freeS(servers, currentT); while(currentQ0 != NULL) { temp = dequeue(currentQ0,remS); if (temp != NULL) { remS -= temp->nofstasks; servers = busyS(servers,temp,currentT); //set servers to work temp->timelq = currentT; queue = recordQ(queue,temp); mergedq(temp, fel ,currentT); deltask(temp); } else { break; } } while(currentQ1 != NULL) { temp = dequeue(currentQ1,remS); if (temp != NULL) { remS -= temp->nofstasks; servers = busyS(servers,temp,currentT); temp->timelq = currentT; queue = recordQ(queue,temp); mergedq(temp,fel,currentT); deltask(temp); } else { break; } } } else if (fel->type == 1)//if the head of the fel is a task { if(fel->nofstasks <= remS) { remS -= fel->nofstasks; servers = busyS(servers,fel,currentT); fel->timelq = currentT; queue = recordQ(queue,fel); mergedq(fel,fel,currentT); } else { newQ = copyQ(fel); if(fel->priority == 0) { currentQ0 = mergeFEL(currentQ0,newQ);//mergeFEL will return the head of currentQ } else { currentQ1 = mergeFEL(currentQ1,newQ); } } } temp = fel->next; deltask(fel); fel = temp; } task_destroy(findhead(currentQ0)); task_destroy(findhead(currentQ1)); return queue; }
int Davidson::reSolve(int numEigen, Epetra_MultiVector &Q, double *lambda, int startingEV) { // Computes the smallest eigenvalues and the corresponding eigenvectors // of the generalized eigenvalue problem // // K X = M X Lambda // // using a generalized Davidson algorithm // // Note that if M is not specified, then K X = X Lambda is solved. // // Input variables: // // numEigen (integer) = Number of eigenmodes requested // // Q (Epetra_MultiVector) = Converged eigenvectors // The number of columns of Q must be at least numEigen + blockSize. // The rows of Q are distributed across processors. // At exit, the first numEigen columns contain the eigenvectors requested. // // lambda (array of doubles) = Converged eigenvalues // At input, it must be of size numEigen + blockSize. // At exit, the first numEigen locations contain the eigenvalues requested. // // startingEV (integer) = Number of existing converged eigenvectors // We assume that the user has check the eigenvectors and // their M-orthonormality. // // Return information on status of computation // // info >= 0 >> Number of converged eigenpairs at the end of computation // // // Failure due to input arguments // // info = - 1 >> The stiffness matrix K has not been specified. // info = - 2 >> The maps for the matrix K and the matrix M differ. // info = - 3 >> The maps for the matrix K and the preconditioner P differ. // info = - 4 >> The maps for the vectors and the matrix K differ. // info = - 5 >> Q is too small for the number of eigenvalues requested. // info = - 6 >> Q is too small for the computation parameters. // // info = - 8 >> The number of blocks is too small for the number of eigenvalues. // // info = - 10 >> Failure during the mass orthonormalization // // info = - 30 >> MEMORY // // Check the input parameters if (numEigen <= startingEV) { return startingEV; } int info = myVerify.inputArguments(numEigen, K, M, Prec, Q, minimumSpaceDimension(numEigen)); if (info < 0) return info; int myPid = MyComm.MyPID(); if (numBlock*blockSize < numEigen) { if (myPid == 0) { cerr << endl; cerr << " !!! The space dimension (# of blocks x size of blocks) must be greater than "; cerr << " the number of eigenvalues !!!\n"; cerr << " Number of blocks = " << numBlock << endl; cerr << " Size of blocks = " << blockSize << endl; cerr << " Number of eigenvalues = " << numEigen << endl; cerr << endl; } return -8; } // Get the weight for approximating the M-inverse norm Epetra_Vector *vectWeight = 0; if (normWeight) { vectWeight = new Epetra_Vector(View, Q.Map(), normWeight); } int knownEV = startingEV; int localVerbose = verbose*(myPid==0); // Define local block vectors // // MX = Working vectors (storing M*X if M is specified, else pointing to X) // KX = Working vectors (storing K*X) // // R = Residuals int xr = Q.MyLength(); int dimSearch = blockSize*numBlock; Epetra_MultiVector X(View, Q, 0, dimSearch + blockSize); if (knownEV > 0) { Epetra_MultiVector copyX(View, Q, knownEV, blockSize); copyX.Random(); } else { X.Random(); } int tmp; tmp = (M == 0) ? 2*blockSize*xr : 3*blockSize*xr; double *work1 = new (nothrow) double[tmp]; if (work1 == 0) { if (vectWeight) delete vectWeight; info = -30; return info; } memRequested += sizeof(double)*tmp/(1024.0*1024.0); highMem = (highMem > currentSize()) ? highMem : currentSize(); double *tmpD = work1; Epetra_MultiVector KX(View, Q.Map(), tmpD, xr, blockSize); tmpD = tmpD + xr*blockSize; Epetra_MultiVector MX(View, Q.Map(), (M) ? tmpD : X.Values(), xr, blockSize); tmpD = (M) ? tmpD + xr*blockSize : tmpD; Epetra_MultiVector R(View, Q.Map(), tmpD, xr, blockSize); // Define arrays // // theta = Store the local eigenvalues (size: dimSearch) // normR = Store the norm of residuals (size: blockSize) // // KK = Local stiffness matrix (size: dimSearch x dimSearch) // // S = Local eigenvectors (size: dimSearch x dimSearch) // // tmpKK = Local workspace (size: blockSize x blockSize) int lwork2 = blockSize + dimSearch + 2*dimSearch*dimSearch + blockSize*blockSize; double *work2 = new (nothrow) double[lwork2]; if (work2 == 0) { if (vectWeight) delete vectWeight; delete[] work1; info = -30; return info; } memRequested += sizeof(double)*lwork2/(1024.0*1024.0); highMem = (highMem > currentSize()) ? highMem : currentSize(); tmpD = work2; double *theta = tmpD; tmpD = tmpD + dimSearch; double *normR = tmpD; tmpD = tmpD + blockSize; double *KK = tmpD; tmpD = tmpD + dimSearch*dimSearch; memset(KK, 0, dimSearch*dimSearch*sizeof(double)); double *S = tmpD; tmpD = tmpD + dimSearch*dimSearch; double *tmpKK = tmpD; // Define an array to store the residuals history if (localVerbose > 2) { resHistory = new (nothrow) double[maxIterEigenSolve*blockSize]; spaceSizeHistory = new (nothrow) int[maxIterEigenSolve]; if ((resHistory == 0) || (spaceSizeHistory == 0)) { if (vectWeight) delete vectWeight; delete[] work1; delete[] work2; info = -30; return info; } historyCount = 0; } // Miscellaneous definitions bool reStart = false; numRestart = 0; bool criticalExit = false; int bStart = 0; int offSet = 0; numBlock = (dimSearch/blockSize) - (knownEV/blockSize); int nFound = blockSize; int i, j; if (localVerbose > 0) { cout << endl; cout << " *|* Problem: "; if (M) cout << "K*Q = M*Q D "; else cout << "K*Q = Q D "; if (Prec) cout << " with preconditioner"; cout << endl; cout << " *|* Algorithm = Davidson algorithm (block version)" << endl; cout << " *|* Size of blocks = " << blockSize << endl; cout << " *|* Largest size of search space = " << numBlock*blockSize << endl; cout << " *|* Number of requested eigenvalues = " << numEigen << endl; cout.precision(2); cout.setf(ios::scientific, ios::floatfield); cout << " *|* Tolerance for convergence = " << tolEigenSolve << endl; cout << " *|* Norm used for convergence: "; if (vectWeight) cout << "weighted L2-norm with user-provided weights" << endl; else cout << "L^2-norm" << endl; if (startingEV > 0) cout << " *|* Input converged eigenvectors = " << startingEV << endl; cout << "\n -- Start iterations -- \n"; } int maxBlock = (dimSearch/blockSize) - (knownEV/blockSize); timeOuterLoop -= MyWatch.WallTime(); outerIter = 0; while (outerIter <= maxIterEigenSolve) { highMem = (highMem > currentSize()) ? highMem : currentSize(); int nb; for (nb = bStart; nb < maxBlock; ++nb) { outerIter += 1; if (outerIter > maxIterEigenSolve) break; int localSize = nb*blockSize; Epetra_MultiVector Xcurrent(View, X, localSize + knownEV, blockSize); timeMassOp -= MyWatch.WallTime(); if (M) M->Apply(Xcurrent, MX); timeMassOp += MyWatch.WallTime(); massOp += blockSize; // Orthonormalize X against the known eigenvectors and the previous vectors // Note: Use R as a temporary work space timeOrtho -= MyWatch.WallTime(); if (nb == bStart) { if (nFound > 0) { if (knownEV == 0) { info = modalTool.massOrthonormalize(Xcurrent, MX, M, Q, nFound, 2, R.Values()); } else { Epetra_MultiVector copyQ(View, X, 0, knownEV + localSize); info = modalTool.massOrthonormalize(Xcurrent, MX, M, copyQ, nFound, 0, R.Values()); } } nFound = 0; } else { Epetra_MultiVector copyQ(View, X, 0, knownEV + localSize); info = modalTool.massOrthonormalize(Xcurrent, MX, M, copyQ, blockSize, 0, R.Values()); } timeOrtho += MyWatch.WallTime(); // Exit the code when the number of vectors exceeds the space dimension if (info < 0) { delete[] work1; delete[] work2; if (vectWeight) delete vectWeight; return -10; } timeStifOp -= MyWatch.WallTime(); K->Apply(Xcurrent, KX); timeStifOp += MyWatch.WallTime(); stifOp += blockSize; // Check the orthogonality properties of X if (verbose > 2) { if (knownEV + localSize == 0) accuracyCheck(&Xcurrent, &MX, 0); else { Epetra_MultiVector copyQ(View, X, 0, knownEV + localSize); accuracyCheck(&Xcurrent, &MX, ©Q); } if (localVerbose > 0) cout << endl; } // if (verbose > 2) // Define the local stiffness matrix // Note: S is used as a workspace timeLocalProj -= MyWatch.WallTime(); for (j = 0; j <= nb; ++j) { callBLAS.GEMM('T', 'N', blockSize, blockSize, xr, 1.0, X.Values()+(knownEV+j*blockSize)*xr, xr, KX.Values(), xr, 0.0, tmpKK, blockSize); MyComm.SumAll(tmpKK, S, blockSize*blockSize); int iC; for (iC = 0; iC < blockSize; ++iC) { double *Kpointer = KK + localSize*dimSearch + j*blockSize + iC*dimSearch; memcpy(Kpointer, S + iC*blockSize, blockSize*sizeof(double)); } } timeLocalProj += MyWatch.WallTime(); // Perform a spectral decomposition timeLocalSolve -= MyWatch.WallTime(); int nevLocal = localSize + blockSize; info = modalTool.directSolver(localSize+blockSize, KK, dimSearch, 0, 0, nevLocal, S, dimSearch, theta, localVerbose, 10); timeLocalSolve += MyWatch.WallTime(); if (info != 0) { // Stop as spectral decomposition has a critical failure if (info < 0) { criticalExit = true; break; } // Restart as spectral decomposition failed if (localVerbose > 0) { cout << " Iteration " << outerIter; cout << "- Failure for spectral decomposition - RESTART with new random search\n"; } reStart = true; numRestart += 1; timeRestart -= MyWatch.WallTime(); Epetra_MultiVector Xinit(View, X, knownEV, blockSize); Xinit.Random(); timeRestart += MyWatch.WallTime(); nFound = blockSize; bStart = 0; break; } // if (info != 0) // Update the search space // Note: Use KX as a workspace timeLocalUpdate -= MyWatch.WallTime(); callBLAS.GEMM('N', 'N', xr, blockSize, localSize+blockSize, 1.0, X.Values()+knownEV*xr, xr, S, dimSearch, 0.0, KX.Values(), xr); timeLocalUpdate += MyWatch.WallTime(); // Apply the mass matrix for the next block timeMassOp -= MyWatch.WallTime(); if (M) M->Apply(KX, MX); timeMassOp += MyWatch.WallTime(); massOp += blockSize; // Apply the stiffness matrix for the next block timeStifOp -= MyWatch.WallTime(); K->Apply(KX, R); timeStifOp += MyWatch.WallTime(); stifOp += blockSize; // Form the residuals timeResidual -= MyWatch.WallTime(); if (M) { for (j = 0; j < blockSize; ++j) { callBLAS.AXPY(xr, -theta[j], MX.Values() + j*xr, R.Values() + j*xr); } } else { // Note KX contains the updated block for (j = 0; j < blockSize; ++j) { callBLAS.AXPY(xr, -theta[j], KX.Values() + j*xr, R.Values() + j*xr); } } timeResidual += MyWatch.WallTime(); residual += blockSize; // Compute the norm of residuals timeNorm -= MyWatch.WallTime(); if (vectWeight) { R.NormWeighted(*vectWeight, normR); } else { R.Norm2(normR); } // Scale the norms of residuals with the eigenvalues // Count the number of converged eigenvectors nFound = 0; for (j = 0; j < blockSize; ++j) { normR[j] = (theta[j] == 0.0) ? normR[j] : normR[j]/theta[j]; if (normR[j] < tolEigenSolve) nFound += 1; } // for (j = 0; j < blockSize; ++j) timeNorm += MyWatch.WallTime(); // Store the residual history if (localVerbose > 2) { memcpy(resHistory + historyCount*blockSize, normR, blockSize*sizeof(double)); spaceSizeHistory[historyCount] = localSize + blockSize; historyCount += 1; } maxSpaceSize = (maxSpaceSize > localSize+blockSize) ? maxSpaceSize : localSize+blockSize; sumSpaceSize += localSize + blockSize; // Print information on current iteration if (localVerbose > 0) { cout << " Iteration " << outerIter << " - Number of converged eigenvectors "; cout << knownEV + nFound << endl; } // if (localVerbose > 0) if (localVerbose > 1) { cout << endl; cout.precision(2); cout.setf(ios::scientific, ios::floatfield); for (i=0; i<blockSize; ++i) { cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i; cout << " = " << normR[i] << endl; } cout << endl; cout.precision(2); for (i=0; i<nevLocal; ++i) { cout << " Iteration " << outerIter << " - Ritz eigenvalue " << i; cout.setf((fabs(theta[i]) < 0.01) ? ios::scientific : ios::fixed, ios::floatfield); cout << " = " << theta[i] << endl; } cout << endl; } // Exit the loop to treat the converged eigenvectors if (nFound > 0) { nb += 1; offSet = 0; break; } // Apply the preconditioner on the residuals // Note: Use KX as a workspace if (maxBlock == 1) { if (Prec) { timePrecOp -= MyWatch.WallTime(); Prec->ApplyInverse(R, Xcurrent); timePrecOp += MyWatch.WallTime(); precOp += blockSize; } else { memcpy(Xcurrent.Values(), R.Values(), blockSize*xr*sizeof(double)); } timeRestart -= MyWatch.WallTime(); Xcurrent.Update(1.0, KX, -1.0); timeRestart += MyWatch.WallTime(); break; } // if (maxBlock == 1) if (nb == maxBlock - 1) { nb += 1; break; } Epetra_MultiVector Xnext(View, X, knownEV+localSize+blockSize, blockSize); if (Prec) { timePrecOp -= MyWatch.WallTime(); Prec->ApplyInverse(R, Xnext); timePrecOp += MyWatch.WallTime(); precOp += blockSize; } else { memcpy(Xnext.Values(), R.Values(), blockSize*xr*sizeof(double)); } } // for (nb = bStart; nb < maxBlock; ++nb) if (outerIter > maxIterEigenSolve) break; if (reStart == true) { reStart = false; continue; } if (criticalExit == true) break; // Store the final converged eigenvectors if (knownEV + nFound >= numEigen) { for (j = 0; j < blockSize; ++j) { if (normR[j] < tolEigenSolve) { memcpy(X.Values() + knownEV*xr, KX.Values() + j*xr, xr*sizeof(double)); lambda[knownEV] = theta[j]; knownEV += 1; } } if (localVerbose == 1) { cout << endl; cout.precision(2); cout.setf(ios::scientific, ios::floatfield); for (i=0; i<blockSize; ++i) { cout << " Iteration " << outerIter << " - Scaled Norm of Residual " << i; cout << " = " << normR[i] << endl; } cout << endl; } break; } // if (knownEV + nFound >= numEigen) // Treat the particular case of 1 block if (maxBlock == 1) { if (nFound > 0) { double *Xpointer = X.Values() + (knownEV+nFound)*xr; nFound = 0; for (j = 0; j < blockSize; ++j) { if (normR[j] < tolEigenSolve) { memcpy(X.Values() + knownEV*xr, KX.Values() + j*xr, xr*sizeof(double)); lambda[knownEV] = theta[j]; knownEV += 1; nFound += 1; } else { memcpy(Xpointer + (j-nFound)*xr, KX.Values() + j*xr, xr*sizeof(double)); } } Epetra_MultiVector Xnext(View, X, knownEV + blockSize - nFound, nFound); Xnext.Random(); } else { nFound = blockSize; } continue; } // Define the restarting block when maxBlock > 1 if (nFound > 0) { int firstIndex = blockSize; for (j = 0; j < blockSize; ++j) { if (normR[j] >= tolEigenSolve) { firstIndex = j; break; } } // for (j = 0; j < blockSize; ++j) while (firstIndex < nFound) { for (j = firstIndex; j < blockSize; ++j) { if (normR[j] < tolEigenSolve) { // Swap the j-th and firstIndex-th position callFortran.SWAP(nb*blockSize, S + j*dimSearch, 1, S + firstIndex*dimSearch, 1); callFortran.SWAP(1, theta + j, 1, theta + firstIndex, 1); callFortran.SWAP(1, normR + j, 1, normR + firstIndex, 1); break; } } // for (j = firstIndex; j < blockSize; ++j) for (j = 0; j < blockSize; ++j) { if (normR[j] >= tolEigenSolve) { firstIndex = j; break; } } // for (j = 0; j < blockSize; ++j) } // while (firstIndex < nFound) // Copy the converged eigenvalues memcpy(lambda + knownEV, theta, nFound*sizeof(double)); } // if (nFound > 0) // Define the restarting size bStart = ((nb - offSet) > 2) ? (nb - offSet)/2 : 0; // Define the restarting space and local stiffness timeRestart -= MyWatch.WallTime(); memset(KK, 0, nb*blockSize*dimSearch*sizeof(double)); for (j = 0; j < bStart*blockSize; ++j) { KK[j + j*dimSearch] = theta[j + nFound]; } // Form the restarting space int oldCol = nb*blockSize; int newCol = nFound + (bStart+1)*blockSize; newCol = (newCol > oldCol) ? oldCol : newCol; callFortran.GEQRF(oldCol, newCol, S, dimSearch, theta, R.Values(), xr*blockSize, &info); callFortran.ORMQR('R', 'N', xr, oldCol, newCol, S, dimSearch, theta, X.Values()+knownEV*xr, xr, R.Values(), blockSize*xr, &info); timeRestart += MyWatch.WallTime(); if (nFound == 0) offSet += 1; knownEV += nFound; maxBlock = (dimSearch/blockSize) - (knownEV/blockSize); // Put random vectors if the Rayleigh Ritz vectors are not enough newCol = nFound + (bStart+1)*blockSize; if (newCol > oldCol) { Epetra_MultiVector Xnext(View, X, knownEV+blockSize-nFound, nFound); Xnext.Random(); continue; } nFound = 0; } // while (outerIter <= maxIterEigenSolve) timeOuterLoop += MyWatch.WallTime(); highMem = (highMem > currentSize()) ? highMem : currentSize(); // Clean memory delete[] work1; delete[] work2; if (vectWeight) delete vectWeight; // Sort the eigenpairs timePostProce -= MyWatch.WallTime(); if ((info == 0) && (knownEV > 0)) { mySort.sortScalars_Vectors(knownEV, lambda, Q.Values(), Q.MyLength()); } timePostProce += MyWatch.WallTime(); return (info == 0) ? knownEV : info; }