void IncrEigbicg_Z( int n, int lde,int nrhs, Complex_Z *X, Complex_Z *B, int *ncurEvals, int ldh, Complex_Z *evecsl, Complex_Z *evecsr, Complex_Z *evals, Complex_Z *H, void (*matvec) (void *, void *, void *), void (*mathvec)(void *, void *, void *), void *params, double *AnormEst, Complex_Z *work, Complex_Z *VL, int ldvl, Complex_Z *VR, int ldvr, Complex_Z *ework, int esize, double tol, double *restartTol, int maxit, char SRT_OPT, double epsi, int ConvTestOpt, int plvl, int nev, int v_max,FILE *outputFile) { /* Timing vars */ double wt1,wt2,ut1,ut2,st1,st2,wE,wI; /* Pointers */ Complex_Z tempc, tempc1, tempc2, *tmpH, *x, *resid, *b; double *rnorms, *reshist, normb, curTol,resNorm,leftTol; int i,j,k, *IPIV, ONE = 1; int zs, ds, tmpsize, is, phase, allelems ; int numIts, flag, flag2,nAdded, nev_used, iters_used, info; Complex_Z tpone = {+1.0e+00,+0.0e00}, tzero = {+0.0e+00,+0.0e00}; char cR = 'R'; char cL = 'L'; char cN ='N'; char cV = 'V'; char cU = 'U'; char cC ='C'; double *xrnorms,*xlnorms,*ernorms; Complex_Z *angles; /* ------------------------------------------------------------------- */ /* Work allocations */ /* ------------------------------------------------------------------- */ zs = sizeof(Complex_Z); ds = sizeof(double); is = sizeof(int); if( (IPIV = (int *) calloc(ldh,is)) == NULL) { fprintf(stderr,"ERROR IncrEigbicg could not allocate IPIV\n"); exit(1);} if( (x = (Complex_Z *) calloc(lde,zs)) == NULL) { fprintf(stderr,"ERROR IncrEigbicg could not allocate x\n"); exit(1);} if( (resid = (Complex_Z *) calloc(lde,zs)) == NULL) { fprintf(stderr,"ERROR IncrEigbicg could not allocate resid\n"); exit(1);} if( (tmpH = (Complex_Z *) calloc(ldh*ldh,zs)) == NULL) { fprintf(stderr,"ERROR IncrEigbicg could not allocate tmpH\n"); exit(1);} if ((work = (Complex_Z *) calloc(6*lde, zs)) == NULL) {fprintf(stderr, "ERROR IncrEigbicg could not allocate work\n"); exit(1);} if ((ework = (Complex_Z *) calloc(esize, zs)) == NULL) {fprintf(stderr, "ERROR IncrEigbicg could not allocate ework\n"); exit(1);} if ((VL = (Complex_Z *) calloc(v_max*ldvl, zs)) == NULL) {fprintf(stderr, "ERROR IncrEigbicg could not allocate VL\n"); exit(1);} if ((VR = (Complex_Z *) calloc(v_max*ldvr, zs)) == NULL) {fprintf(stderr, "ERROR IncrEigbicg could not allocate VR\n"); exit(1);} if ( (rnorms = (double *) calloc(ldh, ds)) == NULL ) {fprintf(stderr, "ERROR IncrEigbicg could not allocate rnorms\n"); exit(1);} if ( (reshist = (double *) calloc(maxit, ds)) == NULL) {fprintf(stderr, "ERROR IncrEigbicg could not allocate reshist\n"); exit(1);} if( (xlnorms = (double *) calloc(ldh,ds)) == NULL){ fprintf(stderr,"ERROR: IncrEigbicg couldn't allocate xlnorms\n"); exit(1);} if( (xrnorms = (double *) calloc(ldh,ds)) == NULL){ fprintf(stderr,"ERROR: IncrEigbicg couldn't allocate xrnorms\n"); exit(1);} if( (ernorms = (double *) calloc(ldh,ds)) == NULL){ fprintf(stderr,"ERROR: IncrEigbicg couldn't allocate ernorms\n"); exit(1);} if( (angles = (Complex_Z *) calloc(ldh,zs)) == NULL){ fprintf(stderr,"ERROR: IncrEigbicg couldn't allocate angles\n"); exit(1);} /* ------------------------------------------------------------------- */ /* end Work allocations */ /* ------------------------------------------------------------------- */ /* --------------------------------------------------------------------------------- */ /* Solving one by one the nrhs systems with incremental init-eigbicg or init-bicgstab */ /* ---------------------------------------------------------------------------------- */ for (j=0; j<nrhs; j++) { b = &B[j*lde]; tempc = wrap_zdot(&n,b,&ONE,b,&ONE,params); printf("bnorm=%g\n",sqrt(tempc.r)); normb = sqrt(tempc.r); numIts = 0; //choose eigbicg or bicgstab if(ldh-(*ncurEvals) >= nev ) phase=1; else phase=2; if (plvl) fprintf(outputFile, "\n\nSystem %d\n", j); wE = 0.0; wI = 0.0; /* Start accumulator timers */ if ( (*ncurEvals > 0) && (phase==1)) { /* --------------------------------------------------------- */ /* Perform init-BICG with evecsl and evecsr vectors */ /* xinit = xinit + evecsr*inv(H)*evecl'*(b-Ax0) */ /* --------------------------------------------------------- */ wt1 = primme_get_wtime(); /* copy H into tmpH otherwise it will be changed by CGEV */ allelems = ldh*(*ncurEvals); BLAS_ZCOPY(&allelems,H,&ONE,tmpH,&ONE); // LU factorization of tmpH BLAS_ZGETRF(ncurEvals,ncurEvals,tmpH,&ldh,IPIV,&info); for(i=0; i<n; i++) x[i]=tzero; matvec(&X[j*lde],resid,params); for(i=0; i<n; i++){ resid[i].r = b[i].r - resid[i].r; resid[i].i = b[i].i - resid[i].i;} init_BICG_Z(evecsl,ldvl,evecsr,ldvr,(*ncurEvals), x,resid,lde,n,tmpH,ldh,IPIV,work,matvec,params,&info); if(phase==1){ for(i=0; i<n; i++){ X[j*lde+i].r = X[j*lde+i].r + x[i].r; X[j*lde+i].i = X[j*lde+i].i + x[i].i;}} wt2 = primme_get_wtime(); wI = wI + wt2-wt1; } /* end of init-BICG with evecsl and evecsr vectors */ /* ------------------------------------------------------------ */ if(phase == 1){ /* ------------------------------------------------------------ */ /* Solve Ax = b with x initial guess using eigbicg and compute new nev eigenvectors */ /* ------------------------------------------------------------ */ wt1 = primme_get_wtime(); Zeigbicg(n, lde, &X[j*lde], b, &normb, tol, maxit, SRT_OPT,epsi,ConvTestOpt, &numIts, reshist, &flag, plvl, work, matvec, mathvec, params, AnormEst, nev, &evals[(*ncurEvals)], &rnorms[(*ncurEvals)], v_max, VR,ldvr,VL,ldvl,esize,ework); wt2 = primme_get_wtime(); wE = wE + wt2-wt1; /* ---------- */ /* Reporting */ /* ---------- */ tempc1 = wrap_zdot(&n,&X[j*lde],&ONE,&X[j*lde],&ONE,params); if (plvl) { fprintf(outputFile, "For this rhs:\n"); fprintf(outputFile, "Norm(solution) %-16.12E, AnormEst %-16.12E\n",sqrt(tempc1.r),(*AnormEst)); fprintf(outputFile, "Total initBICG Wallclock : %-f\n", wI); fprintf(outputFile, "Total eigbicg Wallclock : %-f\n", wE); fprintf(outputFile, "Iterations: %-d\n", numIts); fprintf(outputFile, "Actual Resid of LinSys : %e\n", reshist[numIts-1]); if (plvl > 1) for (i=0; i < nev; i++) fprintf(outputFile, "Eval[%d]: %-22.15E %-22.15E rnorm: %-22.15E\n", i+1, evals[*ncurEvals+i].r, evals[*ncurEvals+i].i, rnorms[*ncurEvals+i]); if (plvl >1) { fprintf(outputFile,"Residual norm\n"); for( i=0; i < numIts; i++) fprintf(outputFile,"%-d %-22.15E\n",i,reshist[i]); } if (flag != 0) { fprintf(outputFile, "Error: eigbicg returned with nonzero exit status\n"); return;} } /* ------------------------------------------------------------------- */ /* Update the evecsl, evecsr, and evecsl'*A*evecsr */ /* ------------------------------------------------------------------- */ wt1 = primme_get_wtime(); primme_get_time(&ut1,&st1); /* Append new Ritz pairs to evecs */ for (i=0; i<nev; i++){ BLAS_ZCOPY(&n, &VL[i*ldvl], &ONE, &evecsl[((*ncurEvals)+i)*ldvl], &ONE); BLAS_ZCOPY(&n, &VR[i*ldvr], &ONE, &evecsr[((*ncurEvals)+i)*ldvr], &ONE);} /* Bi-Orthogonalize the new Ritz vectors */ /* Use a simple biorthogonalization that uses all vectors */ nAdded = nev; //for the moment, we add all the vectors biortho_global_Z(evecsl,ldvl,evecsr,ldvr,n,(*ncurEvals)+1,(*ncurEvals)+nev,3,params); //check the biorthogonality of the vectors /* for(i=0; i<(*ncurEvals)+nev; i++) for(k=0; k<(*ncurEvals)+nev; k++) { tempc = wrap_zdot(&n,&evecsl[i*ldvl],&ONE,&evecsr[k*ldvr],&ONE,params); fprintf(outputFile,"evecsl[%d]'*evecsr[%d]=%g %g\n",i,k,tempc.r,tempc.i); } */ /* Augument H */ /* (1:ncurEvals+nAdded,ncurEvals+1:ncurEvals+nAdded) block */ for(k=(*ncurEvals); k<(*ncurEvals)+nAdded; k++) { matvec(&evecsr[k*ldvr],VR,params); for(i=0; i<(*ncurEvals)+nAdded; i++) { tempc=wrap_zdot(&n,&evecsl[i*ldvl],&ONE,VR,&ONE,params); H[i+k*ldh]=tempc; } } /* (ncurEvals+1:ncurEvals+nAdded,1:ncurEvals) block */ for(k=(*ncurEvals); k<(*ncurEvals)+nAdded; k++) { mathvec(&evecsl[k*ldvl],VL,params); for(i=0; i<(*ncurEvals); i++) { tempc = wrap_zdot(&n,&evecsr[i*ldvr],&ONE,VL,&ONE,params); H[k+i*ldh].r = tempc.r; H[k+i*ldh].i = -tempc.i; } } (*ncurEvals) = (*ncurEvals) + nAdded; /* Reporting */ wt2 = primme_get_wtime(); primme_get_time(&ut2,&st2); if (plvl) { fprintf(outputFile, "Update\n"); fprintf(outputFile, "Added %d vecs\n",nAdded); fprintf(outputFile, "U Wallclock : %-f\n", wt2-wt1); fprintf(outputFile, "U User Time : %f seconds\n", ut2-ut1); fprintf(outputFile, "U Syst Time : %f seconds\n", st2-st1);} } /* if phase==1 */ /****************************************************/ if(phase==2) //solve deflated bicgstab the correction equation { fprintf(outputFile,"\n\nDeflated bicgstab\n");; LRD_BICGSTAB_Z(evecsl, ldvl,evecsr,ldvr,(*ncurEvals),&X[j*lde],b, lde,n,H,ldh,IPIV,work,matvec,params,(*AnormEst),maxit,(*restartTol),tol,ConvTestOpt,outputFile,&info); } /*end of if(phase==2)*/ } /*for(j=0; j<nrhs; j++) */ // compute final evecs,etc. ComputeFinalEvecs_Z ( (*ncurEvals),n,evecsl,ldvl,evecsr,ldvr,H,ldh,SRT_OPT,epsi, evals, ernorms, xlnorms, xrnorms, angles, matvec,params,work,6*lde); fprintf(outputFile,"\n\n Final Evals\n"); for(i=0; i< (*ncurEvals); i++){ fprintf(outputFile,"EVAL %-16.12E %-16.12E, ERNORM %-16.12E\n", evals[i].r,evals[i].i,ernorms[i]); fprintf(outputFile,"XLNORM %-16.12E, XRNORM %-16.12E, ANGLE %-16.12E %-16.12E\n\n", xlnorms[i],xrnorms[i],angles[i].r,angles[i].i);} fprintf(outputFile,"===================================================\n"); return; }
int main (int argc, char *argv[]) { /* Timing vars */ double ut1,ut2,st1,st2,wt1,wt2; /* Matrix */ int n, nLocal, nnz; double fnorm; CSRMatrix matrix; // The matrix in simple SPARSKIT CSR format Matrix *Par_matrix; // Pointer to the matrix in Parasails CSR format /* Permutation/partitioning stuff */ int *mask, *map; int *fg2or, *or2fg; /* Preconditioner */ ParaSails *Par_Factors; // Pointer to the matrix in Parasails CSR format /* Files */ char *DriverConfigFileName, *SolverConfigFileName; char partFileName[512]; FILE *partFile; /* Driver and solver I/O arrays and parameters */ double *evals, *evecs, *rnorms; driver_params driver; primme_params primme; primme_preset_method method; #ifdef Cplusplus /* C++ has a stricter type checking */ void (*precond_function)(void *, void *, int *, primme_params *); #else void *precond_function; #endif /* Other miscellaneous items */ int i,j; int ret, modulo; int rangeStart; int rangeEnd; int numProcs, procID; MPI_Comm comm; /* --------------------------------------------------------------------- */ /* MPI INITIALIZATION */ /* --------------------------------------------------------------------- */ MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &numProcs); MPI_Comm_rank(MPI_COMM_WORLD, &procID); comm = MPI_COMM_WORLD; /* --------------------------------------------------------------------- */ /* Read matrix and driver setup */ /* --------------------------------------------------------------------- */ /* ------------------------------------------------------- */ /* Get from command line the names for the 2 config files */ /* ------------------------------------------------------- */ if (argc == 3) { DriverConfigFileName = argv[1]; SolverConfigFileName = argv[2]; } else { MPI_Finalize(); return(-1); } /* ----------------------------- */ /* Read in the driver parameters */ /* ----------------------------- */ if (read_driver_params(DriverConfigFileName, &driver) < 0) { fprintf(stderr, "Reading driver parameters failed\n"); fflush(stderr); MPI_Finalize(); return(-1); } /* ------------------------------------------ */ /* Read the matrix and store it in CSR format */ /* ------------------------------------------ */ if (procID == 0) { fprintf(stdout," Matrix: %s\n",driver.matrixFileName); fflush(stdout); if (!strcmp("mtx", &driver.matrixFileName[strlen(driver.matrixFileName)-3])) { // coordinate format storing both lower and upper triangular parts ret = readfullMTX(driver.matrixFileName, &matrix.AElts, &matrix.JA, &matrix.IA, &n, &nnz); if (ret < 0) { fprintf(stderr, "ERROR: Could not read matrix file\n"); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); return(-1); } } else if (driver.matrixFileName[strlen(driver.matrixFileName)-1] == 'U') { // coordinate format storing only upper triangular part ret = readUpperMTX(driver.matrixFileName, &matrix.AElts, &matrix.JA, &matrix.IA, &n, &nnz); if (ret < 0) { fprintf(stderr, "ERROR: Could not read matrix file\n"); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); return(-1); } } else { //Harwell Boeing format NOT IMPLEMENTED //ret = readmt() ret = -1; if (ret < 0) { fprintf(stderr, "ERROR: Could not read matrix file\n"); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); return(-1); } } } // if procID == 0 /* ----------------------------------------------------------- */ // Allocate space on other processors and broadcast the matrix /* ----------------------------------------------------------- */ MPI_Bcast(&nnz, 1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD); if (procID != 0) { matrix.AElts = (double *)primme_calloc(nnz, sizeof(double), "A"); matrix.JA = (int *)primme_calloc(nnz, sizeof(int), "JA"); matrix.IA = (int *)primme_calloc(n+1, sizeof(int), "IA"); } else { // Proc 0 converts CSR to C indexing for (i=0; i < n+1; i++) { matrix.IA[i]--; } for (i=0; i < nnz; i++) { matrix.JA[i]--; } } MPI_Bcast(matrix.AElts, nnz, MPI_DOUBLE, 0, MPI_COMM_WORLD); MPI_Bcast(matrix.IA, n+1, MPI_INT, 0, MPI_COMM_WORLD); MPI_Bcast(matrix.JA, nnz, MPI_INT, 0, MPI_COMM_WORLD); fnorm = frobeniusNorm(n, matrix.IA, matrix.AElts); /* ---------------------------------------------------------------------- */ /* Partitioning of the matrix among the processors */ /* ---------------------------------------------------------------------- */ mask = (int *)primme_calloc(n, sizeof(int), "mask"); map = (int *)primme_calloc(numProcs+1, sizeof(int), "map"); fg2or = (int *)primme_calloc(n, sizeof(int), "fg2or"); or2fg = (int *)primme_calloc(n, sizeof(int), "or2fg"); // /* * * * * * * * * * * * * * * * * * // * Read the partition from a file // * * * * * * * * * * * * * * * * * */ // sprintf(partFileName, "%s/%s", driver.partDir, driver.partId); // partFile = fopen(partFileName, "r"); // // if (partFile == 0) { // fprintf(stderr, "ERROR: Could not open '%s'\n", partFileName); // MPI_Finalize(); // return(-1); // } // // for (i = 0; i < n; i++) { // fscanf(partFile, "%d", &mask[i]); // } // // fclose(partFile); /* * * * * * * * * * * * * * * * * * * * * * * */ /* Simplistic assignment of processors to rows */ /* * * * * * * * * * * * * * * * * * * * * * * */ nLocal = n / numProcs; modulo = n % numProcs; rangeStart = 0; for (i=0; i<numProcs; i++) { rangeEnd = rangeStart + nLocal; if (i < modulo) rangeEnd = rangeEnd + 1; for (j = rangeStart; j< rangeEnd; j++) mask[j] = i; rangeStart = rangeEnd; } /* * * * * * * * * * * * * * * * * * * * * * * */ generatePermutations(n, numProcs, mask, or2fg, fg2or, map); rangeStart = map[procID]; rangeEnd = map[procID+1]-1; nLocal = rangeEnd - rangeStart+1; Par_matrix = csrToParaSails(procID, map, fg2or, or2fg, matrix.IA, matrix.JA, matrix.AElts, comm); /* ------------------------------------------------------------------------- */ /* Set up preconditioner if needed. For parallel programs the only choice */ /* in driver.PrecChoice: (driver.PrecChoice is read in read_driver_params) */ /* choice = 4 Parallel ParaSails preconditioners */ /* with parameters (level, threshold, filter, isymm) */ /* as read in read_driver_params(). */ /* ------------------------------------------------------------------------- */ if (driver.PrecChoice == 4) { Par_Factors = generate_precond(&matrix, driver.shift, n, procID, map, fg2or, or2fg, rangeStart, rangeEnd, driver.isymm, driver.level, driver.threshold, driver.filter, comm); precond_function = par_ApplyParasailsPrec; } else { Par_Factors = NULL; precond_function = NULL; // Free A as it is not further needed free(matrix.AElts); free(matrix.IA); free(matrix.JA); } free(mask); free(map); free(fg2or); free(or2fg); /* --------------------------------------------------------------------- */ /* Primme solver setup */ /* primme_initialize (not needed if ALL primme struct members set)*/ /* primme_set_method (bypass it to fully customize your solver) */ /* --------------------------------------------------------------------- */ /* ----------------------------- */ /* Initialize defaults in primme */ /* ----------------------------- */ primme_initialize(&primme); /* --------------------------------------- */ /* Read in the primme configuration file */ /* --------------------------------------- */ primme.n = n; primme.aNorm = fnorm; /* ||A||_frobenius. A configFile entry overwrites it */ if (procID == 0) { if (read_solver_params(SolverConfigFileName, driver.outputFileName, &primme, &method) < 0) { fprintf(stderr, "Reading solver parameters failed\n"); MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE); return(-1); } } /* ------------------------------------------------- */ // Send read common primme members to all processors // Setup the primme members local to this processor /* ------------------------------------------------- */ broadCast(&primme, &method, comm); primme.procID = procID; primme.numProcs = numProcs; primme.nLocal = nLocal; primme.commInfo = &comm; primme.globalSumDouble = par_GlobalSumDouble; /* --------------------------------------- */ /* Pick one of the default methods(if set) */ /* --------------------------------------- */ if (primme_set_method(method, &primme) < 0 ) { fprintf(primme.outputFile, "No preset method. Using custom settings\n"); } /* --------------------------------------- */ /* Optional: report memory requirements */ /* --------------------------------------- */ ret = dprimme(NULL,NULL,NULL,&primme); fprintf(primme.outputFile,"PRIMME will allocate the following memory:\n"); fprintf(primme.outputFile," processor %d, real workspace, %ld bytes\n", procID, primme.realWorkSize); fprintf(primme.outputFile," processor %d, int workspace, %d bytes\n", procID, primme.intWorkSize); /* --------------------------------------- */ /* Set up matrix vector and preconditioner */ /* --------------------------------------- */ primme.matrixMatvec = par_MatrixMatvec; primme.applyPreconditioner = precond_function; /* --------------------------------------- */ /* Optional: provide matrix/preconditioner */ /* --------------------------------------- */ primme.matrix = Par_matrix; primme.preconditioner = Par_Factors; /* --------------------------------------- */ /* Display given parameter configuration */ /* Place this after the dprimme() to see */ /* any changes dprimme() made to primme */ /* --------------------------------------- */ if (procID >= 0) { fprintf(primme.outputFile," Matrix: %s\n",driver.matrixFileName); primme_display_params(primme); } /* --------------------------------------------------------------------- */ /* Run the dprimme solver */ /* --------------------------------------------------------------------- */ /* Allocate space for converged Ritz values and residual norms */ evals = (double *)primme_calloc(primme.numEvals, sizeof(double), "evals"); evecs = (double *)primme_calloc(primme.nLocal* (primme.numEvals+primme.maxBlockSize), sizeof(double), "evecs"); rnorms = (double *)primme_calloc(primme.numEvals, sizeof(double), "rnorms"); /* ------------------------ */ /* Initial guess (optional) */ /* ------------------------ */ for (i=0;i<primme.nLocal;i++) evecs[i]=1/sqrt(primme.n); /* ------------- */ /* Call primme */ /* ------------- */ wt1 = primme_get_wtime(); primme_get_time(&ut1,&st1); ret = dprimme(evals, evecs, rnorms, &primme); wt2 = primme_get_wtime(); primme_get_time(&ut2,&st2); /* --------------------------------------------------------------------- */ /* Reporting */ /* --------------------------------------------------------------------- */ if (procID == 0) { primme_PrintStackTrace(primme); } fprintf(primme.outputFile, "Wallclock Runtime : %-f\n", wt2-wt1); fprintf(primme.outputFile, "User Time : %f seconds\n", ut2-ut1); fprintf(primme.outputFile, "Syst Time : %f seconds\n", st2-st1); if (primme.procID == 0) { for (i=0; i < primme.numEvals; i++) { fprintf(primme.outputFile, "Eval[%d]: %-22.15E rnorm: %-22.15E\n", i+1, evals[i], rnorms[i]); } fprintf(primme.outputFile, " %d eigenpairs converged\n", primme.initSize); fprintf(primme.outputFile, "Tolerance : %-22.15E\n", primme.aNorm*primme.eps); fprintf(primme.outputFile, "Iterations: %-d\n", primme.stats.numOuterIterations); fprintf(primme.outputFile, "Restarts : %-d\n", primme.stats.numRestarts); fprintf(primme.outputFile, "Matvecs : %-d\n", primme.stats.numMatvecs); fprintf(primme.outputFile, "Preconds : %-d\n", primme.stats.numPreconds); fprintf(primme.outputFile, "\n\n#,%d,%.1f\n\n", primme.stats.numMatvecs, wt2-wt1); switch (primme.dynamicMethodSwitch) { case -1: fprintf(primme.outputFile, "Recommended method for next run: DEFAULT_MIN_MATVECS\n"); break; case -2: fprintf(primme.outputFile, "Recommended method for next run: DEFAULT_MIN_TIME\n"); break; case -3: fprintf(primme.outputFile, "Recommended method for next run: DYNAMIC (close call)\n"); break; } } if (ret != 0 && procID == 0) { fprintf(primme.outputFile, "Error: dprimme returned with nonzero exit status: %d \n",ret); return -1; } fflush(primme.outputFile); fclose(primme.outputFile); primme_Free(&primme); fflush(stdout); fflush(stderr); MPI_Barrier(comm); MPI_Finalize(); return(0); }