PetscErrorCode PetscLogPrintSummaryToPy(MPI_Comm comm, PetscViewer viewer)
{
PetscViewer_ASCII *ascii = (PetscViewer_ASCII*)viewer->data;
FILE *fd = ascii->fd;
PetscLogDouble zero = 0.0;
StageLog stageLog;
StageInfo *stageInfo = PETSC_NULL;
EventPerfInfo *eventInfo = PETSC_NULL;
ClassPerfInfo *classInfo;
const char *name;
PetscLogDouble locTotalTime, TotalTime, TotalFlops;
PetscLogDouble numMessages, messageLength, avgMessLen, numReductions;
PetscLogDouble stageTime, flops, mem, mess, messLen, red;
PetscLogDouble fracTime, fracFlops, fracMessages, fracLength;
PetscLogDouble fracReductions;
PetscLogDouble tot,avg,x,y,*mydata;
PetscMPIInt minCt, maxCt;
PetscMPIInt size, rank, *mycount;
PetscTruth *localStageUsed, *stageUsed;
PetscTruth *localStageVisible, *stageVisible;
int numStages, localNumEvents, numEvents;
int stage, lastStage;
PetscLogEvent event;
PetscErrorCode ierr;
PetscInt i;
/* remove these two lines! */
PetscLogDouble PETSC_DLLEXPORT BaseTime = 0.0;
int numObjects = 0;
PetscFunctionBegin;
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscMalloc(size*sizeof(PetscLogDouble), &mydata);CHKERRQ(ierr);
ierr = PetscMalloc(size*sizeof(PetscMPIInt), &mycount);CHKERRQ(ierr);
/* Pop off any stages the user forgot to remove */
lastStage = 0;
ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
ierr = StageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
while (stage >= 0) {
lastStage = stage;
ierr = StageLogPop(stageLog);CHKERRQ(ierr);
ierr = StageLogGetCurrent(stageLog, &stage);CHKERRQ(ierr);
}
/* Get the total elapsed time */
PetscTime(locTotalTime); locTotalTime -= BaseTime;
ierr = PetscFPrintf(comm, fd, "\n#------ PETSc Performance Summary ----------\n\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fd, "Nproc = %d\n",size);CHKERRQ(ierr);
/* Must preserve reduction count before we go on */
red = (allreduce_ct + gather_ct + scatter_ct)/((PetscLogDouble) size);
/* Calculate summary information */
/* Time */
ierr = MPI_Gather(&locTotalTime,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "Time = [ " );CHKERRQ(ierr);
tot = 0.0;
for (i=0; i<size; i++){
tot += mydata[i];
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
avg = (tot)/((PetscLogDouble) size);
TotalTime = tot;
}
/* Objects */
avg = (PetscLogDouble) numObjects;
ierr = MPI_Gather(&avg,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "Objects = [ " );CHKERRQ(ierr);
for (i=0; i<size; i++){
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
}
/* Flops */
ierr = MPI_Gather(&_TotalFlops,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "Flops = [ " );CHKERRQ(ierr);
tot = 0.0;
for (i=0; i<size; i++){
tot += mydata[i];
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n");CHKERRQ(ierr);
TotalFlops = tot;
}
/* Memory */
ierr = PetscMallocGetMaximumUsage(&mem);CHKERRQ(ierr);
ierr = MPI_Gather(&mem,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "Memory = [ " );CHKERRQ(ierr);
for (i=0; i<size; i++){
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
}
/* Messages */
mess = 0.5*(irecv_ct + isend_ct + recv_ct + send_ct);
ierr = MPI_Gather(&mess,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "MPIMessages = [ " );CHKERRQ(ierr);
tot = 0.0;
for (i=0; i<size; i++){
tot += mydata[i];
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
numMessages = tot;
}
/* Message Lengths */
mess = 0.5*(irecv_len + isend_len + recv_len + send_len);
ierr = MPI_Gather(&mess,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "MPIMessageLengths = [ " );CHKERRQ(ierr);
tot = 0.0;
for (i=0; i<size; i++){
tot += mydata[i];
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
messageLength = tot;
}
/* Reductions */
ierr = MPI_Gather(&red,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "MPIReductions = [ " );CHKERRQ(ierr);
tot = 0.0;
for (i=0; i<size; i++){
tot += mydata[i];
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
numReductions = tot;
}
/* Get total number of stages --
Currently, a single processor can register more stages than another, but stages must all be registered in order.
We can removed this requirement if necessary by having a global stage numbering and indirection on the stage ID.
This seems best accomplished by assoicating a communicator with each stage.
*/
ierr = MPI_Allreduce(&stageLog->numStages, &numStages, 1, MPI_INT, MPI_MAX, comm);CHKERRQ(ierr);
ierr = PetscMalloc(numStages * sizeof(PetscTruth), &localStageUsed);CHKERRQ(ierr);
ierr = PetscMalloc(numStages * sizeof(PetscTruth), &stageUsed);CHKERRQ(ierr);
ierr = PetscMalloc(numStages * sizeof(PetscTruth), &localStageVisible);CHKERRQ(ierr);
ierr = PetscMalloc(numStages * sizeof(PetscTruth), &stageVisible);CHKERRQ(ierr);
if (numStages > 0) {
stageInfo = stageLog->stageInfo;
for(stage = 0; stage < numStages; stage++) {
if (stage < stageLog->numStages) {
localStageUsed[stage] = stageInfo[stage].used;
localStageVisible[stage] = stageInfo[stage].perfInfo.visible;
} else {
localStageUsed[stage] = PETSC_FALSE;
localStageVisible[stage] = PETSC_TRUE;
}
}
ierr = MPI_Allreduce(localStageUsed, stageUsed, numStages, MPI_INT, MPI_LOR, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(localStageVisible, stageVisible, numStages, MPI_INT, MPI_LAND, comm);CHKERRQ(ierr);
for(stage = 0; stage < numStages; stage++) {
if (stageUsed[stage]) {
ierr = PetscFPrintf(comm, fd, "\n#Summary of Stages: ----- Time ------ ----- Flops ----- --- Messages --- -- Message Lengths -- -- Reductions --\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fd, "# Avg %%Total Avg %%Total counts %%Total Avg %%Total counts %%Total \n");CHKERRQ(ierr);
break;
}
}
for(stage = 0; stage < numStages; stage++) {
if (!stageUsed[stage]) continue;
if (localStageUsed[stage]) {
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.time, &stageTime, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.flops, &flops, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.numMessages, &mess, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.messageLength, &messLen, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.numReductions, &red, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
name = stageInfo[stage].name;
} else {
ierr = MPI_Allreduce(&zero, &stageTime, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &flops, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &mess, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &messLen, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &red, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
name = "";
}
mess *= 0.5; messLen *= 0.5; red /= size;
if (TotalTime != 0.0) fracTime = stageTime/TotalTime; else fracTime = 0.0;
if (TotalFlops != 0.0) fracFlops = flops/TotalFlops; else fracFlops = 0.0;
/* Talk to Barry if (stageTime != 0.0) flops = (size*flops)/stageTime; else flops = 0.0; */
if (numMessages != 0.0) fracMessages = mess/numMessages; else fracMessages = 0.0;
if (numMessages != 0.0) avgMessLen = messLen/numMessages; else avgMessLen = 0.0;
if (messageLength != 0.0) fracLength = messLen/messageLength; else fracLength = 0.0;
if (numReductions != 0.0) fracReductions = red/numReductions; else fracReductions = 0.0;
ierr = PetscFPrintf(comm, fd, "# ");
ierr = PetscFPrintf(comm, fd, "%2d: %15s: %6.4e %5.1f%% %6.4e %5.1f%% %5.3e %5.1f%% %5.3e %5.1f%% %5.3e %5.1f%% \n",
stage, name, stageTime/size, 100.0*fracTime, flops, 100.0*fracFlops,
mess, 100.0*fracMessages, avgMessLen, 100.0*fracLength, red, 100.0*fracReductions);CHKERRQ(ierr);
}
}
/* Report events */
ierr = PetscFPrintf(comm, fd,"\n# Event\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm,fd,"# ------------------------------------------------------\n");
CHKERRQ(ierr);
/* Problem: The stage name will not show up unless the stage executed on proc 1 */
for(stage = 0; stage < numStages; stage++) {
if (!stageVisible[stage]) continue;
if (localStageUsed[stage]) {
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.time, &stageTime, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.flops, &flops, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.numMessages, &mess, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.messageLength, &messLen, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&stageInfo[stage].perfInfo.numReductions, &red, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
} else {
ierr = PetscFPrintf(comm, fd, "\n--- Event Stage %d: Unknown\n\n", stage);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &stageTime, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &flops, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &mess, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &messLen, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
ierr = MPI_Allreduce(&zero, &red, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, comm);CHKERRQ(ierr);
}
mess *= 0.5; messLen *= 0.5; red /= size;
/* Get total number of events in this stage --
Currently, a single processor can register more events than another, but events must all be registered in order,
just like stages. We can removed this requirement if necessary by having a global event numbering and indirection
on the event ID. This seems best accomplished by assoicating a communicator with each stage.
Problem: If the event did not happen on proc 1, its name will not be available.
Problem: Event visibility is not implemented
*/
if (!rank){
ierr = PetscFPrintf(comm, fd, "class Dummy(object):\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fd, " def foo(x):\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fd, " print x\n");CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fd, "Event = {}\n");CHKERRQ(ierr);
}
if (localStageUsed[stage]) {
eventInfo = stageLog->stageInfo[stage].eventLog->eventInfo;
localNumEvents = stageLog->stageInfo[stage].eventLog->numEvents;
} else {
localNumEvents = 0;
}
ierr = MPI_Allreduce(&localNumEvents, &numEvents, 1, MPI_INT, MPI_MAX, comm);CHKERRQ(ierr);
for(event = 0; event < numEvents; event++) {
if (localStageUsed[stage] && (event < stageLog->stageInfo[stage].eventLog->numEvents) && (eventInfo[event].depth == 0)) {
ierr = MPI_Allreduce(&eventInfo[event].count, &maxCt, 1, MPI_INT, MPI_MAX, comm);CHKERRQ(ierr);
name = stageLog->eventLog->eventInfo[event].name;
} else {
ierr = MPI_Allreduce(&ierr, &maxCt, 1, MPI_INT, MPI_MAX, comm);CHKERRQ(ierr);
name = "";
}
if (maxCt != 0) {
ierr = PetscFPrintf(comm, fd,"#\n");CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "%s = Dummy()\n",name);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fd, "Event['%s'] = %s\n",name,name);CHKERRQ(ierr);
}
/* Count */
ierr = MPI_Gather(&eventInfo[event].count,1,MPI_INT,mycount,1,MPI_INT,0,comm);CHKERRQ(ierr);
ierr = PetscFPrintf(comm, fd, "%s.Count = [ ", name);CHKERRQ(ierr);
for (i=0; i<size; i++){
ierr = PetscFPrintf(comm, fd, " %7d,",mycount[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
/* Time */
ierr = MPI_Gather(&eventInfo[event].time,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "%s.Time = [ ", name);CHKERRQ(ierr);
for (i=0; i<size; i++){
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
}
/* Flops */
ierr = MPI_Gather(&eventInfo[event].flops,1,MPIU_PETSCLOGDOUBLE,mydata,1,MPIU_PETSCLOGDOUBLE,0,comm);CHKERRQ(ierr);
if (!rank){
ierr = PetscFPrintf(comm, fd, "%s.Flops = [ ", name);CHKERRQ(ierr);
for (i=0; i<size; i++){
ierr = PetscFPrintf(comm, fd, " %5.3e,",mydata[i] );CHKERRQ(ierr);
}
ierr = PetscFPrintf(comm, fd, "]\n" );CHKERRQ(ierr);
}
}
}
}
/* Right now, only stages on the first processor are reported here, meaning only objects associated with
the global communicator, or MPI_COMM_SELF for proc 1. We really should report global stats and then
stats for stages local to processor sets.
*/
for(stage = 0; stage < numStages; stage++) {
if (localStageUsed[stage]) {
classInfo = stageLog->stageInfo[stage].classLog->classInfo;
} else {
ierr = PetscFPrintf(comm, fd, "\n--- Event Stage %d: Unknown\n\n", stage);CHKERRQ(ierr);
}
}
ierr = PetscFree(localStageUsed);CHKERRQ(ierr);
ierr = PetscFree(stageUsed);CHKERRQ(ierr);
ierr = PetscFree(localStageVisible);CHKERRQ(ierr);
ierr = PetscFree(stageVisible);CHKERRQ(ierr);
ierr = PetscFree(mydata);CHKERRQ(ierr);
ierr = PetscFree(mycount);CHKERRQ(ierr);
/* Information unrelated to this particular run */
ierr = PetscFPrintf(comm, fd,
"# ========================================================================================================================\n");CHKERRQ(ierr);
PetscTime(y);
PetscTime(x);
PetscTime(y); PetscTime(y); PetscTime(y); PetscTime(y); PetscTime(y);
PetscTime(y); PetscTime(y); PetscTime(y); PetscTime(y); PetscTime(y);
ierr = PetscFPrintf(comm,fd,"AveragetimetogetPetscTime = %g\n", (y-x)/10.0);CHKERRQ(ierr);
/* MPI information */
if (size > 1) {
MPI_Status status;
PetscMPIInt tag;
MPI_Comm newcomm;
ierr = MPI_Barrier(comm);CHKERRQ(ierr);
PetscTime(x);
ierr = MPI_Barrier(comm);CHKERRQ(ierr);
ierr = MPI_Barrier(comm);CHKERRQ(ierr);
ierr = MPI_Barrier(comm);CHKERRQ(ierr);
ierr = MPI_Barrier(comm);CHKERRQ(ierr);
ierr = MPI_Barrier(comm);CHKERRQ(ierr);
PetscTime(y);
ierr = PetscFPrintf(comm, fd, "AveragetimeforMPI_Barrier = %g\n", (y-x)/5.0);CHKERRQ(ierr);
ierr = PetscCommDuplicate(comm,&newcomm, &tag);CHKERRQ(ierr);
ierr = MPI_Barrier(comm);CHKERRQ(ierr);
if (rank) {
ierr = MPI_Recv(0, 0, MPI_INT, rank-1, tag, newcomm, &status);CHKERRQ(ierr);
ierr = MPI_Send(0, 0, MPI_INT, (rank+1)%size, tag, newcomm);CHKERRQ(ierr);
} else {
PetscTime(x);
ierr = MPI_Send(0, 0, MPI_INT, 1, tag, newcomm);CHKERRQ(ierr);
ierr = MPI_Recv(0, 0, MPI_INT, size-1, tag, newcomm, &status);CHKERRQ(ierr);
PetscTime(y);
ierr = PetscFPrintf(comm,fd,"AveragetimforzerosizeMPI_Send = %g\n", (y-x)/size);CHKERRQ(ierr);
}
ierr = PetscCommDestroy(&newcomm);CHKERRQ(ierr);
}
if (!rank) { /* print Optiontable */
ierr = PetscFPrintf(comm,fd,"# ");CHKERRQ(ierr);
//ierr = PetscOptionsPrint(fd);CHKERRQ(ierr);
}
/* Cleanup */
ierr = PetscFPrintf(comm, fd, "\n");CHKERRQ(ierr);
ierr = StageLogPush(stageLog, lastStage);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode MatPtAPNumeric_SeqAIJ_SeqAIJ(Mat A,Mat P,Mat C)
{
PetscErrorCode ierr;
Mat_SeqAIJ *a = (Mat_SeqAIJ*) A->data;
Mat_SeqAIJ *p = (Mat_SeqAIJ*) P->data;
Mat_SeqAIJ *c = (Mat_SeqAIJ*) C->data;
const PetscInt *ai=a->i,*aj=a->j,*pi=p->i,*pj=p->j,*ci=c->i,*cj=c->j;
const PetscScalar *aa=a->a,*pa=p->a,*pval;
const PetscInt *apj,*pcol,*cjj;
const PetscInt am=A->rmap->N,cm=C->rmap->N;
PetscInt i,j,k,anz,apnz,pnz,prow,crow,cnz;
PetscScalar *apa,*ca=c->a,*caj,pvalj;
Mat_PtAP *ptap = c->ptap;
#if defined(PROFILE_MatPtAPNumeric)
PetscLogDouble t0,tf,time_Cseq0=0.0,time_Cseq1=0.0;
PetscInt flops0=0,flops1=0;
#endif
PetscFunctionBegin;
/* Get temporary array for storage of one row of A*P */
apa = ptap->apa;
/* Clear old values in C */
ierr = PetscMemzero(ca,ci[cm]*sizeof(MatScalar));CHKERRQ(ierr);
for (i=0; i<am; i++) {
/* Form sparse row of AP[i,:] = A[i,:]*P */
#if defined(PROFILE_MatPtAPNumeric)
ierr = PetscTime(&t0);CHKERRQ(ierr);
#endif
anz = ai[i+1] - ai[i];
apnz = 0;
for (j=0; j<anz; j++) {
prow = aj[j];
pnz = pi[prow+1] - pi[prow];
pcol = pj + pi[prow];
pval = pa + pi[prow];
for (k=0; k<pnz; k++) {
apa[pcol[k]] += aa[j]*pval[k];
}
ierr = PetscLogFlops(2.0*pnz);CHKERRQ(ierr);
#if defined(PROFILE_MatPtAPNumeric)
flops0 += 2.0*pnz;
#endif
}
aj += anz; aa += anz;
#if defined(PROFILE_MatPtAPNumeric)
ierr = PetscTime(&tf);CHKERRQ(ierr);
time_Cseq0 += tf - t0;
#endif
/* Compute P^T*A*P using outer product P[i,:]^T*AP[i,:]. */
#if defined(PROFILE_MatPtAPNumeric)
ierr = PetscTime(&t0);CHKERRQ(ierr);
#endif
apj = ptap->apj + ptap->api[i];
apnz = ptap->api[i+1] - ptap->api[i];
pnz = pi[i+1] - pi[i];
pcol = pj + pi[i];
pval = pa + pi[i];
/* Perform dense axpy */
for (j=0; j<pnz; j++) {
crow = pcol[j];
cjj = cj + ci[crow];
caj = ca + ci[crow];
pvalj = pval[j];
cnz = ci[crow+1] - ci[crow];
for (k=0; k<cnz; k++) caj[k] += pvalj*apa[cjj[k]];
ierr = PetscLogFlops(2.0*cnz);CHKERRQ(ierr);
#if defined(PROFILE_MatPtAPNumeric)
flops1 += 2.0*cnz;
#endif
}
#if defined(PROFILE_MatPtAPNumeric)
ierr = PetscTime(&tf);CHKERRQ(ierr);
time_Cseq1 += tf - t0;
#endif
/* Zero the current row info for A*P */
for (j=0; j<apnz; j++) apa[apj[j]] = 0.0;
}
/* Assemble the final matrix and clean up */
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
#if defined(PROFILE_MatPtAPNumeric)
printf("PtAPNumeric_SeqAIJ time %g + %g, flops %d %d\n",time_Cseq0,time_Cseq1,flops0,flops1);
#endif
PetscFunctionReturn(0);
}
PetscErrorCode FluidFieldSetup( FluidField f )
{
PetscLogDouble t1,t2;
PetscErrorCode ierr;
PetscFunctionBegin;
// Assemble viscous matricies
ierr = FluidFieldMatAssemble( f ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Lengths: %e %e %e\n", f->lens.x, f->lens.y, f->lens.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "Size: %d %d %d\n", f->dims.x, f->dims.y, f->dims.z ); CHKERRQ(ierr);
ierr = PetscInfo3( 0, "dx: %e %e %e\n", f->dh.x, f->dh.y, f->dh.z ); CHKERRQ(ierr);
ierr = PetscTime(&t1); CHKERRQ(ierr);
// Create vectors
ierr = GACreate( f->daV, &f->ga); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daV,&f->rhs); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel); CHKERRQ(ierr);
ierr = VecDuplicate(f->rhs,&f->vel0); CHKERRQ(ierr);
// ierr = DACreateGlobalVector(f->daE,&f->E); CHKERRQ(ierr);
ierr = DMCreateGlobalVector(f->daB,&f->buf); CHKERRQ(ierr);
// Set up the outer solver
ierr = KSPCreate(f->comm,&f->ksp); CHKERRQ(ierr);
ierr = KSPSetOperators(f->ksp,f->mat,f->mat, SAME_PRECONDITIONER); CHKERRQ(ierr);
ierr = KSPSetType(f->ksp,KSPFGMRES); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(f->ksp,PETSC_TRUE); CHKERRQ(ierr);
ierr = KSPSetTolerances(f->ksp,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT); CHKERRQ(ierr);
ierr = KSPSetFromOptions(f->ksp);CHKERRQ(ierr);
// Split pressure from velocity [ u v w | p ]
PC pc;
ierr = KSPGetPC(f->ksp,&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR); CHKERRQ(ierr);
if( f->is3D ) {
const PetscInt ufields[] = {U_FACE,V_FACE,W_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,4); CHKERRQ(ierr); // [p u v w]
ierr = PCFieldSplitSetFields(pc,"v",3,ufields,ufields); CHKERRQ(ierr); // [u v w]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
} else {
const PetscInt ufields[] = {U_FACE,V_FACE};
const PetscInt pfields[] = {CELL_CENTER};
ierr = PCFieldSplitSetBlockSize(pc,3); CHKERRQ(ierr); // [p u v]
ierr = PCFieldSplitSetFields(pc,"v",2,ufields,ufields); CHKERRQ(ierr); // [u v]
ierr = PCFieldSplitSetFields(pc,"p",1,pfields,pfields); CHKERRQ(ierr); // [ p ]
}
ierr = PCSetUp(pc); CHKERRQ(ierr);
int nVelP;
KSP *kspVelP;
ierr = PCFieldSplitGetSubKSP(pc,&nVelP,&kspVelP); CHKERRQ(ierr);
ierr = KSPSetTolerances(kspVelP[1],PETSC_DEFAULT,PETSC_DEFAULT,PETSC_DEFAULT,4); CHKERRQ(ierr);
ierr = KSPSetType(kspVelP[1],KSPGMRES); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[1],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc,PCNONE); CHKERRQ(ierr);
ierr = KSPSetFromOptions(kspVelP[1]);CHKERRQ(ierr);
// Split velocity [u v w] into component matricies [u], [v], [w]
ierr = KSPSetType(kspVelP[0],KSPPREONLY); CHKERRQ(ierr);
ierr = KSPGetPC(kspVelP[0],&pc); CHKERRQ(ierr);
ierr = PCSetType(pc, PCFIELDSPLIT); CHKERRQ(ierr);
ierr = PCFieldSplitSetType(pc,PC_COMPOSITE_ADDITIVE); CHKERRQ(ierr);
ierr = PCFieldSplitSetBlockSize(pc,f->is3D?3:2); CHKERRQ(ierr);
ierr = PCSetUp(pc); CHKERRQ(ierr);
/* Set solver for each velocity component
* Split component velocity as parallel blocks along processors
* Use direct solver for each block
* TODO: use MG, w/FFT on coarse grid
*/
ierr = PetscTime(&t2); CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Finished Solver Setup: %f sec\n",t2-t1); CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void PETScLinearSolver::Solver()
{
//TEST
#ifdef TEST_MEM_PETSC
PetscLogDouble mem1, mem2;
PetscMemoryGetCurrentUsage(&mem1);
#endif
/*
//TEST
PetscViewer viewer;
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "x.txt", &viewer);
PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);
PetscObjectSetName((PetscObject)x,"Solution");
VecView(x, viewer);
*/
int its;
PetscLogDouble v1,v2;
KSPConvergedReason reason;
// #define PETSC34
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v1);
#else
PetscGetTime(&v1);
#endif
#if (PETSC_VERSION_MAJOR == 3) && (PETSC_VERSION_MINOR > 4)
KSPSetOperators(lsolver, A, A);
#else
KSPSetOperators(lsolver, A, A, DIFFERENT_NONZERO_PATTERN);
#endif
KSPSolve(lsolver, b, x);
KSPGetConvergedReason(lsolver,&reason); //CHKERRQ(ierr);
if (reason==KSP_DIVERGED_INDEFINITE_PC)
{
PetscPrintf(PETSC_COMM_WORLD,"\nDivergence because of indefinite preconditioner;\n");
PetscPrintf(PETSC_COMM_WORLD,"Run the executable again but with -pc_factor_shift_positive_definite option.\n");
}
else if (reason<0)
{
PetscPrintf(PETSC_COMM_WORLD,"\nOther kind of divergence: this should not happen.\n");
}
else
{
const char *slv_type;
const char *prc_type;
KSPGetType(lsolver, &slv_type);
PCGetType(prec, &prc_type);
PetscPrintf(PETSC_COMM_WORLD,"\n================================================");
PetscPrintf(PETSC_COMM_WORLD, "\nLinear solver %s with %s preconditioner",
slv_type, prc_type);
KSPGetIterationNumber(lsolver,&its); //CHKERRQ(ierr);
PetscPrintf(PETSC_COMM_WORLD,"\nConvergence in %d iterations.\n",(int)its);
PetscPrintf(PETSC_COMM_WORLD,"\n================================================");
}
PetscPrintf(PETSC_COMM_WORLD,"\n");
//VecAssemblyBegin(x);
//VecAssemblyEnd(x);
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v2);
#else
PetscGetTime(&v2);
#endif
time_elapsed += v2-v1;
#define aTEST_OUT
#ifdef TEST_OUT
//TEST
PetscViewer viewer;
PetscViewerASCIIOpen(PETSC_COMM_WORLD, "x2.txt", &viewer);
PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_MATLAB);
PetscObjectSetName((PetscObject)A,"Matrix");
MatView(A, viewer);
PetscObjectSetName((PetscObject)x,"Solution");
VecView(x, viewer);
PetscObjectSetName((PetscObject)b,"RHS");
VecView(b, viewer);
VecDestroy(&b);
VecDestroy(&x);
MatDestroy(&A);
if(lsolver) KSPDestroy(&lsolver);
// if(prec) PCDestroy(&prec);
if(global_x0)
delete [] global_x0;
if(global_x1)
delete [] global_x1;
PetscFinalize();
exit(0);
#endif
#ifdef TEST_MEM_PETSC
//TEST
PetscMemoryGetCurrentUsage(&mem2);
PetscPrintf(PETSC_COMM_WORLD, "###Memory usage by solver. Before :%f After:%f Increase:%d\n", mem1, mem2, (int)(mem2 - mem1));
#endif
}
int main(int argc,char **argv)
{
PetscLogDouble x,y;
int i,ierr;
PetscInitialize(&argc,&argv,0,0);
/* To take care of paging effects */
PetscTime(y);
for (i=0; i<2; i++) {
PetscTime(x);
PetscTime(y);
PetscTime(y);
PetscTime(y);
PetscTime(y);
PetscTime(y);
PetscTime(y);
PetscTime(y);
PetscTime(y);
PetscTime(y);
PetscTime(y);
fprintf(stdout,"%-15s : %e sec\n","PetscTime",(y-x)/10.0);
}
PetscTime(x);
ierr = PetscSleep(10);CHKERRQ(ierr);
PetscTime(y);
fprintf(stdout,"%-15s : %e sec - Slept for 10 sec \n","PetscTime",(y-x));
ierr = PetscFinalize();CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main(int argc,char **argv)
{
Mat M,Mo,C,K,Ko,A[3]; /* problem matrices */
PEP pep; /* polynomial eigenproblem solver context */
IS isf,isbc,is;
PetscInt n=200,nele,Istart,Iend,i,j,mloc,nloc,bc[2];
PetscReal width=0.05,height=0.005,glength=1.0,dlen,EI,area,rho;
PetscScalar K1[4],K2[4],K2t[4],K3[4],M1[4],M2[4],M2t[4],M3[4],damp=5.0;
PetscBool terse;
PetscErrorCode ierr;
PetscLogDouble time1,time2;
SlepcInitialize(&argc,&argv,(char*)0,help);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
nele = n/2;
n = 2*nele;
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nSimply supported beam damped in the middle, n=%D (nele=%D)\n\n",n,nele);CHKERRQ(ierr);
dlen = glength/nele;
EI = 7e10*width*height*height*height/12.0;
area = width*height;
rho = 0.674/(area*glength);
K1[0] = 12; K1[1] = 6*dlen; K1[2] = 6*dlen; K1[3] = 4*dlen*dlen;
K2[0] = -12; K2[1] = 6*dlen; K2[2] = -6*dlen; K2[3] = 2*dlen*dlen;
K2t[0] = -12; K2t[1] = -6*dlen; K2t[2] = 6*dlen; K2t[3] = 2*dlen*dlen;
K3[0] = 12; K3[1] = -6*dlen; K3[2] = -6*dlen; K3[3] = 4*dlen*dlen;
M1[0] = 156; M1[1] = 22*dlen; M1[2] = 22*dlen; M1[3] = 4*dlen*dlen;
M2[0] = 54; M2[1] = -13*dlen; M2[2] = 13*dlen; M2[3] = -3*dlen*dlen;
M2t[0] = 54; M2t[1] = 13*dlen; M2t[2] = -13*dlen; M2t[3] = -3*dlen*dlen;
M3[0] = 156; M3[1] = -22*dlen; M3[2] = -22*dlen; M3[3] = 4*dlen*dlen;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the matrices that define the eigensystem, (k^2*M+k*C+K)x=0
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* K is block-tridiagonal */
ierr = MatCreate(PETSC_COMM_WORLD,&Ko);CHKERRQ(ierr);
ierr = MatSetSizes(Ko,PETSC_DECIDE,PETSC_DECIDE,n+2,n+2);CHKERRQ(ierr);
ierr = MatSetBlockSize(Ko,2);CHKERRQ(ierr);
ierr = MatSetFromOptions(Ko);CHKERRQ(ierr);
ierr = MatSetUp(Ko);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Ko,&Istart,&Iend);CHKERRQ(ierr);
for (i=Istart/2;i<Iend/2;i++) {
if (i>0) {
j = i-1;
ierr = MatSetValuesBlocked(Ko,1,&i,1,&j,K2t,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValuesBlocked(Ko,1,&i,1,&i,K3,ADD_VALUES);CHKERRQ(ierr);
}
if (i<nele) {
j = i+1;
ierr = MatSetValuesBlocked(Ko,1,&i,1,&j,K2,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValuesBlocked(Ko,1,&i,1,&i,K1,ADD_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(Ko,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Ko,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatScale(Ko,EI/(dlen*dlen*dlen));CHKERRQ(ierr);
/* M is block-tridiagonal */
ierr = MatCreate(PETSC_COMM_WORLD,&Mo);CHKERRQ(ierr);
ierr = MatSetSizes(Mo,PETSC_DECIDE,PETSC_DECIDE,n+2,n+2);CHKERRQ(ierr);
ierr = MatSetBlockSize(Mo,2);CHKERRQ(ierr);
ierr = MatSetFromOptions(Mo);CHKERRQ(ierr);
ierr = MatSetUp(Mo);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(Mo,&Istart,&Iend);CHKERRQ(ierr);
for (i=Istart/2;i<Iend/2;i++) {
if (i>0) {
j = i-1;
ierr = MatSetValuesBlocked(Mo,1,&i,1,&j,M2t,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValuesBlocked(Mo,1,&i,1,&i,M3,ADD_VALUES);CHKERRQ(ierr);
}
if (i<nele) {
j = i+1;
ierr = MatSetValuesBlocked(Mo,1,&i,1,&j,M2,ADD_VALUES);CHKERRQ(ierr);
ierr = MatSetValuesBlocked(Mo,1,&i,1,&i,M1,ADD_VALUES);CHKERRQ(ierr);
}
}
ierr = MatAssemblyBegin(Mo,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(Mo,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatScale(Mo,rho*area*dlen/420);CHKERRQ(ierr);
/* remove rows/columns from K and M corresponding to boundary conditions */
ierr = ISCreateStride(PETSC_COMM_WORLD,Iend-Istart,Istart,1,&isf);CHKERRQ(ierr);
bc[0] = 0; bc[1] = n;
ierr = ISCreateGeneral(PETSC_COMM_SELF,2,bc,PETSC_USE_POINTER,&isbc);CHKERRQ(ierr);
ierr = ISDifference(isf,isbc,&is);CHKERRQ(ierr);
ierr = MatGetSubMatrix(Ko,is,is,MAT_INITIAL_MATRIX,&K);CHKERRQ(ierr);
ierr = MatGetSubMatrix(Mo,is,is,MAT_INITIAL_MATRIX,&M);CHKERRQ(ierr);
ierr = MatGetLocalSize(M,&mloc,&nloc);CHKERRQ(ierr);
/* C is zero except for the (nele,nele)-entry */
ierr = MatCreate(PETSC_COMM_WORLD,&C);CHKERRQ(ierr);
ierr = MatSetSizes(C,mloc,nloc,PETSC_DECIDE,PETSC_DECIDE);CHKERRQ(ierr);
ierr = MatSetFromOptions(C);CHKERRQ(ierr);
ierr = MatSetUp(C);CHKERRQ(ierr);
ierr = MatGetOwnershipRange(C,&Istart,&Iend);CHKERRQ(ierr);
if (nele-1>=Istart && nele-1<Iend) {
ierr = MatSetValue(C,nele-1,nele-1,damp,INSERT_VALUES);CHKERRQ(ierr);
}
ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the eigensolver and solve the problem
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PEPCreate(PETSC_COMM_WORLD,&pep);CHKERRQ(ierr);
A[0] = K; A[1] = C; A[2] = M;
ierr = PEPSetOperators(pep,3,A);CHKERRQ(ierr);
ierr = PEPSetFromOptions(pep);CHKERRQ(ierr);
ierr = PetscTime(&time1); CHKERRQ(ierr);
ierr = PEPSolve(pep);CHKERRQ(ierr);
ierr = PetscTime(&time2); CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Display solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* show detailed info unless -terse option is given by user */
ierr = PetscOptionsHasName(NULL,"-terse",&terse);CHKERRQ(ierr);
if (terse) {
ierr = PEPErrorView(pep,PEP_ERROR_BACKWARD,NULL);CHKERRQ(ierr);
} else {
ierr = PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO_DETAIL);CHKERRQ(ierr);
ierr = PEPReasonView(pep,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PEPErrorView(pep,PEP_ERROR_BACKWARD,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_WORLD,"Time: %g\n\n\n",time2-time1);CHKERRQ(ierr);
ierr = PEPDestroy(&pep);CHKERRQ(ierr);
ierr = ISDestroy(&isf);CHKERRQ(ierr);
ierr = ISDestroy(&isbc);CHKERRQ(ierr);
ierr = ISDestroy(&is);CHKERRQ(ierr);
ierr = MatDestroy(&M);CHKERRQ(ierr);
ierr = MatDestroy(&C);CHKERRQ(ierr);
ierr = MatDestroy(&K);CHKERRQ(ierr);
ierr = MatDestroy(&Ko);CHKERRQ(ierr);
ierr = MatDestroy(&Mo);CHKERRQ(ierr);
ierr = SlepcFinalize();CHKERRQ(ierr);
return 0;
}
int test2(void)
{
PetscLogDouble t1,t2;
double value;
int i,ierr,z[20000],zi[20000],intval,tmp;
PetscScalar x[20000],y[20000];
PetscRandom r;
ierr = PetscRandomCreate(PETSC_COMM_SELF,&r);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(r);CHKERRQ(ierr);
/* Take care of paging effects */
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<20000; i++) {
x[i] = i;
y[i] = i;
z[i] = i;
zi[i] = i;
}
/* Form the random set of integers */
for (i=0; i<20000; i++) {
ierr = PetscRandomGetValue(r,&value);CHKERRQ(ierr);
intval = (int)(value*20000.0);
tmp = z[i];
z[i] = z[intval];
z[intval] = tmp;
}
for (i=0; i<20000; i++) {
ierr = PetscRandomGetValue(r,&value);CHKERRQ(ierr);
intval = (int)(value*20000.0);
tmp = zi[i];
zi[i] = zi[intval];
zi[intval] = tmp;
}
/* fprintf(stdout,"Done setup\n"); */
/* ierr = BlastCache();CHKERRQ(ierr); */
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<2000; i++) x[i] = y[i];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[i] = y[i]",(t2-t1)/2000.0);
/* ierr = BlastCache();CHKERRQ(ierr); */
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<2000; i++) y[i] = x[z[i]];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[i] = y[idx[i]]",(t2-t1)/2000.0);
/* ierr = BlastCache();CHKERRQ(ierr); */
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<2000; i++) x[z[i]] = y[i];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[z[i]] = y[i]",(t2-t1)/2000.0);
/* ierr = BlastCache();CHKERRQ(ierr); */
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<2000; i++) y[z[i]] = x[zi[i]];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[z[i]] = y[zi[i]]",(t2-t1)/2000.0);
ierr = PetscRandomDestroy(&r);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
int main ( int argc, char* argv[] )
{
/* parse command line arguments */
std::string anArg;
std::string modelRoot;
for( int i = 1; i < argc; i++ )
{
anArg = std::string( argv[i] );
if( anArg == "--help" || anArg == "-h")
{
std::cout << "Usage: ogs [MODEL_ROOT] [OPTIONS]\n"
<< "Where OPTIONS are:\n"
<< " -h [--help] print this message and exit\n"
<< " -b [--build-info] print build info and exit\n"
<< " --output-directory DIR put output files into DIR\n"
<< " --version print ogs version and exit" << "\n";
continue;
}
if( anArg == "--build-info" || anArg == "-b" )
{
std::cout << "ogs version: " << OGS_VERSION << "\n"
<< "ogs date: " << OGS_DATE << "\n";
#ifdef CMAKE_CMD_ARGS
std::cout << "cmake command line arguments: " << CMAKE_CMD_ARGS << "\n";
#endif // CMAKE_CMD_ARGS
#ifdef GIT_COMMIT_INFO
std::cout << "git commit info: " << GIT_COMMIT_INFO << "\n";
#endif // GIT_COMMIT_INFO
#ifdef SVN_REVISION
std::cout << "subversion info: " << SVN_REVISION << "\n";
#endif // SVN_REVISION
#ifdef BUILD_TIMESTAMP
std::cout << "build timestamp: " << BUILD_TIMESTAMP << "\n";
#endif // BUILD_TIMESTAMP
continue;
}
if( anArg == "--version" )
{
std::cout << OGS_VERSION << "\n";
continue;
}
if( anArg == "--model-root" || anArg == "-m" )
{
if (i+1 >= argc) {
std::cerr << "Error: Parameter " << anArg << " needs an additional argument" << std::endl;
std::exit(EXIT_FAILURE);
}
modelRoot = std::string( argv[++i] );
continue;
}
if (anArg == "--output-directory")
{
if (i+1 >= argc) {
std::cerr << "Error: Parameter " << anArg << " needs an additional argument" << std::endl;
std::exit(EXIT_FAILURE);
}
std::string path = argv[++i];
if (! path.empty()) defaultOutputPath = path;
continue;
}
// anything left over must be the model root, unless already found
if ( modelRoot == "" )
modelRoot = std::string( argv[i] );
} // end of parse argc loop
if( argc > 1 && modelRoot == "" ) // non-interactive mode and no model given
exit(0); // e.g. just wanted the build info
char* dateiname(NULL);
#ifdef SUPERCOMPUTER
// *********************************************************************
// buffered output ... important for performance on cray
// (unbuffered output is limited to 10 bytes per second)
// [email protected] 11.10.2007
char buf[1024 * 1024];
int bsize;
bsize = 1024 * 1024; // question: what happens if buffer is full?
// according to documentation the buffer is flushed when full.
// If we have a lot of output, increasing buffer is usefull.
if(bsize > 0)
// bufstd = malloc(bsize);
setvbuf(stdout, buf, _IOFBF, bsize);
//**********************************************************************
#endif
/*---------- MPI Initialization ----------------------------------*/
#if defined(USE_MPI) || defined(USE_MPI_PARPROC) || defined(USE_MPI_REGSOIL) || \
defined(USE_MPI_GEMS) || defined(USE_MPI_KRC)
printf("Before MPI_Init\n");
#if defined(USE_MPI_GEMS)
int prov;
MPI_Init_thread(&argc,&argv,MPI_THREAD_FUNNELED, &prov);
#else
MPI_Init(&argc,&argv);
#endif
MPI_Barrier (MPI_COMM_WORLD); // 12.09.2007 WW
elapsed_time_mpi = -MPI_Wtime(); // 12.09.2007 WW
MPI_Comm_size(MPI_COMM_WORLD,&mysize);
MPI_Comm_rank(MPI_COMM_WORLD,&myrank);
std::cout << "After MPI_Init myrank = " << myrank << '\n';
time_ele_paral = 0.0;
#endif
/*---------- MPI Initialization ----------------------------------*/
#ifdef USE_PETSC
int rank, r_size;
PetscLogDouble v1,v2;
char help[] = "OGS with PETSc \n";
//PetscInitialize(argc, argv, help);
PetscInitialize(&argc,&argv,(char *)0,help);
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v1);
#else
PetscGetTime(&v1);
#endif
MPI_Comm_rank(PETSC_COMM_WORLD, &rank);
MPI_Comm_size(PETSC_COMM_WORLD, &r_size);
PetscSynchronizedPrintf(PETSC_COMM_WORLD, "===\nUse PETSc solver");
PetscSynchronizedPrintf(PETSC_COMM_WORLD, "Number of CPUs: %d, rank: %d\n", r_size, rank);
#endif
/*---------- LIS solver -----------------------------------------*/
#ifdef LIS
//Initialization of the lis solver.
lis_initialize(&argc, &argv);
#endif
/*========================================================================*/
/* Kommunikation mit Betriebssystem */
/* Timer fuer Gesamtzeit starten */
#ifdef TESTTIME
TStartTimer(0);
#endif
/* Intro ausgeben */
#if defined(USE_MPI) //WW
if(myrank == 0)
#endif
#ifdef USE_PETSC
if(rank == 0 )
#endif
DisplayStartMsg();
/* Speicherverwaltung initialisieren */
if (!InitMemoryTest())
{
DisplayErrorMsg("Fehler: Speicherprotokoll kann nicht erstellt werden!");
DisplayErrorMsg(" Programm vorzeitig beendet!");
return 1; // LB changed from 0 to 1 because 0 is indicating success
}
if( argc == 1 ) // interactive mode
dateiname = ReadString();
else // non-interactive mode
{
if ( argc == 2 ) // a model root was supplied
{
dateiname = (char*) Malloc((int)strlen(argv[1]) + 1);
dateiname = strcpy(dateiname,argv[1]);
}
else // several args supplied
if( modelRoot != "")
{
dateiname = (char*) Malloc( (int) modelRoot.size() + 1 );
dateiname = strcpy( dateiname, modelRoot.c_str() );
}
DisplayMsgLn(dateiname);
}
//WW DisplayMsgLn("");
//WW DisplayMsgLn("");
// ----------23.02.2009. WW-----------------
// LB Check if file exists
std::string tmpFilename = dateiname;
tmpFilename.append(".pcs");
if(!IsFileExisting(tmpFilename))
{
std::cout << " Error: Cannot find file " << dateiname << "\n";
return 1;
}
// If no option is given, output files are placed in the same directory as the input files
if (defaultOutputPath.empty()) defaultOutputPath = pathDirname(std::string(dateiname));
FileName = dateiname;
size_t indexChWin, indexChLinux;
indexChWin = indexChLinux = 0;
indexChWin = FileName.find_last_of('\\');
indexChLinux = FileName.find_last_of('/');
//
if(indexChWin != std::string::npos)
FilePath = FileName.substr(0,indexChWin) + "\\";
else if(indexChLinux != std::string::npos)
FilePath = FileName.substr(0,indexChLinux) + "/";
// ---------------------------WW
Problem* aproblem = new Problem(dateiname);
#ifdef USE_PETSC
aproblem->setRankandSize(rank, r_size);
#endif
#if defined(USE_MPI) || defined(USE_MPI_PARPROC) || defined(USE_MPI_REGSOIL) || defined(USE_MPI_GEMS) || defined(USE_MPI_KRC)
aproblem->setRankandSize(myrank, mysize);
#endif
aproblem->Euler_TimeDiscretize();
delete aproblem;
aproblem = NULL;
if(ClockTimeVec.size()>0)
ClockTimeVec[0]->PrintTimes(); //CB time
DestroyClockTime();
#ifdef TESTTIME
#if defined(USE_MPI)
if(myrank == 0)
#endif
#if defined(USE_PETSC)
if(rank == 0)
#endif
std::cout << "Simulation time: " << TGetTimer(0) << "s" << "\n";
#endif
/* Abspann ausgeben */
/*--------- MPI Finalize ------------------*/
#if defined(USE_MPI) || defined(USE_MPI_PARPROC) || defined(USE_MPI_REGSOIL) || defined(USE_MPI_KRC)
elapsed_time_mpi += MPI_Wtime(); // 12.09.2007 WW
std::cout << "\n *** Total CPU time of parallel modeling: " << elapsed_time_mpi <<
"\n"; //WW
// Count CPU time of post time loop WW
MPI_Finalize();
#endif
/*--------- MPI Finalize ------------------*/
/*--------- LIS Finalize ------------------*/
#ifdef LIS
lis_finalize();
#endif
/*--------- LIS Finalize ------------------*/
free(dateiname);
#ifdef USE_PETSC
//kg44 quick fix to compile PETSC with version PETSCV3.4
#ifdef USEPETSC34
PetscTime(&v2);
#else
PetscGetTime(&v2);
#endif
PetscPrintf(PETSC_COMM_WORLD,"\t\n>>Total elapsed time by using PETSC:%f s\n",v2-v1);
PetscFinalize();
#endif
return 0;
}
PetscErrorCode MatLUFactorNumeric_SuperLU_DIST(Mat F,Mat A,const MatFactorInfo *info)
{
Mat *tseq,A_seq = NULL;
Mat_SeqAIJ *aa,*bb;
Mat_SuperLU_DIST *lu = (Mat_SuperLU_DIST*)(F)->spptr;
PetscErrorCode ierr;
PetscInt M=A->rmap->N,N=A->cmap->N,i,*ai,*aj,*bi,*bj,nz,rstart,*garray,
m=A->rmap->n, colA_start,j,jcol,jB,countA,countB,*bjj,*ajj;
int sinfo; /* SuperLU_Dist info flag is always an int even with long long indices */
PetscMPIInt size;
SuperLUStat_t stat;
double *berr=0;
IS isrow;
PetscLogDouble time0,time,time_min,time_max;
Mat F_diag=NULL;
#if defined(PETSC_USE_COMPLEX)
doublecomplex *av, *bv;
#else
double *av, *bv;
#endif
PetscFunctionBegin;
ierr = MPI_Comm_size(PetscObjectComm((PetscObject)A),&size);CHKERRQ(ierr);
if (lu->options.PrintStat) { /* collect time for mat conversion */
ierr = MPI_Barrier(PetscObjectComm((PetscObject)A));CHKERRQ(ierr);
ierr = PetscTime(&time0);CHKERRQ(ierr);
}
if (lu->MatInputMode == GLOBAL) { /* global mat input */
if (size > 1) { /* convert mpi A to seq mat A */
ierr = ISCreateStride(PETSC_COMM_SELF,M,0,1,&isrow);CHKERRQ(ierr);
ierr = MatGetSubMatrices(A,1,&isrow,&isrow,MAT_INITIAL_MATRIX,&tseq);CHKERRQ(ierr);
ierr = ISDestroy(&isrow);CHKERRQ(ierr);
A_seq = *tseq;
ierr = PetscFree(tseq);CHKERRQ(ierr);
aa = (Mat_SeqAIJ*)A_seq->data;
} else {
PetscBool flg;
ierr = PetscObjectTypeCompare((PetscObject)A,MATMPIAIJ,&flg);CHKERRQ(ierr);
if (flg) {
Mat_MPIAIJ *At = (Mat_MPIAIJ*)A->data;
A = At->A;
}
aa = (Mat_SeqAIJ*)A->data;
}
/* Convert Petsc NR matrix to SuperLU_DIST NC.
Note: memories of lu->val, col and row are allocated by CompRow_to_CompCol_dist()! */
if (lu->options.Fact != DOFACT) {/* successive numeric factorization, sparsity pattern is reused. */
PetscStackCall("SuperLU_DIST:Destroy_CompCol_Matrix_dist",Destroy_CompCol_Matrix_dist(&lu->A_sup));
if (lu->FactPattern == SamePattern_SameRowPerm) {
lu->options.Fact = SamePattern_SameRowPerm; /* matrix has similar numerical values */
} else { /* lu->FactPattern == SamePattern */
PetscStackCall("SuperLU_DIST:Destroy_LU",Destroy_LU(N, &lu->grid, &lu->LUstruct));
lu->options.Fact = SamePattern;
}
}
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCompRow_to_CompCol_dist",zCompRow_to_CompCol_dist(M,N,aa->nz,(doublecomplex*)aa->a,aa->j,aa->i,&lu->val,&lu->col, &lu->row));
#else
PetscStackCall("SuperLU_DIST:dCompRow_to_CompCol_dist",dCompRow_to_CompCol_dist(M,N,aa->nz,aa->a,aa->j,aa->i,&lu->val, &lu->col, &lu->row));
#endif
/* Create compressed column matrix A_sup. */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCreate_CompCol_Matrix_dist",zCreate_CompCol_Matrix_dist(&lu->A_sup, M, N, aa->nz, lu->val, lu->col, lu->row, SLU_NC, SLU_Z, SLU_GE));
#else
PetscStackCall("SuperLU_DIST:dCreate_CompCol_Matrix_dist",dCreate_CompCol_Matrix_dist(&lu->A_sup, M, N, aa->nz, lu->val, lu->col, lu->row, SLU_NC, SLU_D, SLU_GE));
#endif
} else { /* distributed mat input */
Mat_MPIAIJ *mat = (Mat_MPIAIJ*)A->data;
aa=(Mat_SeqAIJ*)(mat->A)->data;
bb=(Mat_SeqAIJ*)(mat->B)->data;
ai=aa->i; aj=aa->j;
bi=bb->i; bj=bb->j;
#if defined(PETSC_USE_COMPLEX)
av=(doublecomplex*)aa->a;
bv=(doublecomplex*)bb->a;
#else
av=aa->a;
bv=bb->a;
#endif
rstart = A->rmap->rstart;
nz = aa->nz + bb->nz;
garray = mat->garray;
if (lu->options.Fact == DOFACT) { /* first numeric factorization */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zallocateA_dist",zallocateA_dist(m, nz, &lu->val, &lu->col, &lu->row));
#else
PetscStackCall("SuperLU_DIST:dallocateA_dist",dallocateA_dist(m, nz, &lu->val, &lu->col, &lu->row));
#endif
} else { /* successive numeric factorization, sparsity pattern and perm_c are reused. */
/* Destroy_CompRowLoc_Matrix_dist(&lu->A_sup); */ /* this leads to crash! However, see SuperLU_DIST_2.5/EXAMPLE/pzdrive2.c */
if (lu->FactPattern == SamePattern_SameRowPerm) {
lu->options.Fact = SamePattern_SameRowPerm; /* matrix has similar numerical values */
} else {
PetscStackCall("SuperLU_DIST:Destroy_LU",Destroy_LU(N, &lu->grid, &lu->LUstruct)); /* Deallocate storage associated with the L and U matrices. */
lu->options.Fact = SamePattern;
}
}
nz = 0;
for (i=0; i<m; i++) {
lu->row[i] = nz;
countA = ai[i+1] - ai[i];
countB = bi[i+1] - bi[i];
ajj = aj + ai[i]; /* ptr to the beginning of this row */
bjj = bj + bi[i];
/* B part, smaller col index */
colA_start = rstart + ajj[0]; /* the smallest global col index of A */
jB = 0;
for (j=0; j<countB; j++) {
jcol = garray[bjj[j]];
if (jcol > colA_start) {
jB = j;
break;
}
lu->col[nz] = jcol;
lu->val[nz++] = *bv++;
if (j==countB-1) jB = countB;
}
/* A part */
for (j=0; j<countA; j++) {
lu->col[nz] = rstart + ajj[j];
lu->val[nz++] = *av++;
}
/* B part, larger col index */
for (j=jB; j<countB; j++) {
lu->col[nz] = garray[bjj[j]];
lu->val[nz++] = *bv++;
}
}
lu->row[m] = nz;
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:zCreate_CompRowLoc_Matrix_dist",zCreate_CompRowLoc_Matrix_dist(&lu->A_sup, M, N, nz, m, rstart,lu->val, lu->col, lu->row, SLU_NR_loc, SLU_Z, SLU_GE));
#else
PetscStackCall("SuperLU_DIST:dCreate_CompRowLoc_Matrix_dist",dCreate_CompRowLoc_Matrix_dist(&lu->A_sup, M, N, nz, m, rstart,lu->val, lu->col, lu->row, SLU_NR_loc, SLU_D, SLU_GE));
#endif
}
if (lu->options.PrintStat) {
ierr = PetscTime(&time);CHKERRQ(ierr);
time0 = time - time0;
}
/* Factor the matrix. */
PetscStackCall("SuperLU_DIST:PStatInit",PStatInit(&stat)); /* Initialize the statistics variables. */
if (lu->MatInputMode == GLOBAL) { /* global mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx_ABglobal",pzgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, M, 0,&lu->grid, &lu->LUstruct, berr, &stat, &sinfo));
#else
PetscStackCall("SuperLU_DIST:pdgssvx_ABglobal",pdgssvx_ABglobal(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, M, 0,&lu->grid, &lu->LUstruct, berr, &stat, &sinfo));
#endif
} else { /* distributed mat input */
#if defined(PETSC_USE_COMPLEX)
PetscStackCall("SuperLU_DIST:pzgssvx",pzgssvx(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, m, 0, &lu->grid,&lu->LUstruct, &lu->SOLVEstruct, berr, &stat, &sinfo));
if (sinfo) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pzgssvx fails, info: %d\n",sinfo);
#else
PetscStackCall("SuperLU_DIST:pdgssvx",pdgssvx(&lu->options, &lu->A_sup, &lu->ScalePermstruct, 0, m, 0, &lu->grid,&lu->LUstruct, &lu->SOLVEstruct, berr, &stat, &sinfo));
if (sinfo) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_LIB,"pdgssvx fails, info: %d\n",sinfo);
#endif
}
if (lu->MatInputMode == GLOBAL && size > 1) {
ierr = MatDestroy(&A_seq);CHKERRQ(ierr);
}
if (lu->options.PrintStat) {
ierr = MPI_Reduce(&time0,&time_max,1,MPI_DOUBLE,MPI_MAX,0,PetscObjectComm((PetscObject)A));
ierr = MPI_Reduce(&time0,&time_min,1,MPI_DOUBLE,MPI_MIN,0,PetscObjectComm((PetscObject)A));
ierr = MPI_Reduce(&time0,&time,1,MPI_DOUBLE,MPI_SUM,0,PetscObjectComm((PetscObject)A));
time = time/size; /* average time */
ierr = PetscPrintf(PetscObjectComm((PetscObject)A), " Mat conversion(PETSc->SuperLU_DIST) time (max/min/avg): \n %g / %g / %g\n",time_max,time_min,time);CHKERRQ(ierr);
PStatPrint(&lu->options, &stat, &lu->grid); /* Print the statistics. */
}
PStatFree(&stat);
if (size > 1) {
F_diag = ((Mat_MPIAIJ*)(F)->data)->A;
F_diag->assembled = PETSC_TRUE;
}
(F)->assembled = PETSC_TRUE;
(F)->preallocated = PETSC_TRUE;
lu->options.Fact = FACTORED; /* The factored form of A is supplied. Local option used by this func. only */
PetscFunctionReturn(0);
}
void tik() {PetscTime(&time_);}
int main(int argc,char **args)
{
Mat A; /* linear system matrix */
PetscErrorCode ierr;
PetscMPIInt rank=0;
PetscBool flg;
PetscViewer fd; /* viewer */
PetscViewer log;
char file[PETSC_MAX_PATH_LEN];
char logfile[PETSC_MAX_PATH_LEN];
char lockfile[PETSC_MAX_PATH_LEN], tmpstr[PETSC_MAX_PATH_LEN], dirname[PETSC_MAX_PATH_LEN], matrix[PETSC_MAX_PATH_LEN];
char hash[20];
PetscLogDouble solveTime,endTime,startTime;
PetscInt its;
PetscReal norm;
KSP ksp; // Linear solver context
Vec b,x,u; // RHS, solution, vector for norm calculation
PetscScalar one = 1.0;
PetscInt m, n, i;
FILE *lock;
/*
if (rank == 0) {
printf("Command line arguments:\n");
for (i=0; i < argc; i++)
printf("%d: %s\n", i, args[i]);
}
// Save args
int argcount = argc;
char **argv = (char**) malloc (argc*sizeof(char*));
for (i=0; i < argc; i++) {
argv[i] = (char*) malloc(strlen(args[i]) + 1);
strcpy(argv[i],args[i]);
}
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
*/
PetscInitialize(&argc,&args,(char *)0,help);
ierr = MPI_Comm_rank(PETSC_COMM_WORLD,&rank);CHKERRQ(ierr);
ierr = PetscOptionsGetString(PETSC_NULL,"-hash",hash,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
strcpy(hash,"nohash");
}
ierr = PetscOptionsGetString(PETSC_NULL,"-f",file,PETSC_MAX_PATH_LEN,&flg);CHKERRQ(ierr);
if (!flg) {
PetscPrintf(PETSC_COMM_WORLD,"Must indicate matrix file with the -f option");
}
/* Create lock file */
if (rank == 0) {
for (i = strlen(file); i> 0; i--) if (file[i] == '.') break;
strncpy(tmpstr, file, i-1);
for (i = strlen(tmpstr); i> 0; i--) if (file[i] == '/') break;
strncpy(dirname, tmpstr, i);
dirname[i] = '\0';
sprintf(lockfile,"%s/../timing/.%s.%s", dirname, basename(tmpstr), hash);
sprintf(logfile,"%s/../timing/%s.%s.log", dirname, basename(tmpstr), hash);
lock = fopen(lockfile, "w");
fprintf(lock, "%s\n", file);
fclose(lock);
}
/* Read file */
ierr = PetscViewerBinaryOpen(PETSC_COMM_WORLD,file,FILE_MODE_READ,&fd);CHKERRQ(ierr);
// Create matrix
ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr);
ierr = MatSetType(A,MATMPIAIJ); CHKERRQ(ierr);
ierr = MatSetFromOptions(A);CHKERRQ(ierr);
// Load matrix from file
ierr = MatLoad(A,fd);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&fd);CHKERRQ(ierr);
ierr = MatGetSize(A, &m, &n); CHKERRQ(ierr);
// Assemble matrix
//ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
//ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
// Create RHS vector
ierr = VecCreate(PETSC_COMM_WORLD,&b);CHKERRQ(ierr);
ierr = VecSetSizes(b,PETSC_DECIDE,n); CHKERRQ(ierr);
ierr = VecSetFromOptions(b);CHKERRQ(ierr);
ierr = VecSet(b,one); CHKERRQ(ierr);
//ierr = VecLoad(b,fd);CHKERRQ(ierr);
// Create vectors x and u
ierr = VecDuplicate(b,&x);CHKERRQ(ierr);
ierr = VecDuplicate(b,&u);CHKERRQ(ierr);
// Create KSP
ierr = KSPCreate(PETSC_COMM_WORLD,&ksp); CHKERRQ(ierr);
ierr = KSPSetInitialGuessNonzero(ksp,PETSC_FALSE);CHKERRQ(ierr);
ierr = KSPSetOperators(ksp,A,A);CHKERRQ(ierr);
ierr = KSPSetFromOptions(ksp); CHKERRQ(ierr);
// Setup KSP
ierr = KSPSetUp(ksp);CHKERRQ(ierr);
ierr = KSPSetUpOnBlocks(ksp);CHKERRQ(ierr);
// Get start time
ierr = PetscTime(&startTime);CHKERRQ(ierr);
// Get KSP and PC type
KSPType kt;
ierr = KSPGetType(ksp,&kt);
PC pc;
ierr = KSPGetPC(ksp,&pc);
PCType pt;
ierr = PCGetType(pc,&pt);
// Print method info
ierr = PetscViewerASCIIOpen(PETSC_COMM_WORLD, logfile, &log); CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(log, "Hash: %s\n", hash);
ierr = PetscViewerASCIIPrintf(log, "%s | %s",kt,pt);CHKERRQ(ierr);
// Make sure the program doesn't crash
// while trying to solve the system
PetscPushErrorHandler(PetscIgnoreErrorHandler,NULL);
ierr = KSPSolve(ksp,b,x);
PetscPopErrorHandler();
// Check if anything went wrong
if(ierr == 0 || ierr == -1){
// If no error occurred or stopped by MyKSPMonitor,
// compute normal and stuff
ierr = KSPGetIterationNumber(ksp,&its);CHKERRQ(ierr);
ierr = MatMult(A,x,u);CHKERRQ(ierr);
ierr = VecAXPY(u,-1.0,b);CHKERRQ(ierr);
ierr = VecNorm(u,NORM_2,&norm);CHKERRQ(ierr);
ierr = PetscTime(&endTime);CHKERRQ(ierr);
// Compute solve time
solveTime = endTime - startTime;
// Check if KSP converged
KSPConvergedReason reason;
KSPGetConvergedReason(ksp,&reason);
// Print convergence code, solve time, preconditioned norm, iterations
ierr = PetscViewerASCIIPrintf(log, " | %D | %e | %g | %D\n",reason,solveTime,norm,its);CHKERRQ(ierr);
ierr = KSPView(ksp,log);
ierr = PCView(pc,log);
ierr = PetscLogView(log);
}
else{
// Disaster happened, bail out
if (rank == 0) remove(lockfile);
PetscFinalize();
return 0;
}
// Again, destroy KSP and vector
ierr = KSPDestroy(&ksp);CHKERRQ(ierr);
ierr = VecDestroy(&x);CHKERRQ(ierr);
ierr = VecDestroy(&b);CHKERRQ(ierr);
ierr = VecDestroy(&u);CHKERRQ(ierr);
if (rank == 0) remove(lockfile);
PetscFinalize();
return 0;
}
int main(int argc,char **argv)
{
Mat A; /* operator matrix */
EPS eps; /* eigenproblem solver context */
EPSType type;
DM da;
Vec v0;
PetscReal error,tol,re,im,*exact;
PetscScalar kr,ki;
PetscInt M,N,P,m,n,p,nev,maxit,i,its,nconv,seed;
PetscLogDouble t1,t2,t3;
PetscBool flg;
PetscRandom rctx;
PetscErrorCode ierr;
SlepcInitialize(&argc,&argv,(char*)0,help);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n3-D Laplacian Eigenproblem\n\n");CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the operator matrix that defines the eigensystem, Ax=kx
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = DMDACreate3d(PETSC_COMM_WORLD,DM_BOUNDARY_NONE,DM_BOUNDARY_NONE,
DM_BOUNDARY_NONE,DMDA_STENCIL_STAR,-10,-10,-10,
PETSC_DECIDE,PETSC_DECIDE,PETSC_DECIDE,
1,1,NULL,NULL,NULL,&da);CHKERRQ(ierr);
/* print DM information */
ierr = DMDAGetInfo(da,NULL,&M,&N,&P,&m,&n,&p,NULL,NULL,NULL,NULL,NULL,NULL);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Grid partitioning: %D %D %D\n",m,n,p);CHKERRQ(ierr);
/* create and fill the matrix */
ierr = DMCreateMatrix(da,&A);CHKERRQ(ierr);
ierr = FillMatrix(da,A);CHKERRQ(ierr);
/* create random initial vector */
seed = 1;
ierr = PetscOptionsGetInt(NULL,"-seed",&seed,NULL);CHKERRQ(ierr);
if (seed<0) SETERRQ(PETSC_COMM_WORLD,1,"Seed must be >=0");
ierr = MatGetVecs(A,&v0,NULL);CHKERRQ(ierr);
ierr = PetscRandomCreate(PETSC_COMM_WORLD,&rctx);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(rctx);CHKERRQ(ierr);
for (i=0;i<seed;i++) { /* simulate different seeds in the random generator */
ierr = VecSetRandom(v0,rctx);CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the eigensolver and set various options
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Create eigensolver context
*/
ierr = EPSCreate(PETSC_COMM_WORLD,&eps);CHKERRQ(ierr);
/*
Set operators. In this case, it is a standard eigenvalue problem
*/
ierr = EPSSetOperators(eps,A,NULL);CHKERRQ(ierr);
ierr = EPSSetProblemType(eps,EPS_HEP);CHKERRQ(ierr);
/*
Set specific solver options
*/
ierr = EPSSetWhichEigenpairs(eps,EPS_SMALLEST_REAL);CHKERRQ(ierr);
ierr = EPSSetTolerances(eps,1e-8,PETSC_DEFAULT);CHKERRQ(ierr);
ierr = EPSSetInitialSpace(eps,1,&v0);CHKERRQ(ierr);
/*
Set solver parameters at runtime
*/
ierr = EPSSetFromOptions(eps);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Solve the eigensystem
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PetscTime(&t1);CHKERRQ(ierr);
ierr = EPSSetUp(eps);CHKERRQ(ierr);
ierr = PetscTime(&t2);CHKERRQ(ierr);
ierr = EPSSolve(eps);CHKERRQ(ierr);
ierr = PetscTime(&t3);CHKERRQ(ierr);
ierr = EPSGetIterationNumber(eps,&its);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of iterations of the method: %D\n",its);CHKERRQ(ierr);
/*
Optional: Get some information from the solver and display it
*/
ierr = EPSGetType(eps,&type);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Solution method: %s\n\n",type);CHKERRQ(ierr);
ierr = EPSGetDimensions(eps,&nev,NULL,NULL);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of requested eigenvalues: %D\n",nev);CHKERRQ(ierr);
ierr = EPSGetTolerances(eps,&tol,&maxit);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Stopping condition: tol=%.4g, maxit=%D\n",(double)tol,maxit);CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Display solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/*
Get number of converged approximate eigenpairs
*/
ierr = EPSGetConverged(eps,&nconv);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD," Number of converged approximate eigenpairs: %D\n\n",nconv);CHKERRQ(ierr);
if (nconv>0) {
ierr = PetscMalloc1(nconv,&exact);CHKERRQ(ierr);
ierr = GetExactEigenvalues(M,N,P,nconv,exact);CHKERRQ(ierr);
/*
Display eigenvalues and relative errors
*/
ierr = PetscPrintf(PETSC_COMM_WORLD,
" k ||Ax-kx||/||kx|| Eigenvalue Error \n"
" ----------------- ------------------ ------------------\n");CHKERRQ(ierr);
for (i=0;i<nconv;i++) {
/*
Get converged eigenpairs: i-th eigenvalue is stored in kr (real part) and
ki (imaginary part)
*/
ierr = EPSGetEigenpair(eps,i,&kr,&ki,NULL,NULL);CHKERRQ(ierr);
/*
Compute the relative error associated to each eigenpair
*/
ierr = EPSComputeRelativeError(eps,i,&error);CHKERRQ(ierr);
#if defined(PETSC_USE_COMPLEX)
re = PetscRealPart(kr);
im = PetscImaginaryPart(kr);
#else
re = kr;
im = ki;
#endif
if (im!=0.0) SETERRQ(PETSC_COMM_WORLD,1,"Eigenvalue should be real");
else {
ierr = PetscPrintf(PETSC_COMM_WORLD," %12g %12g %12g\n",(double)re,(double)error,(double)PetscAbsReal(re-exact[i]));CHKERRQ(ierr);
}
}
ierr = PetscFree(exact);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"\n");CHKERRQ(ierr);
}
/*
Show computing times
*/
ierr = PetscOptionsHasName(NULL,"-showtimes",&flg);CHKERRQ(ierr);
if (flg) {
ierr = PetscPrintf(PETSC_COMM_WORLD," Elapsed time: %g (setup), %g (solve)\n",(double)(t2-t1),(double)(t3-t2));CHKERRQ(ierr);
}
/*
Free work space
*/
ierr = EPSDestroy(&eps);CHKERRQ(ierr);
ierr = MatDestroy(&A);CHKERRQ(ierr);
ierr = VecDestroy(&v0);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&rctx);CHKERRQ(ierr);
ierr = DMDestroy(&da);CHKERRQ(ierr);
ierr = SlepcFinalize();
return 0;
}
PetscErrorCode PetscLogEventEndComplete(PetscLogEvent event, int t, PetscObject o1, PetscObject o2, PetscObject o3, PetscObject o4)
{
PetscStageLog stageLog;
PetscEventRegLog eventRegLog;
PetscEventPerfLog eventPerfLog = NULL;
Action *tmpAction;
PetscLogDouble start, end;
PetscLogDouble curTime;
int stage;
PetscErrorCode ierr;
PetscFunctionBegin;
/* Dynamically enlarge logging structures */
if (petsc_numActions >= petsc_maxActions) {
PetscTime(&start);
ierr = PetscMalloc1(petsc_maxActions*2, &tmpAction);
CHKERRQ(ierr);
ierr = PetscMemcpy(tmpAction, petsc_actions, petsc_maxActions * sizeof(Action));
CHKERRQ(ierr);
ierr = PetscFree(petsc_actions);
CHKERRQ(ierr);
petsc_actions = tmpAction;
petsc_maxActions *= 2;
PetscTime(&end);
petsc_BaseTime += (end - start);
}
/* Record the event */
ierr = PetscLogGetStageLog(&stageLog);
CHKERRQ(ierr);
ierr = PetscStageLogGetCurrent(stageLog, &stage);
CHKERRQ(ierr);
ierr = PetscStageLogGetEventRegLog(stageLog, &eventRegLog);
CHKERRQ(ierr);
ierr = PetscStageLogGetEventPerfLog(stageLog, stage, &eventPerfLog);
CHKERRQ(ierr);
PetscTime(&curTime);
if (petsc_logActions) {
petsc_actions[petsc_numActions].time = curTime - petsc_BaseTime;
petsc_actions[petsc_numActions].action = ACTIONEND;
petsc_actions[petsc_numActions].event = event;
petsc_actions[petsc_numActions].classid = eventRegLog->eventInfo[event].classid;
if (o1) petsc_actions[petsc_numActions].id1 = o1->id;
else petsc_actions[petsc_numActions].id1 = -1;
if (o2) petsc_actions[petsc_numActions].id2 = o2->id;
else petsc_actions[petsc_numActions].id2 = -1;
if (o3) petsc_actions[petsc_numActions].id3 = o3->id;
else petsc_actions[petsc_numActions].id3 = -1;
petsc_actions[petsc_numActions].flops = petsc_TotalFlops;
ierr = PetscMallocGetCurrentUsage(&petsc_actions[petsc_numActions].mem);
CHKERRQ(ierr);
ierr = PetscMallocGetMaximumUsage(&petsc_actions[petsc_numActions].maxmem);
CHKERRQ(ierr);
petsc_numActions++;
}
/* Check for double counting */
eventPerfLog->eventInfo[event].depth--;
if (eventPerfLog->eventInfo[event].depth > 0) PetscFunctionReturn(0);
else if (eventPerfLog->eventInfo[event].depth < 0) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONGSTATE, "Logging event had unbalanced begin/end pairs");
/* Log the performance info */
eventPerfLog->eventInfo[event].count++;
eventPerfLog->eventInfo[event].time += curTime;
eventPerfLog->eventInfo[event].flops += petsc_TotalFlops;
eventPerfLog->eventInfo[event].numMessages += petsc_irecv_ct + petsc_isend_ct + petsc_recv_ct + petsc_send_ct;
eventPerfLog->eventInfo[event].messageLength += petsc_irecv_len + petsc_isend_len + petsc_recv_len + petsc_send_len;
eventPerfLog->eventInfo[event].numReductions += petsc_allreduce_ct + petsc_gather_ct + petsc_scatter_ct;
PetscFunctionReturn(0);
}
/*@
PetscGetTime - Returns the current time of day in seconds. This
returns wall-clock time.
Not Collective
Output Parameter:
. v - time counter
Usage:
.vb
#include "petsctime.h"
PetscLogDouble v1,v2,elapsed_time;
ierr = PetscGetTime(&v1);CHKERRQ(ierr);
.... perform some calculation ...
ierr = PetscGetTime(&v2);CHKERRQ(ierr);
elapsed_time = v2 - v1;
.ve
Notes:
Requires explicitly including "petsctime.h"
Since the PETSc libraries incorporate timing of phases and operations,
PetscGetTime() is intended only for timing of application codes.
The options database commands -log, -log_summary, and -log_all activate
PETSc library timing. See the users manual for further details.
Level: intermediate
.seealso: PetscLogEventRegister(), PetscLogEventBegin(), PetscLogEventEnd(), PetscLogStagePush(),
PetscLogStagePop(), PetscLogStageRegister(), PetscGetFlops()
.keywords: get, time
@*/
PetscErrorCode PETSC_DLLEXPORT PetscGetTime(PetscLogDouble *t)
{
PetscFunctionBegin;
PetscTime(*t);
PetscFunctionReturn(0);
}
int test1(void)
{
PetscLogDouble t1,t2;
double value;
int i,ierr,*z,*zi,intval;
PetscScalar *x,*y;
PetscRandom r;
ierr = PetscRandomCreate(PETSC_COMM_SELF,&r);CHKERRQ(ierr);
ierr = PetscRandomSetFromOptions(r);CHKERRQ(ierr);
ierr = PetscMalloc1(20000,&x);CHKERRQ(ierr);
ierr = PetscMalloc1(20000,&y);CHKERRQ(ierr);
ierr = PetscMalloc1(2000,&z);CHKERRQ(ierr);
ierr = PetscMalloc1(2000,&zi);CHKERRQ(ierr);
/* Take care of paging effects */
ierr = PetscTime(&t1);CHKERRQ(ierr);
/* Form the random set of integers */
for (i=0; i<2000; i++) {
ierr = PetscRandomGetValue(r,&value);CHKERRQ(ierr);
intval = (int)(value*20000.0);
z[i] = intval;
}
for (i=0; i<2000; i++) {
ierr = PetscRandomGetValue(r,&value);CHKERRQ(ierr);
intval = (int)(value*20000.0);
zi[i] = intval;
}
/* fprintf(stdout,"Done setup\n"); */
ierr = BlastCache();CHKERRQ(ierr);
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<2000; i++) x[i] = y[i];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[i] = y[i]",(t2-t1)/2000.0);
ierr = BlastCache();CHKERRQ(ierr);
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<500; i+=4) {
x[i] = y[z[i]];
x[1+i] = y[z[1+i]];
x[2+i] = y[z[2+i]];
x[3+i] = y[z[3+i]];
}
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[i] = y[idx[i]] - unroll 4",(t2-t1)/2000.0);
ierr = BlastCache();CHKERRQ(ierr);
ierr = PetscTime(&t1);CHKERRQ(ierr)
for (i=0; i<2000; i++) x[i] = y[z[i]];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[i] = y[idx[i]]",(t2-t1)/2000.0);
ierr = BlastCache();CHKERRQ(ierr);
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<1000; i+=2) { x[i] = y[z[i]]; x[1+i] = y[z[1+i]]; }
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[i] = y[idx[i]] - unroll 2",(t2-t1)/2000.0);
ierr = BlastCache();CHKERRQ(ierr);
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<2000; i++) x[z[i]] = y[i];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[z[i]] = y[i]",(t2-t1)/2000.0);
ierr = BlastCache();CHKERRQ(ierr);
ierr = PetscTime(&t1);CHKERRQ(ierr);
for (i=0; i<2000; i++) x[z[i]] = y[zi[i]];
ierr = PetscTime(&t2);CHKERRQ(ierr);
fprintf(stdout,"%-27s : %e sec\n","x[z[i]] = y[zi[i]]",(t2-t1)/2000.0);
ierr = PetscMemcpy(x,y,10);CHKERRQ(ierr);
ierr = PetscMemcpy(z,zi,10);CHKERRQ(ierr);
ierr = PetscFree(z);CHKERRQ(ierr);
ierr = PetscFree(zi);CHKERRQ(ierr);
ierr = PetscFree(x);CHKERRQ(ierr);
ierr = PetscFree(y);CHKERRQ(ierr);
ierr = PetscRandomDestroy(&r);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
PetscErrorCode PetscLogView_VecScatter(PetscViewer viewer)
{
MPI_Comm comm = PetscObjectComm((PetscObject) viewer);
PetscEventPerfInfo *eventInfo = NULL;
PetscLogDouble locTotalTime,stats[6],maxstats[6],minstats[6],sumstats[6],avetime,ksptime;
PetscStageLog stageLog;
const int stage = 2;
int event,events[] = {VEC_ScatterBegin,VEC_ScatterEnd};
PetscMPIInt rank,size;
PetscErrorCode ierr;
PetscInt i;
char arch[128],hostname[128],username[128],pname[PETSC_MAX_PATH_LEN],date[128],version[256];
PetscFunctionBegin;
PetscTime(&locTotalTime); locTotalTime -= petsc_BaseTime;
ierr = MPI_Comm_size(comm, &size);CHKERRQ(ierr);
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscLogGetStageLog(&stageLog);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"numProcs = %d\n",size);CHKERRQ(ierr);
ierr = PetscGetArchType(arch,sizeof(arch));CHKERRQ(ierr);
ierr = PetscGetHostName(hostname,sizeof(hostname));CHKERRQ(ierr);
ierr = PetscGetUserName(username,sizeof(username));CHKERRQ(ierr);
ierr = PetscGetProgramName(pname,sizeof(pname));CHKERRQ(ierr);
ierr = PetscGetDate(date,sizeof(date));CHKERRQ(ierr);
ierr = PetscGetVersion(version,sizeof(version));CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer,"%s on a %s named %s with %d processors, by %s %s\n", pname, arch, hostname, size, username, date);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "Using %s\n", version);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "Configure options: %s",petscconfigureoptions);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "%s", petscmachineinfo);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "%s", petsccompilerinfo);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "%s", petsccompilerflagsinfo);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "%s", petsclinkerinfo);CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer, "%s\n", PETSC_MPICC_SHOW);CHKERRQ(ierr);
ierr = PetscOptionsView(NULL,viewer);CHKERRQ(ierr);
#if defined(PETSC_HAVE_HWLOC)
ierr = PetscProcessPlacementView(viewer);CHKERRQ(ierr);
#endif
ierr = PetscViewerASCIIPrintf(viewer, "----------------------------------------------------\n");CHKERRQ(ierr);
ierr = PetscViewerASCIIPrintf(viewer," Time Min to Max Range Proportion of KSP\n");CHKERRQ(ierr);
eventInfo = stageLog->stageInfo[stage].eventLog->eventInfo;
ierr = MPI_Allreduce(&eventInfo[KSP_Solve].time,&ksptime,1,MPIU_PETSCLOGDOUBLE,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
ksptime = ksptime/size;
for (i=0; i<(int)(sizeof(events)/sizeof(int)); i++) {
event = events[i];
stats[COUNT] = eventInfo[event].count;
stats[TIME] = eventInfo[event].time;
stats[NUMMESS] = eventInfo[event].numMessages;
stats[MESSLEN] = eventInfo[event].messageLength;
stats[REDUCT] = eventInfo[event].numReductions;
stats[FLOPS] = eventInfo[event].flops;
ierr = MPI_Allreduce(stats,maxstats,6,MPIU_PETSCLOGDOUBLE,MPI_MAX,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = MPI_Allreduce(stats,minstats,6,MPIU_PETSCLOGDOUBLE,MPI_MIN,PETSC_COMM_WORLD);CHKERRQ(ierr);
ierr = MPI_Allreduce(stats,sumstats,6,MPIU_PETSCLOGDOUBLE,MPI_SUM,PETSC_COMM_WORLD);CHKERRQ(ierr);
avetime = sumstats[1]/size;
ierr = PetscViewerASCIIPrintf(viewer,"%s %4.2e -%5.1f %% %5.1f %% %4.2e %%\n",stageLog->eventLog->eventInfo[event].name,
avetime,100.*(avetime-minstats[1])/avetime,100.*(maxstats[1]-avetime)/avetime,100.*avetime/ksptime);CHKERRQ(ierr);
}
ierr = PetscViewerFlush(viewer);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
static double Second()
{
double t;
PetscTime(&t);
return t;
}
int main(int argc, char **argv)
{
MPI_Comm comm;
PetscMPIInt rank;
PetscErrorCode ierr;
User user;
PetscLogDouble v1, v2;
PetscInt nplot = 0;
char fileName[2048];
ierr = PetscInitialize(&argc, &argv, (char*) 0, help);CHKERRQ(ierr);
comm = PETSC_COMM_WORLD;
ierr = MPI_Comm_rank(comm, &rank);CHKERRQ(ierr);
ierr = PetscNew(&user);CHKERRQ(ierr);
ierr = PetscNew(&user->algebra);CHKERRQ(ierr);
ierr = PetscNew(&user->model);CHKERRQ(ierr);
ierr = PetscNew(&user->model->physics);CHKERRQ(ierr);
Algebra algebra = user->algebra;
ierr = LoadOptions(comm, user);CHKERRQ(ierr);
ierr = PetscTime(&v1);CHKERRQ(ierr);
ierr = CreateMesh(comm, user);CHKERRQ(ierr);
ierr = PetscTime(&v2);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,
"Read and Distribute mesh takes %f sec \n", v2 - v1);CHKERRQ(ierr);
ierr = SetUpLocalSpace(user);CHKERRQ(ierr); //Set up the dofs of each element
ierr = ConstructGeometryFVM(&user->facegeom, &user->cellgeom, user);CHKERRQ(ierr);
ierr = LimiterSetup(user);CHKERRQ(ierr);
if (user->TimeIntegralMethod == EXPLICITMETHOD) { // explicit method
if(user->myownexplicitmethod){// Using the fully explicit method based on my own routing
ierr = PetscPrintf(PETSC_COMM_WORLD,"Using the fully explicit method based on my own routing\n");CHKERRQ(ierr);
user->current_time = user->initial_time;
user->current_step = 1;
ierr = DMCreateGlobalVector(user->dm, &algebra->solution);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) algebra->solution, "solution");CHKERRQ(ierr);
ierr = VecSet(algebra->solution, 0);CHKERRQ(ierr);
ierr = SetInitialCondition(user->dm, algebra->solution, user);CHKERRQ(ierr);
if(1){
PetscViewer viewer;
ierr = OutputVTK(user->dm, "intialcondition.vtk", &viewer);CHKERRQ(ierr);
ierr = VecView(algebra->solution, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Outputing the initial condition intialcondition.vtk!!! \n");CHKERRQ(ierr);
}
ierr = VecDuplicate(algebra->solution, &algebra->fn);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &algebra->oldsolution);CHKERRQ(ierr);
if(user->Explicit_RK2||user->Explicit_RK4){
ierr = PetscPrintf(PETSC_COMM_WORLD,"Use the second order Runge Kutta method \n");CHKERRQ(ierr);
}else{
ierr = PetscPrintf(PETSC_COMM_WORLD,"Use the first order forward Euler method \n");CHKERRQ(ierr);
}
nplot = 0; //the plot step
while(user->current_time < (user->final_time - 0.05 * user->dt)){
user->current_time = user->current_time + user->dt;
ierr = FormTimeStepFunction(user, algebra, algebra->solution, algebra->fn);CHKERRQ(ierr);
if(0){
PetscViewer viewer;
ierr = OutputVTK(user->dm, "function.vtk", &viewer);CHKERRQ(ierr);
ierr = VecView(algebra->fn, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
if(user->Explicit_RK2){
/*
U^n_1 = U^n + 0.5*dt*f(U^n)
U^{n+1} = U^n + dt*f(U^n_1)
*/
ierr = VecCopy(algebra->solution, algebra->oldsolution);CHKERRQ(ierr);
//note that algebra->oldsolution and algebra->solution are both U^n
ierr = VecAXPY(algebra->solution, 0.5*user->dt, algebra->fn);CHKERRQ(ierr);
//U^n_1 = U^n + 0.5*dt*f(U^n), now algebra->solution is U^n_1, and algebra->fn is f(U^n)
ierr = FormTimeStepFunction(user, algebra, algebra->solution, algebra->fn);CHKERRQ(ierr);
//algebra->fn is f(U^n_1)
// reset the algebra->solution to U^n
ierr = VecCopy(algebra->oldsolution, algebra->solution);CHKERRQ(ierr);
ierr = VecAXPY(algebra->solution, user->dt, algebra->fn);CHKERRQ(ierr);
// now algebra->solution is U^{n+1} = U^n + dt*f(U^n_1)
}else if(user->Explicit_RK4){
/* refer to https://en.wikipedia.org/wiki/Runge%E2%80%93Kutta_methods
k_1 = f(U^n)
U^n_1 = U^n + 0.5*dt*k_1
k_2 = f(U^n_1)
U^n_2 = U^n + 0.5*dt*k_2
k_3 = f(U^n_2)
U^n_3 = U^n + 0.5*dt*k_3
k_4 = f(U^n_3)
U^{n+1} = U^n + dt/6*(k_1 + 2*k_2 + 2*k_3 + k_4)
*/
Vec VecTemp; // store the U^n_1
Vec k1, k2, k3, k4;
ierr = VecDuplicate(algebra->solution, &k1);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &k2);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &k3);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &k4);CHKERRQ(ierr);
ierr = VecCopy(algebra->solution, algebra->oldsolution);CHKERRQ(ierr);
ierr = VecCopy(algebra->fn, k1);CHKERRQ(ierr);
//note that algebra->oldsolution and algebra->solution are both U^n
ierr = VecAXPY(algebra->solution, 0.5*user->dt, k1);CHKERRQ(ierr);
//U^n_1 = U^n + 0.5*dt*k1, now algebra->solution is U^n_1, and algebra->fn is f(U^n)
ierr = FormTimeStepFunction(user, algebra, algebra->solution, algebra->fn);CHKERRQ(ierr);
//algebra->fn is f(U^n_1)
ierr = VecCopy(algebra->fn, k2);CHKERRQ(ierr);
// reset the algebra->solution to U^n
ierr = VecCopy(algebra->oldsolution, algebra->solution);CHKERRQ(ierr);
ierr = VecAXPY(algebra->solution, 0.5*user->dt, k2);CHKERRQ(ierr);
//U^n_2 = U^n + 0.5*dt*k2, now algebra->solution is U^n_2, and algebra->fn is f(U^n_1)
ierr = FormTimeStepFunction(user, algebra, algebra->solution, algebra->fn);CHKERRQ(ierr);
//algebra->fn is f(U^n_2)
ierr = VecCopy(algebra->fn, k3);CHKERRQ(ierr);
// reset the algebra->solution to U^n
ierr = VecCopy(algebra->oldsolution, algebra->solution);CHKERRQ(ierr);
ierr = VecAXPY(algebra->solution, 0.5*user->dt, k3);CHKERRQ(ierr);
//U^n_3 = U^n + 0.5*dt*k3, now algebra->solution is U^n_3, and algebra->fn is f(U^n_2)
ierr = FormTimeStepFunction(user, algebra, algebra->solution, algebra->fn);CHKERRQ(ierr);
//algebra->fn is f(U^n_3)
ierr = VecCopy(algebra->fn, k4);CHKERRQ(ierr);
//U^{n+1} = U^n + dt/6*(k_1 + 2*k_2 + 2*k_3 + k_4)
PetscReal temp;
temp = user->dt/6;
// reset the algebra->solution to U^n
ierr = VecCopy(algebra->oldsolution, algebra->solution);CHKERRQ(ierr);
ierr = VecAXPY(algebra->solution, temp, k1);CHKERRQ(ierr);
// now algebra->solution is U^n + dt/6*k_1
ierr = VecAXPY(algebra->solution, 2*temp, k2);CHKERRQ(ierr);
// now algebra->solution is U^n + dt/6*k_1 + 2*dt/6*k_2
ierr = VecAXPY(algebra->solution, 2*temp, k3);CHKERRQ(ierr);
// now algebra->solution is U^n + dt/6*k_1 + 2*dt/6*k_2 + 2*dt/6*k_3
ierr = VecAXPY(algebra->solution, temp, k4);CHKERRQ(ierr);
// now algebra->solution is U^n + dt/6*k_1 + 2*dt/6*k_2 + 2*dt/6*k_3 + dt/6*k_4
ierr = VecDestroy(&k1);CHKERRQ(ierr);
ierr = VecDestroy(&k2);CHKERRQ(ierr);
ierr = VecDestroy(&k3);CHKERRQ(ierr);
ierr = VecDestroy(&k4);CHKERRQ(ierr);
}else{
ierr = VecCopy(algebra->solution, algebra->oldsolution);CHKERRQ(ierr);
ierr = VecAXPY(algebra->solution, user->dt, algebra->fn);CHKERRQ(ierr);
}
{// Monitor the solution and function norms
PetscReal norm;
PetscLogDouble space =0;
PetscInt size;
PetscReal fnnorm;
ierr = VecNorm(algebra->fn,NORM_2,&fnnorm);CHKERRQ(ierr);
//ierr = VecView(algebra->fn, PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = VecNorm(algebra->solution,NORM_2,&norm);CHKERRQ(ierr);
ierr = VecGetSize(algebra->solution, &size);CHKERRQ(ierr);
norm = norm/size;
fnnorm = fnnorm/size;
if (norm>1.e5) {
SETERRQ2(PETSC_COMM_WORLD, PETSC_ERR_LIB,
"The norm of the solution is: %f (current time: %f). The explicit method is going to DIVERGE!!!", norm, user->current_time);
}
if (user->current_step%10==0) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Step %D at time %g with solution norm = %g and founction norm = %g \n",
user->current_step, user->current_time, norm, fnnorm);CHKERRQ(ierr);
}
// ierr = PetscMallocGetCurrentUsage(&space);CHKERRQ(ierr);
// if (user->current_step%10==0) {
// ierr = PetscPrintf(PETSC_COMM_WORLD,"Current space PetscMalloc()ed %g M\n",
// space/(1024*1024));CHKERRQ(ierr);
// }
}
{ // Monitor the difference of two steps' solution
PetscReal norm;
ierr = VecAXPY(algebra->oldsolution, -1, algebra->solution);CHKERRQ(ierr);
ierr = VecNorm(algebra->oldsolution,NORM_2,&norm);CHKERRQ(ierr);
if (user->current_step%10==0) {
ierr = PetscPrintf(PETSC_COMM_WORLD,"Step %D at time %g with ||u_k-u_{k-1}|| = %g \n",
user->current_step, user->current_time, norm);CHKERRQ(ierr);
}
if((norm<1.e-6)||(user->current_step > user->max_time_its)){
if(norm<1.e-6) ierr = PetscPrintf(PETSC_COMM_WORLD,"\n Convergence with ||u_k-u_{k-1}|| = %g < 1.e-6\n\n", norm);CHKERRQ(ierr);
if(user->current_step > user->max_time_its) ierr = PetscPrintf(PETSC_COMM_WORLD,"\n Convergence with reaching the max time its\n\n");CHKERRQ(ierr);
break;
}
}
// output the solution
if (user->output_solution && (user->current_step%user->steps_output==0)){
PetscViewer viewer;
Vec solution_unscaled; // Note the the algebra->solution is scaled by the density, so this is for the unscaled solution
nplot = user->current_step/user->steps_output;
// update file name for the current time step
ierr = VecDuplicate(algebra->solution, &solution_unscaled);CHKERRQ(ierr);
ierr = ReformatSolution(algebra->solution, solution_unscaled, user);CHKERRQ(ierr);
ierr = PetscSNPrintf(fileName, sizeof(fileName),"%s_%d.vtk",user->solutionfile, nplot);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Outputing solution %s (current time %f)\n", fileName, user->current_time);CHKERRQ(ierr);
ierr = OutputVTK(user->dm, fileName, &viewer);CHKERRQ(ierr);
ierr = VecView(solution_unscaled, viewer);CHKERRQ(ierr);
ierr = VecDestroy(&solution_unscaled);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
user->current_step++;
}
ierr = VecDestroy(&algebra->fn);CHKERRQ(ierr);
}else{ // Using the fully explicit method based on the PETSC TS routing
PetscReal ftime;
TS ts;
TSConvergedReason reason;
PetscInt nsteps;
//PetscReal minRadius;
//ierr = DMPlexTSGetGeometry(user->dm, NULL, NULL, &minRadius);CHKERRQ(ierr);
//user->dt = 0.9*4 * minRadius / 1.0;
ierr = PetscPrintf(PETSC_COMM_WORLD,"Using the fully explicit method based on the PETSC TS routing\n");CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user->dm, &algebra->solution);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) algebra->solution, "solution");CHKERRQ(ierr);
ierr = VecSet(algebra->solution, 0.0);CHKERRQ(ierr);
ierr = SetInitialCondition(user->dm, algebra->solution, user);CHKERRQ(ierr);
ierr = TSCreate(comm, &ts);CHKERRQ(ierr);
ierr = TSSetType(ts, TSEULER);CHKERRQ(ierr);
ierr = TSSetDM(ts, user->dm);CHKERRQ(ierr);
ierr = TSMonitorSet(ts,TSMonitorFunctionError,(void*)user,NULL);CHKERRQ(ierr);
ierr = TSSetRHSFunction(ts, NULL, MyRHSFunction, user);CHKERRQ(ierr);
ierr = TSSetDuration(ts, 1000, user->final_time);CHKERRQ(ierr);
ierr = TSSetInitialTimeStep(ts, user->initial_time, user->dt);CHKERRQ(ierr);
ierr = TSSetFromOptions(ts);CHKERRQ(ierr);
ierr = TSSolve(ts, algebra->solution);CHKERRQ(ierr);
ierr = TSGetSolveTime(ts, &ftime);CHKERRQ(ierr);
ierr = TSGetTimeStepNumber(ts, &nsteps);CHKERRQ(ierr);
ierr = TSGetConvergedReason(ts, &reason);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"%s at time %g after %D steps\n",TSConvergedReasons[reason],ftime,nsteps);CHKERRQ(ierr);
ierr = TSDestroy(&ts);CHKERRQ(ierr);
}
if(user->benchmark_couette) {
ierr = DMCreateGlobalVector(user->dm, &algebra->exactsolution);CHKERRQ(ierr);
ierr = ComputeExactSolution(user->dm, user->current_time, algebra->exactsolution, user);CHKERRQ(ierr);
}
if(user->benchmark_couette) {
PetscViewer viewer;
PetscReal norm;
ierr = OutputVTK(user->dm, "exact_solution.vtk", &viewer);CHKERRQ(ierr);
ierr = VecView(algebra->exactsolution, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecAXPY(algebra->exactsolution, -1, algebra->solution);CHKERRQ(ierr);
ierr = VecNorm(algebra->exactsolution,NORM_INFINITY,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Final time at %f, Error: ||u_k-u|| = %g \n", user->current_time, norm);CHKERRQ(ierr);
ierr = OutputVTK(user->dm, "Error.vtk", &viewer);CHKERRQ(ierr);
ierr = VecView(algebra->exactsolution, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = VecDestroy(&algebra->solution);CHKERRQ(ierr);
if(user->myownexplicitmethod){ierr = VecDestroy(&algebra->oldsolution);CHKERRQ(ierr);}
ierr = VecDestroy(&algebra->exactsolution);CHKERRQ(ierr);
ierr = DMDestroy(&user->dm);CHKERRQ(ierr);
} else if (user->TimeIntegralMethod == IMPLICITMETHOD) { // Using the fully implicit method
ierr = PetscPrintf(PETSC_COMM_WORLD,"Using the fully implicit method\n");CHKERRQ(ierr);
ierr = SNESCreate(comm,&user->snes);CHKERRQ(ierr);
ierr = SNESSetDM(user->snes,user->dm);CHKERRQ(ierr);
ierr = DMCreateGlobalVector(user->dm, &algebra->solution);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &algebra->oldsolution);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &algebra->f);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &algebra->fn);CHKERRQ(ierr);
ierr = VecDuplicate(algebra->solution, &algebra->oldfn);CHKERRQ(ierr);
ierr = PetscObjectSetName((PetscObject) algebra->solution, "solution");CHKERRQ(ierr);
ierr = SetInitialCondition(user->dm, algebra->solution, user);CHKERRQ(ierr);
ierr = DMSetMatType(user->dm, MATAIJ);CHKERRQ(ierr);
// ierr = DMCreateMatrix(user->dm, &algebra->A);CHKERRQ(ierr);
ierr = DMCreateMatrix(user->dm, &algebra->J);CHKERRQ(ierr);
if (user->JdiffP) {
/*Set up the preconditioner matrix*/
ierr = DMCreateMatrix(user->dm, &algebra->P);CHKERRQ(ierr);
}else{
algebra->P = algebra->J;
}
ierr = MatSetOption(algebra->J, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE);CHKERRQ(ierr);
/*set nonlinear function */
ierr = SNESSetFunction(user->snes, algebra->f, FormFunction, (void*)user);CHKERRQ(ierr);
/* compute Jacobian */
ierr = SNESSetJacobian(user->snes, algebra->J, algebra->P, FormJacobian, (void*)user);CHKERRQ(ierr);
ierr = SNESSetFromOptions(user->snes);CHKERRQ(ierr);
/* do the solve */
if (user->timestep == TIMESTEP_STEADY_STATE) {
ierr = SolveSteadyState(user);CHKERRQ(ierr);
} else {
ierr = SolveTimeDependent(user);CHKERRQ(ierr);
}
if (user->output_solution){
PetscViewer viewer;
Vec solution_unscaled; // Note the the algebra->solution is scaled by the density, so this is for the unscaled solution
ierr = VecDuplicate(algebra->solution, &solution_unscaled);CHKERRQ(ierr);
ierr = ReformatSolution(algebra->solution, solution_unscaled, user);CHKERRQ(ierr);
ierr = OutputVTK(user->dm, "solution.vtk", &viewer);CHKERRQ(ierr);
ierr = VecView(solution_unscaled, viewer);CHKERRQ(ierr);
ierr = VecDestroy(&solution_unscaled);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
if(user->benchmark_couette) {
PetscViewer viewer;
PetscReal norm;
ierr = OutputVTK(user->dm, "exact_solution.vtk", &viewer);CHKERRQ(ierr);
ierr = VecView(algebra->exactsolution, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
ierr = VecAXPY(algebra->exactsolution, -1, algebra->solution);CHKERRQ(ierr);
ierr = VecNorm(algebra->exactsolution,NORM_INFINITY,&norm);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Error: ||u_k-u|| = %g \n", norm);CHKERRQ(ierr);
ierr = OutputVTK(user->dm, "Error.vtk", &viewer);CHKERRQ(ierr);
ierr = VecView(algebra->exactsolution, viewer);CHKERRQ(ierr);
ierr = PetscViewerDestroy(&viewer);CHKERRQ(ierr);
}
ierr = VecDestroy(&algebra->solution);CHKERRQ(ierr);
ierr = VecDestroy(&algebra->f);CHKERRQ(ierr);
ierr = VecDestroy(&algebra->oldsolution);CHKERRQ(ierr);
ierr = VecDestroy(&algebra->fn);CHKERRQ(ierr);
ierr = VecDestroy(&algebra->oldfn);CHKERRQ(ierr);
ierr = SNESDestroy(&user->snes);CHKERRQ(ierr);
ierr = DMDestroy(&user->dm);CHKERRQ(ierr);
} else {
SETERRQ(PETSC_COMM_SELF,PETSC_ERR_USER,"WRONG option for the time integral method. Using the option '-time_integral_method 0 or 1'");
}
ierr = VecDestroy(&user->cellgeom);CHKERRQ(ierr);
ierr = VecDestroy(&user->facegeom);CHKERRQ(ierr);
ierr = DMDestroy(&user->dmGrad);CHKERRQ(ierr);
ierr = PetscFunctionListDestroy(&LimitList);CHKERRQ(ierr);
ierr = PetscFree(user->model->physics);CHKERRQ(ierr);
ierr = PetscFree(user->algebra);CHKERRQ(ierr);
ierr = PetscFree(user->model);CHKERRQ(ierr);
ierr = PetscFree(user);CHKERRQ(ierr);
{
PetscLogDouble space =0;
ierr = PetscMallocGetCurrentUsage(&space);CHKERRQ(ierr);
ierr = PetscPrintf(PETSC_COMM_WORLD,"Unfreed space at the End %g M\n", space/(1024*1024));CHKERRQ(ierr);
}
ierr = PetscFinalize();
return(0);
}
int main(int argc,char **argv)
{
Mat A[NMAT]; /* problem matrices */
PEP pep; /* polynomial eigenproblem solver context */
PetscInt n=128,nlocal,k,Istart,Iend,i,j,start_ct,end_ct;
PetscReal w=9.92918,a=0.0,b=2.0,h,deltasq;
PetscReal nref[NL],K2[NL],q[NL],*md,*supd,*subd;
PetscScalar v,alpha;
PetscBool terse;
PetscErrorCode ierr;
PetscLogDouble time1,time2;
SlepcInitialize(&argc,&argv,(char*)0,help);
ierr = PetscOptionsGetInt(NULL,"-n",&n,NULL);CHKERRQ(ierr);
n = (n/4)*4;
ierr = PetscPrintf(PETSC_COMM_WORLD,"\nPlanar waveguide, n=%D\n\n",n+1);CHKERRQ(ierr);
h = (b-a)/n;
nlocal = (n/4)-1;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Set waveguide parameters used in construction of matrices
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* refractive indices in each layer */
nref[0] = 1.5;
nref[1] = 1.66;
nref[2] = 1.6;
nref[3] = 1.53;
nref[4] = 1.66;
nref[5] = 1.0;
for (i=0;i<NL;i++) K2[i] = w*w*nref[i]*nref[i];
deltasq = K2[0] - K2[NL-1];
for (i=0;i<NL;i++) q[i] = K2[i] - (K2[0] + K2[NL-1])/2;
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Compute the polynomial matrices
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* initialize matrices */
for (i=0;i<NMAT;i++) {
ierr = MatCreate(PETSC_COMM_WORLD,&A[i]);CHKERRQ(ierr);
ierr = MatSetSizes(A[i],PETSC_DECIDE,PETSC_DECIDE,n+1,n+1);CHKERRQ(ierr);
ierr = MatSetFromOptions(A[i]);CHKERRQ(ierr);
ierr = MatSetUp(A[i]);CHKERRQ(ierr);
}
ierr = MatGetOwnershipRange(A[0],&Istart,&Iend);CHKERRQ(ierr);
/* A0 */
alpha = (h/6)*(deltasq*deltasq/16);
for (i=Istart;i<Iend;i++) {
v = 4.0;
if (i==0 || i==n) v = 2.0;
ierr = MatSetValue(A[0],i,i,v*alpha,INSERT_VALUES);CHKERRQ(ierr);
if (i>0) { ierr = MatSetValue(A[0],i,i-1,alpha,INSERT_VALUES);CHKERRQ(ierr); }
if (i<n) { ierr = MatSetValue(A[0],i,i+1,alpha,INSERT_VALUES);CHKERRQ(ierr); }
}
/* A1 */
if (Istart==0) { ierr = MatSetValue(A[1],0,0,-deltasq/4,INSERT_VALUES);CHKERRQ(ierr); }
if (Iend==n+1) { ierr = MatSetValue(A[1],n,n,deltasq/4,INSERT_VALUES);CHKERRQ(ierr); }
/* A2 */
alpha = 1.0/h;
for (i=Istart;i<Iend;i++) {
v = 2.0;
if (i==0 || i==n) v = 1.0;
ierr = MatSetValue(A[2],i,i,v*alpha,ADD_VALUES);CHKERRQ(ierr);
if (i>0) { ierr = MatSetValue(A[2],i,i-1,-alpha,ADD_VALUES);CHKERRQ(ierr); }
if (i<n) { ierr = MatSetValue(A[2],i,i+1,-alpha,ADD_VALUES);CHKERRQ(ierr); }
}
ierr = PetscMalloc3(n+1,&md,n+1,&supd,n+1,&subd);CHKERRQ(ierr);
md[0] = 2.0*q[1];
supd[1] = q[1];
subd[0] = q[1];
for (k=1;k<=NL-2;k++) {
end_ct = k*(nlocal+1);
start_ct = end_ct-nlocal;
for (j=start_ct;j<end_ct;j++) {
md[j] = 4*q[k];
supd[j+1] = q[k];
subd[j] = q[k];
}
if (k < 4) { /* interface points */
md[end_ct] = 4*(q[k] + q[k+1])/2.0;
supd[end_ct+1] = q[k+1];
subd[end_ct] = q[k+1];
}
}
md[n] = 2*q[NL-2];
supd[n] = q[NL-2];
subd[n] = q[NL-2];
alpha = -h/6.0;
for (i=Istart;i<Iend;i++) {
ierr = MatSetValue(A[2],i,i,md[i]*alpha,ADD_VALUES);CHKERRQ(ierr);
if (i>0) { ierr = MatSetValue(A[2],i,i-1,subd[i-1]*alpha,ADD_VALUES);CHKERRQ(ierr); }
if (i<n) { ierr = MatSetValue(A[2],i,i+1,supd[i+1]*alpha,ADD_VALUES);CHKERRQ(ierr); }
}
ierr = PetscFree3(md,supd,subd);CHKERRQ(ierr);
/* A3 */
if (Istart==0) { ierr = MatSetValue(A[3],0,0,1.0,INSERT_VALUES);CHKERRQ(ierr); }
if (Iend==n+1) { ierr = MatSetValue(A[3],n,n,1.0,INSERT_VALUES);CHKERRQ(ierr); }
/* A4 */
alpha = (h/6);
for (i=Istart;i<Iend;i++) {
v = 4.0;
if (i==0 || i==n) v = 2.0;
ierr = MatSetValue(A[4],i,i,v*alpha,INSERT_VALUES);CHKERRQ(ierr);
if (i>0) { ierr = MatSetValue(A[4],i,i-1,alpha,INSERT_VALUES);CHKERRQ(ierr); }
if (i<n) { ierr = MatSetValue(A[4],i,i+1,alpha,INSERT_VALUES);CHKERRQ(ierr); }
}
/* assemble matrices */
for (i=0;i<NMAT;i++) {
ierr = MatAssemblyBegin(A[i],MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
for (i=0;i<NMAT;i++) {
ierr = MatAssemblyEnd(A[i],MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
}
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Create the eigensolver and solve the problem
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
ierr = PEPCreate(PETSC_COMM_WORLD,&pep);CHKERRQ(ierr);
ierr = PEPSetOperators(pep,NMAT,A);CHKERRQ(ierr);
ierr = PEPSetFromOptions(pep);CHKERRQ(ierr);
ierr = PetscTime(&time1); CHKERRQ(ierr);
ierr = PEPSolve(pep);CHKERRQ(ierr);
ierr = PetscTime(&time2); CHKERRQ(ierr);
/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Display solution and clean up
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
/* show detailed info unless -terse option is given by user */
ierr = PetscOptionsHasName(NULL,"-terse",&terse);CHKERRQ(ierr);
if (terse) {
ierr = PEPErrorView(pep,PEP_ERROR_BACKWARD,NULL);CHKERRQ(ierr);
} else {
ierr = PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD,PETSC_VIEWER_ASCII_INFO_DETAIL);CHKERRQ(ierr);
ierr = PEPReasonView(pep,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PEPErrorView(pep,PEP_ERROR_BACKWARD,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
ierr = PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
}
ierr = PetscPrintf(PETSC_COMM_WORLD,"Time: %g\n\n\n",time2-time1);CHKERRQ(ierr);
ierr = PEPDestroy(&pep);CHKERRQ(ierr);
for (i=0;i<NMAT;i++) {
ierr = MatDestroy(&A[i]);CHKERRQ(ierr);
}
ierr = SlepcFinalize();CHKERRQ(ierr);
return 0;
}