void TestInterleavedVecScatter()
{
// Vectors will be twice PROBLEM_SIZE, since this is to be used in bidomain code.
const unsigned PROBLEM_SIZE = 10;
DistributedVectorFactory factory(PROBLEM_SIZE);
// Source vector = [-1 1 -2 2 ... -PROBLEM_SIZE/2+1 PROBLEM_SIZE/2+1]
Vec interleaved_vec=factory.CreateVec(2);
DistributedVector interleaved_dist_vec = factory.CreateDistributedVector(interleaved_vec);
DistributedVector::Stripe first_variable(interleaved_dist_vec, 0);
DistributedVector::Stripe second_variable(interleaved_dist_vec, 1);
for (DistributedVector::Iterator index = interleaved_dist_vec.Begin();
index!= interleaved_dist_vec.End();
++index)
{
first_variable[index] = -1.0 * (index.Global+1);
second_variable[index] = (index.Global+1);
}
// Destination vectors. It is important they have a compatible parallel layout with the source vector, hence the 2nd param of CreateVec()
PetscInt local_num_rows;
VecGetLocalSize(interleaved_vec, &local_num_rows);
Vec first_variable_vec = PetscTools::CreateVec(PROBLEM_SIZE, local_num_rows/2);
Vec second_variable_vec = PetscTools::CreateVec(PROBLEM_SIZE, local_num_rows/2);
// Setup and perform scatter operation
VecScatter first_variable_context;
VecScatter second_variable_context;
PetscVecTools::SetupInterleavedVectorScatterGather(interleaved_vec, first_variable_context, second_variable_context);
#if (PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR >= 6) //PETSc 3.6 or later
// When used in the preconditioner code within a KSP solve, the main (bidomain) vector will be locked.
// Lock the vector so that modern PETSc (3.6) won't want to change it
VecLockPush(interleaved_vec);
#endif
PetscVecTools::DoInterleavedVecScatter(interleaved_vec, first_variable_context, first_variable_vec, second_variable_context, second_variable_vec);
#if (PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR >= 6) //PETSc 3.6 or later
// Unlock the vector (for symmetry)
VecLockPop(interleaved_vec);
#endif
// Check destination vectors are [-1 -2 -3 ...] and [1 2 3 ...] respectively.
DistributedVector dist_1st_var_vec = factory.CreateDistributedVector(first_variable_vec);
DistributedVector dist_2nd_var_vec = factory.CreateDistributedVector(second_variable_vec);
for (DistributedVector::Iterator index = dist_1st_var_vec.Begin();
index!= dist_1st_var_vec.End();
++index)
{
TS_ASSERT_EQUALS(dist_1st_var_vec[index], -1.0 * (index.Global+1));
TS_ASSERT_EQUALS(dist_2nd_var_vec[index], index.Global+1);
}
PetscTools::Destroy(interleaved_vec);
PetscTools::Destroy(first_variable_vec);
PetscTools::Destroy(second_variable_vec);
VecScatterDestroy(PETSC_DESTROY_PARAM(first_variable_context));
VecScatterDestroy(PETSC_DESTROY_PARAM(second_variable_context));
}
/*
Helper rutine to handle user postenvents and recording
*/
static PetscErrorCode TSPostEvent(TS ts,PetscReal t,Vec U)
{
PetscErrorCode ierr;
TSEvent event = ts->event;
PetscBool terminate = PETSC_FALSE;
PetscBool restart = PETSC_FALSE;
PetscInt i,ctr,stepnum;
PetscBool inflag[2],outflag[2];
PetscBool forwardsolve = PETSC_TRUE; /* Flag indicating that TS is doing a forward solve */
PetscFunctionBegin;
if (event->postevent) {
PetscObjectState state_prev,state_post;
ierr = PetscObjectStateGet((PetscObject)U,&state_prev);CHKERRQ(ierr);
ierr = (*event->postevent)(ts,event->nevents_zero,event->events_zero,t,U,forwardsolve,event->ctx);CHKERRQ(ierr);
ierr = PetscObjectStateGet((PetscObject)U,&state_post);CHKERRQ(ierr);
if (state_prev != state_post) restart = PETSC_TRUE;
}
/* Handle termination events and step restart */
for (i=0; i<event->nevents_zero; i++) if (event->terminate[event->events_zero[i]]) terminate = PETSC_TRUE;
inflag[0] = restart; inflag[1] = terminate;
ierr = MPIU_Allreduce(inflag,outflag,2,MPIU_BOOL,MPI_LOR,((PetscObject)ts)->comm);CHKERRQ(ierr);
restart = outflag[0]; terminate = outflag[1];
if (restart) {ierr = TSRestartStep(ts);CHKERRQ(ierr);}
if (terminate) {ierr = TSSetConvergedReason(ts,TS_CONVERGED_EVENT);CHKERRQ(ierr);}
event->status = terminate ? TSEVENT_NONE : TSEVENT_RESET_NEXTSTEP;
/* Reset event residual functions as states might get changed by the postevent callback */
if (event->postevent) {
ierr = VecLockPush(U);CHKERRQ(ierr);
ierr = (*event->eventhandler)(ts,t,U,event->fvalue,event->ctx);CHKERRQ(ierr);
ierr = VecLockPop(U);CHKERRQ(ierr);
}
/* Cache current time and event residual functions */
event->ptime_prev = t;
for (i=0; i<event->nevents; i++)
event->fvalue_prev[i] = event->fvalue[i];
/* Record the event in the event recorder */
ierr = TSGetStepNumber(ts,&stepnum);CHKERRQ(ierr);
ctr = event->recorder.ctr;
if (ctr == event->recsize) {
ierr = TSEventRecorderResize(event);CHKERRQ(ierr);
}
event->recorder.time[ctr] = t;
event->recorder.stepnum[ctr] = stepnum;
event->recorder.nevents[ctr] = event->nevents_zero;
for (i=0; i<event->nevents_zero; i++) event->recorder.eventidx[ctr][i] = event->events_zero[i];
event->recorder.ctr++;
PetscFunctionReturn(0);
}
/*@
MatNullSpaceDestroy - Destroys a data structure used to project vectors
out of null spaces.
Collective on MatNullSpace
Input Parameter:
. sp - the null space context to be destroyed
Level: advanced
.keywords: PC, null space, destroy
.seealso: MatNullSpaceCreate(), MatNullSpaceRemove(), MatNullSpaceSetFunction()
@*/
PetscErrorCode MatNullSpaceDestroy(MatNullSpace *sp)
{
PetscErrorCode ierr;
PetscInt i;
PetscFunctionBegin;
if (!*sp) PetscFunctionReturn(0);
PetscValidHeaderSpecific((*sp),MAT_NULLSPACE_CLASSID,1);
if (--((PetscObject)(*sp))->refct > 0) {*sp = 0; PetscFunctionReturn(0);}
for (i=0; i < (*sp)->n; i++) {
ierr = VecLockPop((*sp)->vecs[i]);CHKERRQ(ierr);
}
ierr = VecDestroyVecs((*sp)->n,&(*sp)->vecs);CHKERRQ(ierr);
ierr = PetscFree((*sp)->alpha);CHKERRQ(ierr);
ierr = PetscHeaderDestroy(sp);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
void TestReadOnlyDistributedVector()
{
DistributedVectorFactory factory(10);
Vec petsc_vec = factory.CreateVec();
{
DistributedVector distributed_vector = factory.CreateDistributedVector(petsc_vec);
for (DistributedVector::Iterator index = distributed_vector.Begin();
index!= distributed_vector.End();
++index)
{
distributed_vector[index] = (double) PetscTools::GetMyRank();
}
}
#if (PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR >= 6) //PETSc 3.6 or later
// Lock the vector so that modern PETSc (3.6) won't want to change it
EXCEPT_IF_NOT(VecLockPush(petsc_vec) == 0);
#endif
{
DistributedVector distributed_vector_read = factory.CreateDistributedVector(petsc_vec, true);
for (DistributedVector::Iterator index = distributed_vector_read.Begin();
index!= distributed_vector_read.End();
++index)
{
TS_ASSERT_EQUALS(distributed_vector_read[index], (double) PetscTools::GetMyRank());
distributed_vector_read[index] = 2.0;
}
}
#if (PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR >= 6) //PETSc 3.6 or later
// Take the lock back
EXCEPT_IF_NOT(VecLockPop(petsc_vec) == 0);
#endif
PetscTools::Destroy(petsc_vec);
}
PetscErrorCode TSEventHandler(TS ts)
{
PetscErrorCode ierr;
TSEvent event;
PetscReal t;
Vec U;
PetscInt i;
PetscReal dt,dt_min;
PetscInt rollback=0,in[2],out[2];
PetscInt fvalue_sign,fvalueprev_sign;
PetscFunctionBegin;
PetscValidHeaderSpecific(ts,TS_CLASSID,1);
if (!ts->event) PetscFunctionReturn(0);
event = ts->event;
ierr = TSGetTime(ts,&t);CHKERRQ(ierr);
ierr = TSGetTimeStep(ts,&dt);CHKERRQ(ierr);
ierr = TSGetSolution(ts,&U);CHKERRQ(ierr);
if (event->status == TSEVENT_NONE) {
if (ts->steps == 1) /* After first successful step */
event->timestep_orig = ts->ptime - ts->ptime_prev;
event->timestep_prev = dt;
}
if (event->status == TSEVENT_RESET_NEXTSTEP) {
/* Restore time step */
dt = PetscMin(event->timestep_orig,event->timestep_prev);
ierr = TSSetTimeStep(ts,dt);CHKERRQ(ierr);
event->status = TSEVENT_NONE;
}
if (event->status == TSEVENT_NONE) {
event->ptime_end = t;
}
ierr = VecLockPush(U);CHKERRQ(ierr);
ierr = (*event->eventhandler)(ts,t,U,event->fvalue,event->ctx);CHKERRQ(ierr);
ierr = VecLockPop(U);CHKERRQ(ierr);
for (i=0; i < event->nevents; i++) {
if (PetscAbsScalar(event->fvalue[i]) < event->vtol[i]) {
event->status = TSEVENT_ZERO;
event->fvalue_right[i] = event->fvalue[i];
continue;
}
fvalue_sign = PetscSign(PetscRealPart(event->fvalue[i]));
fvalueprev_sign = PetscSign(PetscRealPart(event->fvalue_prev[i]));
if (fvalueprev_sign != 0 && (fvalue_sign != fvalueprev_sign) && (PetscAbsScalar(event->fvalue_prev[i]) > event->vtol[i])) {
switch (event->direction[i]) {
case -1:
if (fvalue_sign < 0) {
rollback = 1;
/* Compute new time step */
dt = TSEventComputeStepSize(event->ptime_prev,t,event->ptime_right,event->fvalue_prev[i],event->fvalue[i],event->fvalue_right[i],event->side[i],dt);
if (event->monitor) {
ierr = PetscViewerASCIIPrintf(event->monitor,"TSEvent: iter %D - Event %D interval detected [%g - %g]\n",event->iterctr,i,(double)event->ptime_prev,(double)t);CHKERRQ(ierr);
}
event->fvalue_right[i] = event->fvalue[i];
event->side[i] = 1;
if (!event->iterctr) event->zerocrossing[i] = PETSC_TRUE;
event->status = TSEVENT_LOCATED_INTERVAL;
}
break;
case 1:
if (fvalue_sign > 0) {
rollback = 1;
/* Compute new time step */
dt = TSEventComputeStepSize(event->ptime_prev,t,event->ptime_right,event->fvalue_prev[i],event->fvalue[i],event->fvalue_right[i],event->side[i],dt);
if (event->monitor) {
ierr = PetscViewerASCIIPrintf(event->monitor,"TSEvent: iter %D - Event %D interval detected [%g - %g]\n",event->iterctr,i,(double)event->ptime_prev,(double)t);CHKERRQ(ierr);
}
event->fvalue_right[i] = event->fvalue[i];
event->side[i] = 1;
if (!event->iterctr) event->zerocrossing[i] = PETSC_TRUE;
event->status = TSEVENT_LOCATED_INTERVAL;
}
break;
case 0:
rollback = 1;
/* Compute new time step */
dt = TSEventComputeStepSize(event->ptime_prev,t,event->ptime_right,event->fvalue_prev[i],event->fvalue[i],event->fvalue_right[i],event->side[i],dt);
if (event->monitor) {
ierr = PetscViewerASCIIPrintf(event->monitor,"TSEvent: iter %D - Event %D interval detected [%g - %g]\n",event->iterctr,i,(double)event->ptime_prev,(double)t);CHKERRQ(ierr);
}
event->fvalue_right[i] = event->fvalue[i];
event->side[i] = 1;
if (!event->iterctr) event->zerocrossing[i] = PETSC_TRUE;
event->status = TSEVENT_LOCATED_INTERVAL;
break;
}
}
}
in[0] = event->status; in[1] = rollback;
ierr = MPIU_Allreduce(in,out,2,MPIU_INT,MPI_MAX,PetscObjectComm((PetscObject)ts));CHKERRQ(ierr);
event->status = (TSEventStatus)out[0]; rollback = out[1];
if (rollback) event->status = TSEVENT_LOCATED_INTERVAL;
event->nevents_zero = 0;
if (event->status == TSEVENT_ZERO) {
for (i=0; i < event->nevents; i++) {
if (PetscAbsScalar(event->fvalue[i]) < event->vtol[i]) {
event->events_zero[event->nevents_zero++] = i;
if (event->monitor) {
ierr = PetscViewerASCIIPrintf(event->monitor,"TSEvent: Event %D zero crossing at time %g located in %D iterations\n",i,(double)t,event->iterctr);CHKERRQ(ierr);
}
event->zerocrossing[i] = PETSC_FALSE;
}
event->side[i] = 0;
}
ierr = TSPostEvent(ts,t,U);CHKERRQ(ierr);
dt = event->ptime_end - t;
if (PetscAbsReal(dt) < PETSC_SMALL) { /* we hit the event, continue with the candidate time step */
dt = event->timestep_prev;
event->status = TSEVENT_NONE;
}
ierr = TSSetTimeStep(ts,dt);CHKERRQ(ierr);
event->iterctr = 0;
PetscFunctionReturn(0);
}
if (event->status == TSEVENT_LOCATED_INTERVAL) {
ierr = TSRollBack(ts);CHKERRQ(ierr);
ierr = TSSetConvergedReason(ts,TS_CONVERGED_ITERATING);CHKERRQ(ierr);
event->status = TSEVENT_PROCESSING;
event->ptime_right = t;
} else {
for (i=0; i < event->nevents; i++) {
if (event->status == TSEVENT_PROCESSING && event->zerocrossing[i]) {
/* Compute new time step */
dt = TSEventComputeStepSize(event->ptime_prev,t,event->ptime_right,event->fvalue_prev[i],event->fvalue[i],event->fvalue_right[i],event->side[i],dt);
event->side[i] = -1;
}
event->fvalue_prev[i] = event->fvalue[i];
}
if (event->monitor && event->status == TSEVENT_PROCESSING) {
ierr = PetscViewerASCIIPrintf(event->monitor,"TSEvent: iter %D - Stepping forward as no event detected in interval [%g - %g]\n",event->iterctr,(double)event->ptime_prev,(double)t);CHKERRQ(ierr);
}
event->ptime_prev = t;
}
if (event->status == TSEVENT_PROCESSING) event->iterctr++;
ierr = MPIU_Allreduce(&dt,&dt_min,1,MPIU_REAL,MPIU_MIN,PetscObjectComm((PetscObject)ts));CHKERRQ(ierr);
ierr = TSSetTimeStep(ts,dt_min);CHKERRQ(ierr);
PetscFunctionReturn(0);
}
