void INSStaggeredCenteredConvectiveOperator::deallocateOperatorState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_operator_state);

    // Deallocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        if (level->checkAllocated(d_U_scratch_idx))
        {
            level->deallocatePatchData(d_U_scratch_idx);
        }
    }

    // Deallocate the refine algorithm, operator, patch strategy, and schedules.
    d_hier_bdry_fill.setNull();
    d_bc_helper.setNull();

    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_operator_state);
    return;
} // deallocateOperatorState
void StaggeredStokesOperator::deallocateOperatorState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_operator_state);

    // Deallocate hierarchy math operations object.
    if (!d_hier_math_ops_external) d_hier_math_ops.setNull();

    // Deallocate the interpolation operators.
    d_hier_bdry_fill->deallocateOperatorState();
    d_hier_bdry_fill.setNull();
    d_transaction_comps.clear();
    d_U_fill_pattern.setNull();
    d_P_fill_pattern.setNull();

    // Delete the solution and rhs vectors.
    d_x->resetLevels(d_x->getCoarsestLevelNumber(),
                     std::min(d_x->getFinestLevelNumber(), d_x->getPatchHierarchy()->getFinestLevelNumber()));
    d_x->freeVectorComponents();

    d_b->resetLevels(d_b->getCoarsestLevelNumber(),
                     std::min(d_b->getFinestLevelNumber(), d_b->getPatchHierarchy()->getFinestLevelNumber()));
    d_b->freeVectorComponents();

    d_x.setNull();
    d_b.setNull();

    // Indicate that the operator is NOT initialized.
    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_operator_state);
    return;
} // deallocateOperatorState
void
INSStaggeredVCStokesOperator::initializeOperatorState(
    const SAMRAIVectorReal<NDIM,double>& in,
    const SAMRAIVectorReal<NDIM,double>& /*out*/)
{
    IBAMR_TIMER_START(t_initialize_operator_state);

    if (d_is_initialized) deallocateOperatorState();

    d_x_scratch = in.cloneVector("INSStaggeredVCStokesOperator::x_scratch");
    d_x_scratch->allocateVectorData();

    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;
    InterpolationTransactionComponent U_scratch_component(d_x_scratch->getComponentDescriptorIndex(0), U_DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY);
    InterpolationTransactionComponent P_scratch_component(d_x_scratch->getComponentDescriptorIndex(1), P_DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY);
    InterpolationTransactionComponent mu_component(d_mu_data_idx, MU_DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY);

    std::vector<InterpolationTransactionComponent> U_P_MU_components(3);
    U_P_MU_components[0] = U_scratch_component;
    U_P_MU_components[1] = P_scratch_component;
    U_P_MU_components[2] = mu_component;

    d_U_P_MU_bdry_fill_op = new HierarchyGhostCellInterpolation();
    d_U_P_MU_bdry_fill_op->initializeOperatorState(U_P_MU_components, d_x_scratch->getPatchHierarchy());

    d_is_initialized = true;

    IBAMR_TIMER_STOP(t_initialize_operator_state);
    return;
}// initializeOperatorState
void
IBImplicitModHelmholtzPETScLevelSolver::deallocateSolverState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_solver_state);

    // Deallocate PETSc objects.
    int ierr;
    ierr = KSPDestroy(d_petsc_ksp); IBTK_CHKERRQ(ierr);
    ierr = MatDestroy(d_petsc_mat); IBTK_CHKERRQ(ierr);
    ierr = VecDestroy(d_petsc_x); IBTK_CHKERRQ(ierr);
    ierr = VecDestroy(d_petsc_b); IBTK_CHKERRQ(ierr);
    d_dof_index_fill.setNull();

    d_petsc_ksp = PETSC_NULL;
    d_petsc_mat = PETSC_NULL;
    d_petsc_x = PETSC_NULL;
    d_petsc_b = PETSC_NULL;

    // Deallocate DOF index data.
    Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(d_level_num);
    if (level->checkAllocated(d_dof_index_idx)) level->deallocatePatchData(d_dof_index_idx);

    // Indicate that the solver is NOT initialized.
    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_solver_state);
    return;
}// deallocateSolverState
Beispiel #5
0
void
HierarchyProjector::initializeLevelData(
    const Pointer<BasePatchHierarchy<NDIM> > hierarchy,
    const int level_number,
    const double /*init_data_time*/,
    const bool /*can_be_refined*/,
    const bool /*initial_time*/,
    const Pointer<BasePatchLevel<NDIM> > old_level,
    const bool /*allocate_data*/)
{
    IBAMR_TIMER_START(t_initialize_level_data);

#ifdef DEBUG_CHECK_ASSERTIONS
    TBOX_ASSERT(!hierarchy.isNull());
    TBOX_ASSERT((level_number >= 0)
                && (level_number <= hierarchy->getFinestLevelNumber()));
    if (!old_level.isNull())
    {
        TBOX_ASSERT(level_number == old_level->getLevelNumber());
    }
    TBOX_ASSERT(!(hierarchy->getPatchLevel(level_number)).isNull());
#else
    NULL_USE(hierarchy);
    NULL_USE(level_number);
    NULL_USE(old_level);
#endif

    // intentionally blank

    IBAMR_TIMER_STOP(t_initialize_level_data);
    return;
}// initializeLevelData
void
StaggeredStokesProjectionPreconditioner::initializeSolverState(const SAMRAIVectorReal<NDIM, double>& x,
                                                               const SAMRAIVectorReal<NDIM, double>& b)
{
    IBAMR_TIMER_START(t_initialize_solver_state);

    if (d_is_initialized) deallocateSolverState();

    // Parent class initialization.
    StaggeredStokesBlockPreconditioner::initializeSolverState(x, b);

    // Setup hierarchy operators.
    Pointer<VariableFillPattern<NDIM> > fill_pattern = new CellNoCornersFillPattern(CELLG, false, false, true);
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;
    InterpolationTransactionComponent P_scratch_component(d_Phi_scratch_idx,
                                                          DATA_REFINE_TYPE,
                                                          USE_CF_INTERPOLATION,
                                                          DATA_COARSEN_TYPE,
                                                          BDRY_EXTRAP_TYPE,
                                                          CONSISTENT_TYPE_2_BDRY,
                                                          d_P_bc_coef,
                                                          fill_pattern);
    d_Phi_bdry_fill_op = new HierarchyGhostCellInterpolation();
    d_Phi_bdry_fill_op->setHomogeneousBc(true);
    d_Phi_bdry_fill_op->initializeOperatorState(P_scratch_component, d_hierarchy);

    d_is_initialized = true;

    IBAMR_TIMER_STOP(t_initialize_solver_state);
    return;
} // initializeSolverState
void INSStaggeredCenteredConvectiveOperator::initializeOperatorState(
    const SAMRAIVectorReal<NDIM, double>& in,
    const SAMRAIVectorReal<NDIM, double>& out)
{
    IBAMR_TIMER_START(t_initialize_operator_state);

    if (d_is_initialized) deallocateOperatorState();

    // Get the hierarchy configuration.
    d_hierarchy = in.getPatchHierarchy();
    d_coarsest_ln = in.getCoarsestLevelNumber();
    d_finest_ln = in.getFinestLevelNumber();
#if !defined(NDEBUG)
    TBOX_ASSERT(d_hierarchy == out.getPatchHierarchy());
    TBOX_ASSERT(d_coarsest_ln == out.getCoarsestLevelNumber());
    TBOX_ASSERT(d_finest_ln == out.getFinestLevelNumber());
#else
    NULL_USE(out);
#endif

    // Setup the interpolation transaction information.
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent
    InterpolationTransactionComponent;
    d_transaction_comps.resize(1);
    d_transaction_comps[0] =
        InterpolationTransactionComponent(d_U_scratch_idx,
                                          in.getComponentDescriptorIndex(0),
                                          "CONSERVATIVE_LINEAR_REFINE",
                                          false,
                                          "CONSERVATIVE_COARSEN",
                                          d_bdry_extrap_type,
                                          false,
                                          d_bc_coefs);

    // Initialize the interpolation operators.
    d_hier_bdry_fill = new HierarchyGhostCellInterpolation();
    d_hier_bdry_fill->initializeOperatorState(d_transaction_comps, d_hierarchy);

    // Initialize the BC helper.
    d_bc_helper = new StaggeredStokesPhysicalBoundaryHelper();
    d_bc_helper->cacheBcCoefData(d_bc_coefs, d_solution_time, d_hierarchy);

    // Allocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        if (!level->checkAllocated(d_U_scratch_idx))
        {
            level->allocatePatchData(d_U_scratch_idx);
        }
    }
    d_is_initialized = true;

    IBAMR_TIMER_STOP(t_initialize_operator_state);
    return;
} // initializeOperatorState
bool
IBImplicitModHelmholtzPETScLevelSolver::solveSystem(
    SAMRAIVectorReal<NDIM,double>& x,
    SAMRAIVectorReal<NDIM,double>& b)
{
    IBAMR_TIMER_START(t_solve_system);

    int ierr;

    if (d_enable_logging) plog << d_object_name << "::solveSystem():" << std::endl;

    // Initialize the solver, when necessary.
    const bool deallocate_after_solve = !d_is_initialized;
    if (deallocate_after_solve) initializeSolverState(x,b);
#if 0 // XXXX
    // Configure solver.
    ierr = KSPSetTolerances(d_petsc_ksp, d_rel_residual_tol, d_abs_residual_tol, PETSC_DEFAULT, d_max_iterations); IBTK_CHKERRQ(ierr);
    ierr = KSPSetInitialGuessNonzero(d_petsc_ksp, d_initial_guess_nonzero ? PETSC_TRUE : PETSC_FALSE); IBTK_CHKERRQ(ierr);
#endif

    // Solve the system.
    Pointer<PatchLevel<NDIM> > patch_level = d_hierarchy->getPatchLevel(d_level_num);
    const int x_idx = x.getComponentDescriptorIndex(0);
    Pointer<SideVariable<NDIM,double> > x_var = x.getComponentVariable(0);
    const int b_idx = b.getComponentDescriptorIndex(0);
    Pointer<SideVariable<NDIM,double> > b_var = b.getComponentVariable(0);
    if (d_initial_guess_nonzero) PETScVecUtilities::copyToPatchLevelVec(d_petsc_x, x_idx, x_var, patch_level);
    PETScVecUtilities::copyToPatchLevelVec(d_petsc_b, b_idx, b_var, patch_level);
    PETScVecUtilities::constrainPatchLevelVec(d_petsc_b, d_dof_index_idx, d_dof_index_var, patch_level, d_dof_index_fill);
    ierr = KSPSolve(d_petsc_ksp, d_petsc_b, d_petsc_x); IBTK_CHKERRQ(ierr);
    PETScVecUtilities::copyFromPatchLevelVec(d_petsc_x, x_idx, x_var, patch_level);

    typedef SideDataSynchronization::SynchronizationTransactionComponent SynchronizationTransactionComponent; // XXXX
    SynchronizationTransactionComponent x_synch_transaction = SynchronizationTransactionComponent(x_idx, "CONSERVATIVE_COARSEN");
    Pointer<SideDataSynchronization> side_synch_op = new SideDataSynchronization();
    side_synch_op->initializeOperatorState(x_synch_transaction, x.getPatchHierarchy());
    side_synch_op->synchronizeData(0.0);

    // Log solver info.
    KSPConvergedReason reason;
    ierr = KSPGetConvergedReason(d_petsc_ksp, &reason); IBTK_CHKERRQ(ierr);
    const bool converged = reason > 0;
    if (d_enable_logging)
    {
        plog << d_object_name << "::solveSystem(): solver " << (converged ? "converged" : "diverged") << "\n"
             << "iterations = " << d_current_its << "\n"
             << "residual norm = " << d_current_residual_norm << std::endl;
    }

    // Deallocate the solver, when necessary.
    if (deallocate_after_solve) deallocateSolverState();

    IBAMR_TIMER_STOP(t_solve_system);
    return converged;
}// solveSystem
void
AdvDiffCenteredConvectiveOperator::initializeOperatorState(const SAMRAIVectorReal<NDIM, double>& in,
                                                           const SAMRAIVectorReal<NDIM, double>& out)
{
    IBAMR_TIMER_START(t_initialize_operator_state);

    if (d_is_initialized) deallocateOperatorState();

    // Get the hierarchy configuration.
    d_hierarchy = in.getPatchHierarchy();
    d_coarsest_ln = in.getCoarsestLevelNumber();
    d_finest_ln = in.getFinestLevelNumber();
#if !defined(NDEBUG)
    TBOX_ASSERT(d_hierarchy == out.getPatchHierarchy());
    TBOX_ASSERT(d_coarsest_ln == out.getCoarsestLevelNumber());
    TBOX_ASSERT(d_finest_ln == out.getFinestLevelNumber());
#else
    NULL_USE(out);
#endif
    Pointer<CartesianGridGeometry<NDIM> > grid_geom = d_hierarchy->getGridGeometry();

    // Setup the coarsen algorithm, operator, and schedules.
    Pointer<CoarsenOperator<NDIM> > coarsen_op = grid_geom->lookupCoarsenOperator(d_q_flux_var, "CONSERVATIVE_COARSEN");
    d_coarsen_alg = new CoarsenAlgorithm<NDIM>();
    if (d_difference_form == ADVECTIVE || d_difference_form == SKEW_SYMMETRIC)
        d_coarsen_alg->registerCoarsen(d_q_extrap_idx, d_q_extrap_idx, coarsen_op);
    if (d_difference_form == CONSERVATIVE || d_difference_form == SKEW_SYMMETRIC)
        d_coarsen_alg->registerCoarsen(d_q_flux_idx, d_q_flux_idx, coarsen_op);
    d_coarsen_scheds.resize(d_finest_ln + 1);
    for (int ln = d_coarsest_ln + 1; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        Pointer<PatchLevel<NDIM> > coarser_level = d_hierarchy->getPatchLevel(ln - 1);
        d_coarsen_scheds[ln] = d_coarsen_alg->createSchedule(coarser_level, level);
    }

    // Setup the refine algorithm, operator, patch strategy, and schedules.
    Pointer<RefineOperator<NDIM> > refine_op = grid_geom->lookupRefineOperator(d_Q_var, "CONSERVATIVE_LINEAR_REFINE");
    d_ghostfill_alg = new RefineAlgorithm<NDIM>();
    d_ghostfill_alg->registerRefine(d_Q_scratch_idx, in.getComponentDescriptorIndex(0), d_Q_scratch_idx, refine_op);
    if (d_outflow_bdry_extrap_type != "NONE")
        d_ghostfill_strategy = new CartExtrapPhysBdryOp(d_Q_scratch_idx, d_outflow_bdry_extrap_type);
    d_ghostfill_scheds.resize(d_finest_ln + 1);
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        d_ghostfill_scheds[ln] = d_ghostfill_alg->createSchedule(level, ln - 1, d_hierarchy, d_ghostfill_strategy);
    }

    d_is_initialized = true;

    IBAMR_TIMER_STOP(t_initialize_operator_state);
    return;
} // initializeOperatorState
Beispiel #10
0
void
HierarchyProjector::putToDatabase(
    Pointer<Database> db)
{
    IBAMR_TIMER_START(t_put_to_database);

#ifdef DEBUG_CHECK_ASSERTIONS
    TBOX_ASSERT(!db.isNull());
#endif
    db->putInteger("HIERARCHY_PROJECTOR_VERSION", HIERARCHY_PROJECTOR_VERSION);

    IBAMR_TIMER_STOP(t_put_to_database);
    return;
}// putToDatabase
void
INSStaggeredVCStokesOperator::deallocateOperatorState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_operator_state);

    d_x_scratch->freeVectorComponents();
    d_x_scratch.setNull();

    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_operator_state);
    return;
}// deallocateOperatorState
void
IBImplicitModHelmholtzOperator::deallocateOperatorState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_operator_state);

    d_helmholtz_op->deallocateOperatorState();
//  d_ib_SJSstar_op->deallocateOperatorState();

    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_operator_state);
    return;
}// deallocateOperatorState
void
INSStaggeredUpwindConvectiveOperator::deallocateOperatorState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_operator_state);

    // Deallocate the communications operators and BC helpers.
    d_hier_bdry_fill.setNull();
    d_bc_helper.setNull();

    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_operator_state);
    return;
} // deallocateOperatorState
void
StaggeredStokesProjectionPreconditioner::deallocateSolverState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_solver_state);

    // Parent class deallocation.
    StaggeredStokesBlockPreconditioner::deallocateSolverState();

    // Deallocate hierarchy operators.
    d_Phi_bdry_fill_op.setNull();

    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_solver_state);
    return;
} // deallocateSolverState
void
AdvDiffCenteredConvectiveOperator::deallocateOperatorState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_operator_state);

    // Deallocate the refine algorithm, operator, patch strategy, and schedules.
    d_ghostfill_alg.setNull();
    d_ghostfill_strategy.setNull();
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        d_ghostfill_scheds[ln].setNull();
    }
    d_ghostfill_scheds.clear();

    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_operator_state);
    return;
} // deallocateOperatorState
void
IBImplicitModHelmholtzOperator::apply(
    SAMRAIVectorReal<NDIM,double>& x,
    SAMRAIVectorReal<NDIM,double>& y)
{
    IBAMR_TIMER_START(t_apply);

    SAMRAIVectorReal<NDIM,double> x_u(x.getName(), x.getPatchHierarchy(), x.getCoarsestLevelNumber(), x.getFinestLevelNumber());
    x_u.addComponent(x.getComponentVariable(0), x.getComponentDescriptorIndex(0), x.getControlVolumeIndex(0));
    SAMRAIVectorReal<NDIM,double> y_u(y.getName(), y.getPatchHierarchy(), y.getCoarsestLevelNumber(), y.getFinestLevelNumber());
    y_u.addComponent(y.getComponentVariable(0), y.getComponentDescriptorIndex(0), y.getControlVolumeIndex(0));

    // Apply the linear part of the operator with homogeneous boundary
    // conditions.
    d_helmholtz_op->apply(x_u, y_u);

    // Apply the nonlinear part of the operator.
    d_ib_SJSstar_op->applyAdd(x_u, y_u, y_u);

    IBAMR_TIMER_STOP(t_apply);
    return;
}// apply
void
IBImplicitModHelmholtzOperator::initializeOperatorState(
    const SAMRAIVectorReal<NDIM,double>& in,
    const SAMRAIVectorReal<NDIM,double>& out)
{
    IBAMR_TIMER_START(t_initialize_operator_state);

    if (d_is_initialized) deallocateOperatorState();

    SAMRAIVectorReal<NDIM,double> in_u(in.getName(), in.getPatchHierarchy(), in.getCoarsestLevelNumber(), in.getFinestLevelNumber());
    in_u.addComponent(in.getComponentVariable(0), in.getComponentDescriptorIndex(0), in.getControlVolumeIndex(0));
    SAMRAIVectorReal<NDIM,double> out_u(out.getName(), out.getPatchHierarchy(), out.getCoarsestLevelNumber(), out.getFinestLevelNumber());
    out_u.addComponent(out.getComponentVariable(0), out.getComponentDescriptorIndex(0), out.getControlVolumeIndex(0));

    d_helmholtz_op->initializeOperatorState(in_u, out_u);
//  d_ib_SJSstar_op->initializeOperatorState(in_u, out_u);

    d_is_initialized = true;

    IBAMR_TIMER_STOP(t_initialize_operator_state);
    return;
}// initializeOperatorState
void
INSCollocatedPPMConvectiveOperator::deallocateOperatorState()
{
    if (!d_is_initialized) return;

    IBAMR_TIMER_START(t_deallocate_operator_state);

    // Deallocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        if (level->checkAllocated(d_U_scratch_idx))
        {
            level->deallocatePatchData(d_U_scratch_idx);
        }
        if (level->checkAllocated(d_u_extrap_idx))
        {
            level->deallocatePatchData(d_u_extrap_idx);
        }
        if (level->checkAllocated(d_u_flux_idx))
        {
            level->deallocatePatchData(d_u_flux_idx);
        }
    }

    // Deallocate the refine algorithm, operator, patch strategy, and schedules.
    d_ghostfill_alg.setNull();
    d_ghostfill_strategy.setNull();
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        d_ghostfill_scheds[ln].setNull();
    }
    d_ghostfill_scheds.clear();

    d_is_initialized = false;

    IBAMR_TIMER_STOP(t_deallocate_operator_state);
    return;
}// deallocateOperatorState
void StaggeredStokesOperator::apply(SAMRAIVectorReal<NDIM, double>& x, SAMRAIVectorReal<NDIM, double>& y)
{
    IBAMR_TIMER_START(t_apply);

    // Get the vector components.
    const int U_idx = x.getComponentDescriptorIndex(0);
    const int P_idx = x.getComponentDescriptorIndex(1);
    const int A_U_idx = y.getComponentDescriptorIndex(0);
    const int A_P_idx = y.getComponentDescriptorIndex(1);
    const int U_scratch_idx = d_x->getComponentDescriptorIndex(0);

    Pointer<SideVariable<NDIM, double> > U_sc_var = x.getComponentVariable(0);
    Pointer<CellVariable<NDIM, double> > P_cc_var = x.getComponentVariable(1);
    Pointer<SideVariable<NDIM, double> > A_U_sc_var = y.getComponentVariable(0);
    Pointer<CellVariable<NDIM, double> > A_P_cc_var = y.getComponentVariable(1);

    // Simultaneously fill ghost cell values for all components.
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;
    std::vector<InterpolationTransactionComponent> transaction_comps(2);
    transaction_comps[0] = InterpolationTransactionComponent(U_scratch_idx,
                                                             U_idx,
                                                             DATA_REFINE_TYPE,
                                                             USE_CF_INTERPOLATION,
                                                             DATA_COARSEN_TYPE,
                                                             BDRY_EXTRAP_TYPE,
                                                             CONSISTENT_TYPE_2_BDRY,
                                                             d_U_bc_coefs,
                                                             d_U_fill_pattern);
    transaction_comps[1] = InterpolationTransactionComponent(P_idx,
                                                             DATA_REFINE_TYPE,
                                                             USE_CF_INTERPOLATION,
                                                             DATA_COARSEN_TYPE,
                                                             BDRY_EXTRAP_TYPE,
                                                             CONSISTENT_TYPE_2_BDRY,
                                                             d_P_bc_coef,
                                                             d_P_fill_pattern);
    d_hier_bdry_fill->resetTransactionComponents(transaction_comps);
    d_hier_bdry_fill->setHomogeneousBc(d_homogeneous_bc);
    StaggeredStokesPhysicalBoundaryHelper::setupBcCoefObjects(
        d_U_bc_coefs, d_P_bc_coef, U_scratch_idx, P_idx, d_homogeneous_bc);
    d_hier_bdry_fill->fillData(d_solution_time);
    StaggeredStokesPhysicalBoundaryHelper::resetBcCoefObjects(d_U_bc_coefs, d_P_bc_coef);
    //  d_bc_helper->enforceDivergenceFreeConditionAtBoundary(U_scratch_idx);
    d_hier_bdry_fill->resetTransactionComponents(d_transaction_comps);

    // Compute the action of the operator:
    //
    // A*[U;P] := [A_U;A_P] = [(C*I+D*L)*U + Grad P; -Div U]
    d_hier_math_ops->grad(A_U_idx,
                          A_U_sc_var,
                          /*cf_bdry_synch*/ false,
                          1.0,
                          P_idx,
                          P_cc_var,
                          d_no_fill,
                          d_new_time);
    d_hier_math_ops->laplace(A_U_idx,
                             A_U_sc_var,
                             d_U_problem_coefs,
                             U_scratch_idx,
                             U_sc_var,
                             d_no_fill,
                             d_new_time,
                             1.0,
                             A_U_idx,
                             A_U_sc_var);
    d_hier_math_ops->div(A_P_idx,
                         A_P_cc_var,
                         -1.0,
                         U_scratch_idx,
                         U_sc_var,
                         d_no_fill,
                         d_new_time,
                         /*cf_bdry_synch*/ true);
    d_bc_helper->copyDataAtDirichletBoundaries(A_U_idx, U_scratch_idx);

    IBAMR_TIMER_STOP(t_apply);
    return;
} // apply
void
INSStaggeredVCStokesOperator::apply(
    const bool /*homogeneous_bc*/,
    SAMRAIVectorReal<NDIM,double>& x,
    SAMRAIVectorReal<NDIM,double>& y)
{
    IBAMR_TIMER_START(t_apply);

    // Get the vector components.
//  const int U_in_idx       =            x.getComponentDescriptorIndex(0);
//  const int P_in_idx       =            x.getComponentDescriptorIndex(1);
    const int U_out_idx      =            y.getComponentDescriptorIndex(0);
    const int P_out_idx      =            y.getComponentDescriptorIndex(1);
    const int U_scratch_idx  = d_x_scratch->getComponentDescriptorIndex(0);
    const int P_scratch_idx  = d_x_scratch->getComponentDescriptorIndex(1);

    const Pointer<Variable<NDIM> >& U_out_var = y.getComponentVariable(0);
    const Pointer<Variable<NDIM> >& P_out_var = y.getComponentVariable(1);

    Pointer<SideVariable<NDIM,double> > U_out_sc_var = U_out_var;
    Pointer<CellVariable<NDIM,double> > P_out_cc_var = P_out_var;

    const Pointer<Variable<NDIM> >& U_scratch_var = d_x_scratch->getComponentVariable(0);
    const Pointer<Variable<NDIM> >& P_scratch_var = d_x_scratch->getComponentVariable(1);

    Pointer<SideVariable<NDIM,double> > U_scratch_sc_var = U_scratch_var;
    Pointer<CellVariable<NDIM,double> > P_scratch_cc_var = P_scratch_var;

    d_x_scratch->copyVector(Pointer<SAMRAIVectorReal<NDIM,double> >(&x,false));

    // Reset the interpolation operators and fill the data.
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;
    InterpolationTransactionComponent U_scratch_component(U_scratch_idx, U_DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY);
    InterpolationTransactionComponent P_scratch_component(P_scratch_idx, P_DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY);
    InterpolationTransactionComponent mu_component(d_mu_data_idx, MU_DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY);
    std::vector<InterpolationTransactionComponent> U_P_MU_components(3);
    U_P_MU_components[0] = U_scratch_component;
    U_P_MU_components[1] = P_scratch_component;
    U_P_MU_components[2] = mu_component;
    d_U_P_MU_bdry_fill_op->resetTransactionComponents(U_P_MU_components);
    d_U_P_MU_bdry_fill_op->fillData(d_new_time);

    // Setup hierarchy data ops object for U.
    HierarchyDataOpsManager<NDIM>* hier_ops_manager = HierarchyDataOpsManager<NDIM>::getManager();
    Pointer<PatchHierarchy<NDIM> > hierarchy = y.getPatchHierarchy();
    Pointer<HierarchySideDataOpsReal<NDIM,double> > hier_sc_data_ops =
        hier_ops_manager->getOperationsDouble(U_out_var, hierarchy, true);

    // Compute the action of the operator:
    //      A*[u;p] = [(rho/dt)*u-0.5*div*(mu*(grad u + (grad u)^T)) + grad p; -div u].
    //
    static const bool cf_bdry_synch = true;

    d_hier_math_ops->pointwiseMultiply(
	U_out_idx, U_out_sc_var,
        d_rho_data_idx, d_rho_var,
        U_scratch_idx, U_scratch_sc_var,
        0.0, -1, NULL);

    d_hier_math_ops->vc_laplace(
        U_out_idx, U_out_sc_var,
        -0.5, 0.0, d_mu_data_idx, d_mu_var,
        U_scratch_idx, U_scratch_sc_var, d_no_fill_op, d_new_time,
        1.0/d_dt, U_out_idx, U_out_sc_var);

    d_hier_math_ops->grad(
        U_out_idx, U_out_sc_var,
        cf_bdry_synch,
        1.0, P_scratch_idx, P_scratch_cc_var, d_no_fill_op, d_new_time,
        1.0, U_out_idx, U_out_sc_var);

    d_hier_math_ops->div(
        P_out_idx, P_out_cc_var,
        -1.0, U_scratch_idx, U_scratch_sc_var, d_no_fill_op, d_new_time,
        cf_bdry_synch);

    IBAMR_TIMER_STOP(t_apply);
    return;
}// apply
void
INSCollocatedPPMConvectiveOperator::applyConvectiveOperator(
    const int U_idx,
    const int N_idx)
{
    IBAMR_TIMER_START(t_apply_convective_operator);
#ifdef DEBUG_CHECK_ASSERTIONS
    if (!d_is_initialized)
    {
        TBOX_ERROR("INSCollocatedPPMConvectiveOperator::applyConvectiveOperator():\n"
                   << "  operator must be initialized prior to call to applyConvectiveOperator\n");
    }
#endif

    // Setup communications algorithm.
    Pointer<CartesianGridGeometry<NDIM> > grid_geom = d_hierarchy->getGridGeometry();
    Pointer<RefineAlgorithm<NDIM> > refine_alg = new RefineAlgorithm<NDIM>();
    Pointer<RefineOperator<NDIM> > refine_op = grid_geom->lookupRefineOperator(d_U_var, "CONSERVATIVE_LINEAR_REFINE");
    refine_alg->registerRefine(d_U_scratch_idx, U_idx, d_U_scratch_idx, refine_op);

    // Extrapolate from cell centers to cell faces.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        refine_alg->resetSchedule(d_ghostfill_scheds[ln]);
        d_ghostfill_scheds[ln]->fillData(d_solution_time);
        d_ghostfill_alg->resetSchedule(d_ghostfill_scheds[ln]);
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        for (PatchLevel<NDIM>::Iterator p(level); p; p++)
        {
            Pointer<Patch<NDIM> > patch = level->getPatch(p());

            const Box<NDIM>& patch_box = patch->getBox();
            const IntVector<NDIM>& patch_lower = patch_box.lower();
            const IntVector<NDIM>& patch_upper = patch_box.upper();

            Pointer<CellData<NDIM,double> > U_data = patch->getPatchData(d_U_scratch_idx);
            const IntVector<NDIM>& U_data_gcw = U_data->getGhostCellWidth();
#ifdef DEBUG_CHECK_ASSERTIONS
            TBOX_ASSERT(U_data_gcw.min() == U_data_gcw.max());
#endif
            Pointer<FaceData<NDIM,double> > u_ADV_data = patch->getPatchData(d_u_idx);
            const IntVector<NDIM>& u_ADV_data_gcw = u_ADV_data->getGhostCellWidth();
#ifdef DEBUG_CHECK_ASSERTIONS
            TBOX_ASSERT(u_ADV_data_gcw.min() == u_ADV_data_gcw.max());
#endif
            Pointer<FaceData<NDIM,double> > u_extrap_data = patch->getPatchData(d_u_extrap_idx);
            const IntVector<NDIM>& u_extrap_data_gcw = u_extrap_data->getGhostCellWidth();
#ifdef DEBUG_CHECK_ASSERTIONS
            TBOX_ASSERT(u_extrap_data_gcw.min() == u_extrap_data_gcw.max());
#endif
            CellData<NDIM,double>& U0_data = *U_data;
            CellData<NDIM,double>  U1_data(patch_box, 1, U_data_gcw);
#if (NDIM == 3)
            CellData<NDIM,double>  U2_data(patch_box, 1, U_data_gcw);
#endif
            CellData<NDIM,double>  dU_data(patch_box, 1, U_data_gcw);
            CellData<NDIM,double> U_L_data(patch_box, 1, U_data_gcw);
            CellData<NDIM,double> U_R_data(patch_box, 1, U_data_gcw);

            // Extrapolate from cell centers to cell faces.
            for (unsigned int axis = 0; axis < NDIM; ++axis)
            {
                GODUNOV_EXTRAPOLATE_FC(
#if (NDIM == 2)
                    patch_lower(0), patch_upper(0),
                    patch_lower(1), patch_upper(1),
                    U_data_gcw(0), U_data_gcw(1),
                    U0_data.getPointer(axis), U1_data.getPointer(),
                    dU_data.getPointer(), U_L_data.getPointer(), U_R_data.getPointer(),
                    u_ADV_data_gcw   (0), u_ADV_data_gcw   (1),
                    u_extrap_data_gcw(0), u_extrap_data_gcw(1),
                    u_ADV_data   ->getPointer(0),      u_ADV_data   ->getPointer(1),
                    u_extrap_data->getPointer(0,axis), u_extrap_data->getPointer(1,axis)
#endif
#if (NDIM == 3)
                    patch_lower(0), patch_upper(0),
                    patch_lower(1), patch_upper(1),
                    patch_lower(2), patch_upper(2),
                    U_data_gcw(0), U_data_gcw(1), U_data_gcw(2),
                    U0_data.getPointer(axis), U1_data.getPointer(), U2_data.getPointer(),
                    dU_data.getPointer(), U_L_data.getPointer(), U_R_data.getPointer(),
                    u_ADV_data_gcw   (0), u_ADV_data_gcw   (1), u_ADV_data_gcw   (2),
                    u_extrap_data_gcw(0), u_extrap_data_gcw(1), u_extrap_data_gcw(2),
                    u_ADV_data   ->getPointer(0),      u_ADV_data   ->getPointer(1),      u_ADV_data   ->getPointer(2),
                    u_extrap_data->getPointer(0,axis), u_extrap_data->getPointer(1,axis), u_extrap_data->getPointer(2,axis)
#endif
                                       );
            }

            // If we are using conservative or skew-symmetric differencing,
            // compute the advective fluxes.  These need to be synchronized on
            // the patch hierarchy.
            if (d_difference_form == CONSERVATIVE || d_difference_form == SKEW_SYMMETRIC)
            {
                Pointer<FaceData<NDIM,double> > u_ADV_data = patch->getPatchData(d_u_idx);
                const IntVector<NDIM>& u_ADV_data_gcw = u_ADV_data->getGhostCellWidth();
                Pointer<FaceData<NDIM,double> > u_flux_data = patch->getPatchData(d_u_flux_idx);
                const IntVector<NDIM>& u_flux_data_gcw = u_flux_data->getGhostCellWidth();
                for (unsigned int axis = 0; axis < NDIM; ++axis)
                {
                    static const double dt = 1.0;
                    ADVECT_FLUX_FC(
                        dt,
#if (NDIM == 2)
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
//                      u_extrap_data_gcw(0), u_extrap_data_gcw(1),
                        u_ADV_data_gcw   (0), u_ADV_data_gcw   (1),
                        u_extrap_data_gcw(0), u_extrap_data_gcw(1),
                        u_flux_data_gcw  (0), u_flux_data_gcw  (1),
//                      u_extrap_data->getPointer(0,0),    u_extrap_data->getPointer(1,1),
                        u_ADV_data   ->getPointer(0),      u_ADV_data   ->getPointer(1),
                        u_extrap_data->getPointer(0,axis), u_extrap_data->getPointer(1,axis),
                        u_flux_data  ->getPointer(0,axis), u_flux_data  ->getPointer(1,axis)
#endif
#if (NDIM == 3)
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
                        patch_lower(2), patch_upper(2),
//                      u_extrap_data_gcw(0), u_extrap_data_gcw(1), u_extrap_data_gcw(2),
                        u_ADV_data_gcw   (0), u_ADV_data_gcw   (1), u_ADV_data_gcw   (2),
                        u_extrap_data_gcw(0), u_extrap_data_gcw(1), u_extrap_data_gcw(2),
                        u_flux_data_gcw  (0), u_flux_data_gcw  (1), u_flux_data_gcw  (2),
//                      u_extrap_data->getPointer(0,0),    u_extrap_data->getPointer(1,1),    u_extrap_data->getPointer(2,2),
                        u_ADV_data   ->getPointer(0),      u_ADV_data   ->getPointer(1),      u_ADV_data   ->getPointer(2),
                        u_extrap_data->getPointer(0,axis), u_extrap_data->getPointer(1,axis), u_extrap_data->getPointer(2,axis),
                        u_flux_data  ->getPointer(0,axis), u_flux_data  ->getPointer(1,axis), u_flux_data  ->getPointer(2,axis)
#endif
                                   );
                }
            }
        }
    }

    // Synchronize data on the patch hierarchy.
    for (int ln = d_finest_ln; ln > d_coarsest_ln; --ln)
    {
        d_coarsen_scheds[ln]->coarsenData();
    }

    // Difference values on the patches.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        for (PatchLevel<NDIM>::Iterator p(level); p; p++)
        {
            Pointer<Patch<NDIM> > patch = level->getPatch(p());

            const Box<NDIM>& patch_box = patch->getBox();
            const IntVector<NDIM>& patch_lower = patch_box.lower();
            const IntVector<NDIM>& patch_upper = patch_box.upper();

            const Pointer<CartesianPatchGeometry<NDIM> > patch_geom = patch->getPatchGeometry();
            const double* const dx = patch_geom->getDx();

            Pointer<CellData<NDIM,double> > N_data = patch->getPatchData(N_idx);
            const IntVector<NDIM>& N_data_gcw = N_data->getGhostCellWidth();

            if (d_difference_form == ADVECTIVE || d_difference_form == SKEW_SYMMETRIC)
            {
                Pointer<FaceData<NDIM,double> > u_ADV_data = patch->getPatchData(d_u_idx);
                const IntVector<NDIM>& u_ADV_data_gcw = u_ADV_data->getGhostCellWidth();
                Pointer<FaceData<NDIM,double> > u_extrap_data = patch->getPatchData(d_u_extrap_idx);
                const IntVector<NDIM>& u_extrap_data_gcw = u_extrap_data->getGhostCellWidth();
                for (unsigned int axis = 0; axis < NDIM; ++axis)
                {
                    ADVECT_DERIVATIVE_FC(
                        dx,
#if (NDIM == 2)
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
//                      u_extrap_data_gcw(0), u_extrap_data_gcw(1),
                        u_ADV_data_gcw   (0), u_ADV_data_gcw   (1),
                        u_extrap_data_gcw(0), u_extrap_data_gcw(1),
//                      u_extrap_data->getPointer(0,0),    u_extrap_data->getPointer(1,1),
                        u_ADV_data   ->getPointer(0),      u_ADV_data   ->getPointer(1),
                        u_extrap_data->getPointer(0,axis), u_extrap_data->getPointer(1,axis),
                        N_data_gcw(0), N_data_gcw(1),
#endif
#if (NDIM == 3)
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
                        patch_lower(2), patch_upper(2),
//                      u_extrap_data_gcw(0), u_extrap_data_gcw(1), u_extrap_data_gcw(2),
                        u_ADV_data_gcw   (0), u_ADV_data_gcw   (1), u_ADV_data_gcw   (2),
                        u_extrap_data_gcw(0), u_extrap_data_gcw(1), u_extrap_data_gcw(2),
//                      u_extrap_data->getPointer(0,0),    u_extrap_data->getPointer(1,1),    u_extrap_data->getPointer(2,2),
                        u_ADV_data   ->getPointer(0),      u_ADV_data   ->getPointer(1),      u_ADV_data   ->getPointer(2),
                        u_extrap_data->getPointer(0,axis), u_extrap_data->getPointer(1,axis), u_extrap_data->getPointer(2,axis),
                        N_data_gcw(0), N_data_gcw(1), N_data_gcw(2),
#endif
                        N_data->getPointer(axis));
                }
            }

            if (d_difference_form == CONSERVATIVE)
            {
                Pointer<FaceData<NDIM,double> > u_flux_data = patch->getPatchData(d_u_flux_idx);
                const IntVector<NDIM>& u_flux_data_gcw = u_flux_data->getGhostCellWidth();
                for (unsigned int axis = 0; axis < NDIM; ++axis)
                {
                    static const double alpha = 1.0;
                    F_TO_C_DIV_FC(
                        N_data->getPointer(axis), N_data_gcw.min(),
                        alpha,
#if (NDIM == 2)
                        u_flux_data->getPointer(0,axis), u_flux_data->getPointer(1,axis), u_flux_data_gcw.min(),
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
#endif
#if (NDIM == 3)
                        u_flux_data->getPointer(0,axis), u_flux_data->getPointer(1,axis), u_flux_data->getPointer(2,axis), u_flux_data_gcw.min(),
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
                        patch_lower(2), patch_upper(2),
#endif
                        dx);
                }
            }

            if (d_difference_form == SKEW_SYMMETRIC)
            {
                Pointer<FaceData<NDIM,double> > u_flux_data = patch->getPatchData(d_u_flux_idx);
                const IntVector<NDIM>& u_flux_data_gcw = u_flux_data->getGhostCellWidth();
                for (unsigned int axis = 0; axis < NDIM; ++axis)
                {
                    static const double alpha = 0.5;
                    static const double beta  = 0.5;
                    F_TO_C_DIV_ADD_FC(
                        N_data->getPointer(axis), N_data_gcw.min(),
                        alpha,
#if (NDIM == 2)
                        u_flux_data->getPointer(0,axis), u_flux_data->getPointer(1,axis), u_flux_data_gcw.min(),
                        beta,
                        N_data->getPointer(axis), N_data_gcw.min(),
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
#endif
#if (NDIM == 3)
                        u_flux_data->getPointer(0,axis), u_flux_data->getPointer(1,axis), u_flux_data->getPointer(2,axis), u_flux_data_gcw.min(),
                        beta,
                        N_data->getPointer(axis), N_data_gcw.min(),
                        patch_lower(0), patch_upper(0),
                        patch_lower(1), patch_upper(1),
                        patch_lower(2), patch_upper(2),
#endif
                        dx);
                }
            }
        }
    }
    IBAMR_TIMER_STOP(t_apply_convective_operator);
    return;
}// applyConvectiveOperator
void
AdvDiffCenteredConvectiveOperator::applyConvectiveOperator(const int Q_idx, const int N_idx)
{
    IBAMR_TIMER_START(t_apply_convective_operator);
#if !defined(NDEBUG)
    if (!d_is_initialized)
    {
        TBOX_ERROR("AdvDiffCenteredConvectiveOperator::applyConvectiveOperator():\n"
                   << "  operator must be initialized prior to call to applyConvectiveOperator\n");
    }
#endif

    // Allocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->allocatePatchData(d_Q_scratch_idx);
        level->allocatePatchData(d_q_extrap_idx);
        if (d_difference_form == CONSERVATIVE || d_difference_form == SKEW_SYMMETRIC)
            level->allocatePatchData(d_q_flux_idx);
    }

    // Setup communications algorithm.
    Pointer<CartesianGridGeometry<NDIM> > grid_geom = d_hierarchy->getGridGeometry();
    Pointer<RefineAlgorithm<NDIM> > refine_alg = new RefineAlgorithm<NDIM>();
    Pointer<RefineOperator<NDIM> > refine_op = grid_geom->lookupRefineOperator(d_Q_var, "CONSERVATIVE_LINEAR_REFINE");
    refine_alg->registerRefine(d_Q_scratch_idx, Q_idx, d_Q_scratch_idx, refine_op);

    // Extrapolate from cell centers to cell faces.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        refine_alg->resetSchedule(d_ghostfill_scheds[ln]);
        d_ghostfill_scheds[ln]->fillData(d_solution_time);
        d_ghostfill_alg->resetSchedule(d_ghostfill_scheds[ln]);
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        for (PatchLevel<NDIM>::Iterator p(level); p; p++)
        {
            Pointer<Patch<NDIM> > patch = level->getPatch(p());

            const Box<NDIM>& patch_box = patch->getBox();
            const IntVector<NDIM>& patch_lower = patch_box.lower();
            const IntVector<NDIM>& patch_upper = patch_box.upper();

            Pointer<CellData<NDIM, double> > Q_data = patch->getPatchData(d_Q_scratch_idx);
            const IntVector<NDIM>& Q_data_gcw = Q_data->getGhostCellWidth();
#if !defined(NDEBUG)
            TBOX_ASSERT(Q_data_gcw.min() == Q_data_gcw.max());
#endif
            Pointer<FaceData<NDIM, double> > u_ADV_data = patch->getPatchData(d_u_idx);
            const IntVector<NDIM>& u_ADV_data_gcw = u_ADV_data->getGhostCellWidth();
#if !defined(NDEBUG)
            TBOX_ASSERT(u_ADV_data_gcw.min() == u_ADV_data_gcw.max());
#endif
            Pointer<FaceData<NDIM, double> > q_extrap_data = patch->getPatchData(d_q_extrap_idx);
            const IntVector<NDIM>& q_extrap_data_gcw = q_extrap_data->getGhostCellWidth();
#if !defined(NDEBUG)
            TBOX_ASSERT(q_extrap_data_gcw.min() == q_extrap_data_gcw.max());
#endif
            // Enforce physical boundary conditions at inflow boundaries.
            AdvDiffPhysicalBoundaryUtilities::setPhysicalBoundaryConditions(
                Q_data,
                u_ADV_data,
                patch,
                d_bc_coefs,
                d_solution_time,
                /*inflow_boundary_only*/ d_outflow_bdry_extrap_type != "NONE",
                d_homogeneous_bc);

            // Interpolate from cell centers to cell faces.
            for (unsigned int d = 0; d < d_Q_data_depth; ++d)
            {
                C_TO_F_CWISE_INTERP_2ND_FC(
#if (NDIM == 2)
                    q_extrap_data->getPointer(0, d),
                    q_extrap_data->getPointer(1, d),
                    q_extrap_data_gcw.min(),
                    Q_data->getPointer(d),
                    Q_data_gcw.min(),
                    patch_lower(0),
                    patch_upper(0),
                    patch_lower(1),
                    patch_upper(1)
#endif
#if (NDIM == 3)
                        q_extrap_data->getPointer(0, d),
                    q_extrap_data->getPointer(1, d),
                    q_extrap_data->getPointer(2, d),
                    q_extrap_data_gcw.min(),
                    Q_data->getPointer(d),
                    Q_data_gcw.min(),
                    patch_lower(0),
                    patch_upper(0),
                    patch_lower(1),
                    patch_upper(1),
                    patch_lower(2),
                    patch_upper(2)
#endif
                        );
            }

            // If we are using conservative or skew-symmetric differencing,
            // compute the advective fluxes.  These need to be synchronized on
            // the patch hierarchy.
            if (d_difference_form == CONSERVATIVE || d_difference_form == SKEW_SYMMETRIC)
            {
                Pointer<FaceData<NDIM, double> > q_flux_data = patch->getPatchData(d_q_flux_idx);
                const IntVector<NDIM>& q_flux_data_gcw = q_flux_data->getGhostCellWidth();
                for (unsigned int d = 0; d < d_Q_data_depth; ++d)
                {
                    static const double dt = 1.0;
                    ADVECT_FLUX_FC(dt,
#if (NDIM == 2)
                                   patch_lower(0),
                                   patch_upper(0),
                                   patch_lower(1),
                                   patch_upper(1),
                                   u_ADV_data_gcw(0),
                                   u_ADV_data_gcw(1),
                                   q_extrap_data_gcw(0),
                                   q_extrap_data_gcw(1),
                                   q_flux_data_gcw(0),
                                   q_flux_data_gcw(1),
                                   u_ADV_data->getPointer(0),
                                   u_ADV_data->getPointer(1),
                                   q_extrap_data->getPointer(0, d),
                                   q_extrap_data->getPointer(1, d),
                                   q_flux_data->getPointer(0, d),
                                   q_flux_data->getPointer(1, d)
#endif
#if (NDIM == 3)
                                       patch_lower(0),
                                   patch_upper(0),
                                   patch_lower(1),
                                   patch_upper(1),
                                   patch_lower(2),
                                   patch_upper(2),
                                   u_ADV_data_gcw(0),
                                   u_ADV_data_gcw(1),
                                   u_ADV_data_gcw(2),
                                   q_extrap_data_gcw(0),
                                   q_extrap_data_gcw(1),
                                   q_extrap_data_gcw(2),
                                   q_flux_data_gcw(0),
                                   q_flux_data_gcw(1),
                                   q_flux_data_gcw(2),
                                   u_ADV_data->getPointer(0),
                                   u_ADV_data->getPointer(1),
                                   u_ADV_data->getPointer(2),
                                   q_extrap_data->getPointer(0, d),
                                   q_extrap_data->getPointer(1, d),
                                   q_extrap_data->getPointer(2, d),
                                   q_flux_data->getPointer(0, d),
                                   q_flux_data->getPointer(1, d),
                                   q_flux_data->getPointer(2, d)
#endif
                                       );
                }
            }
        }
    }

    // Synchronize data on the patch hierarchy.
    for (int ln = d_finest_ln; ln > d_coarsest_ln; --ln)
    {
        d_coarsen_scheds[ln]->coarsenData();
    }

    // Difference values on the patches.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        for (PatchLevel<NDIM>::Iterator p(level); p; p++)
        {
            Pointer<Patch<NDIM> > patch = level->getPatch(p());

            const Box<NDIM>& patch_box = patch->getBox();
            const IntVector<NDIM>& patch_lower = patch_box.lower();
            const IntVector<NDIM>& patch_upper = patch_box.upper();

            const Pointer<CartesianPatchGeometry<NDIM> > patch_geom = patch->getPatchGeometry();
            const double* const dx = patch_geom->getDx();

            Pointer<CellData<NDIM, double> > N_data = patch->getPatchData(N_idx);
            const IntVector<NDIM>& N_data_gcw = N_data->getGhostCellWidth();

            if (d_difference_form == ADVECTIVE || d_difference_form == SKEW_SYMMETRIC)
            {
                Pointer<FaceData<NDIM, double> > u_ADV_data = patch->getPatchData(d_u_idx);
                const IntVector<NDIM>& u_ADV_data_gcw = u_ADV_data->getGhostCellWidth();
                Pointer<FaceData<NDIM, double> > q_extrap_data = patch->getPatchData(d_q_extrap_idx);
                const IntVector<NDIM>& q_extrap_data_gcw = q_extrap_data->getGhostCellWidth();
                for (unsigned int d = 0; d < d_Q_data_depth; ++d)
                {
                    ADVECT_DERIVATIVE_FC(dx,
#if (NDIM == 2)
                                         patch_lower(0),
                                         patch_upper(0),
                                         patch_lower(1),
                                         patch_upper(1),
                                         u_ADV_data_gcw(0),
                                         u_ADV_data_gcw(1),
                                         q_extrap_data_gcw(0),
                                         q_extrap_data_gcw(1),
                                         u_ADV_data->getPointer(0),
                                         u_ADV_data->getPointer(1),
                                         q_extrap_data->getPointer(0, d),
                                         q_extrap_data->getPointer(1, d),
                                         N_data_gcw(0),
                                         N_data_gcw(1),
#endif
#if (NDIM == 3)
                                         patch_lower(0),
                                         patch_upper(0),
                                         patch_lower(1),
                                         patch_upper(1),
                                         patch_lower(2),
                                         patch_upper(2),
                                         u_ADV_data_gcw(0),
                                         u_ADV_data_gcw(1),
                                         u_ADV_data_gcw(2),
                                         q_extrap_data_gcw(0),
                                         q_extrap_data_gcw(1),
                                         q_extrap_data_gcw(2),
                                         u_ADV_data->getPointer(0),
                                         u_ADV_data->getPointer(1),
                                         u_ADV_data->getPointer(2),
                                         q_extrap_data->getPointer(0, d),
                                         q_extrap_data->getPointer(1, d),
                                         q_extrap_data->getPointer(2, d),
                                         N_data_gcw(0),
                                         N_data_gcw(1),
                                         N_data_gcw(2),
#endif
                                         N_data->getPointer(d));
                }
            }

            if (d_difference_form == CONSERVATIVE)
            {
                Pointer<FaceData<NDIM, double> > q_flux_data = patch->getPatchData(d_q_flux_idx);
                const IntVector<NDIM>& q_flux_data_gcw = q_flux_data->getGhostCellWidth();
                for (unsigned int d = 0; d < d_Q_data_depth; ++d)
                {
                    static const double alpha = 1.0;
                    F_TO_C_DIV_FC(N_data->getPointer(d),
                                  N_data_gcw.min(),
                                  alpha,
#if (NDIM == 2)
                                  q_flux_data->getPointer(0, d),
                                  q_flux_data->getPointer(1, d),
                                  q_flux_data_gcw.min(),
                                  patch_lower(0),
                                  patch_upper(0),
                                  patch_lower(1),
                                  patch_upper(1),
#endif
#if (NDIM == 3)
                                  q_flux_data->getPointer(0, d),
                                  q_flux_data->getPointer(1, d),
                                  q_flux_data->getPointer(2, d),
                                  q_flux_data_gcw.min(),
                                  patch_lower(0),
                                  patch_upper(0),
                                  patch_lower(1),
                                  patch_upper(1),
                                  patch_lower(2),
                                  patch_upper(2),
#endif
                                  dx);
                }
            }

            if (d_difference_form == SKEW_SYMMETRIC)
            {
                Pointer<FaceData<NDIM, double> > q_flux_data = patch->getPatchData(d_q_flux_idx);
                const IntVector<NDIM>& q_flux_data_gcw = q_flux_data->getGhostCellWidth();
                for (unsigned int d = 0; d < d_Q_data_depth; ++d)
                {
                    static const double alpha = 0.5;
                    static const double beta = 0.5;
                    F_TO_C_DIV_ADD_FC(N_data->getPointer(d),
                                      N_data_gcw.min(),
                                      alpha,
#if (NDIM == 2)
                                      q_flux_data->getPointer(0, d),
                                      q_flux_data->getPointer(1, d),
                                      q_flux_data_gcw.min(),
                                      beta,
                                      N_data->getPointer(d),
                                      N_data_gcw.min(),
                                      patch_lower(0),
                                      patch_upper(0),
                                      patch_lower(1),
                                      patch_upper(1),
#endif
#if (NDIM == 3)
                                      q_flux_data->getPointer(0, d),
                                      q_flux_data->getPointer(1, d),
                                      q_flux_data->getPointer(2, d),
                                      q_flux_data_gcw.min(),
                                      beta,
                                      N_data->getPointer(d),
                                      N_data_gcw.min(),
                                      patch_lower(0),
                                      patch_upper(0),
                                      patch_lower(1),
                                      patch_upper(1),
                                      patch_lower(2),
                                      patch_upper(2),
#endif
                                      dx);
                }
            }
        }
    }

    // Deallocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->deallocatePatchData(d_Q_scratch_idx);
        level->deallocatePatchData(d_q_extrap_idx);
        if (d_difference_form == CONSERVATIVE || d_difference_form == SKEW_SYMMETRIC)
            level->deallocatePatchData(d_q_flux_idx);
    }

    IBAMR_TIMER_STOP(t_apply_convective_operator);
    return;
} // applyConvectiveOperator
void
IBImplicitModHelmholtzPETScLevelSolver::initializeSolverState(
    const SAMRAIVectorReal<NDIM,double>& x,
    const SAMRAIVectorReal<NDIM,double>& b)
{
    IBAMR_TIMER_START(t_initialize_solver_state);

    // Rudimentary error checking.
#ifdef DEBUG_CHECK_ASSERTIONS
    if (x.getNumberOfComponents() != b.getNumberOfComponents())
    {
        TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                   << "  vectors must have the same number of components" << std::endl);
    }

    const Pointer<PatchHierarchy<NDIM> >& patch_hierarchy = x.getPatchHierarchy();
    if (patch_hierarchy != b.getPatchHierarchy())
    {
        TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                   << "  vectors must have the same hierarchy" << std::endl);
    }

    const int coarsest_ln = x.getCoarsestLevelNumber();
    if (coarsest_ln < 0)
    {
        TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                   << "  coarsest level number must not be negative" << std::endl);
    }
    if (coarsest_ln != b.getCoarsestLevelNumber())
    {
        TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                   << "  vectors must have same coarsest level number" << std::endl);
    }

    const int finest_ln = x.getFinestLevelNumber();
    if (finest_ln < coarsest_ln)
    {
        TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                   << "  finest level number must be >= coarsest level number" << std::endl);
    }
    if (finest_ln != b.getFinestLevelNumber())
    {
        TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                   << "  vectors must have same finest level number" << std::endl);
    }

    for (int ln = coarsest_ln; ln <= finest_ln; ++ln)
    {
        if (patch_hierarchy->getPatchLevel(ln).isNull())
        {
            TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                       << "  hierarchy level " << ln << " does not exist" << std::endl);
        }
    }

    if (coarsest_ln != finest_ln)
    {
        TBOX_ERROR(d_object_name << "::initializeSolverState()\n"
                   << "  coarsest_ln != finest_ln in IBImplicitModHelmholtzPETScLevelSolver" << std::endl);
    }
#endif
    // Deallocate the solver state if the solver is already initialized.
    if (d_is_initialized) deallocateSolverState();

    // Get the hierarchy information.
    d_hierarchy = x.getPatchHierarchy();
    d_level_num = x.getCoarsestLevelNumber();
#ifdef DEBUG_CHECK_ASSERTIONS
    TBOX_ASSERT(d_level_num == x.getFinestLevelNumber());
#endif

    const int x_idx = x.getComponentDescriptorIndex(0);
    Pointer<SideVariable<NDIM,double> > x_var = x.getComponentVariable(0);
    const int b_idx = b.getComponentDescriptorIndex(0);
    Pointer<SideVariable<NDIM,double> > b_var = b.getComponentVariable(0);

    // Allocate DOF index data.
    VariableDatabase<NDIM>* var_db = VariableDatabase<NDIM>::getDatabase();
    Pointer<SideDataFactory<NDIM,double> > x_fac =
        var_db->getPatchDescriptor()->getPatchDataFactory(x_idx);
    const int depth = x_fac->getDefaultDepth();
    Pointer<SideDataFactory<NDIM,int> > dof_index_fac =
        var_db->getPatchDescriptor()->getPatchDataFactory(d_dof_index_idx);
    dof_index_fac->setDefaultDepth(depth);
    Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(d_level_num);
    if (!level->checkAllocated(d_dof_index_idx)) level->allocatePatchData(d_dof_index_idx);

    // Setup PETSc objects.
    int ierr;
    PETScVecUtilities::constructPatchLevelVec(d_petsc_x, x_idx, x_var, level);
    PETScVecUtilities::constructPatchLevelVec(d_petsc_b, b_idx, b_var, level);
    PETScVecUtilities::constructPatchLevelDOFIndices(d_dof_index_idx, d_dof_index_var, x_idx, x_var, level);
    const double C = d_poisson_spec.cIsZero() ? 0.0 : d_poisson_spec.getCConstant();
    const double D = d_poisson_spec.getDConstant();
    PETScMatUtilities::constructPatchLevelLaplaceOp(d_petsc_mat, C, D, x_idx, x_var, d_dof_index_idx, d_dof_index_var, level, d_dof_index_fill);
    if (d_SJR_mat != PETSC_NULL)
    {
        ierr = PETScMatOps::MatAXPY(d_petsc_mat, 1.0, d_SJR_mat); IBTK_CHKERRQ(ierr);
    }
    ierr = MatSetBlockSize(d_petsc_mat, NDIM); IBTK_CHKERRQ(ierr);

    ierr = KSPCreate(PETSC_COMM_WORLD, &d_petsc_ksp); IBTK_CHKERRQ(ierr);
    ierr = KSPSetOperators(d_petsc_ksp, d_petsc_mat, d_petsc_mat, SAME_PRECONDITIONER); IBTK_CHKERRQ(ierr);
    if (!d_options_prefix.empty())
    {
        ierr = KSPSetOptionsPrefix(d_petsc_ksp, d_options_prefix.c_str()); IBTK_CHKERRQ(ierr);
    }
    ierr = KSPSetFromOptions(d_petsc_ksp); IBTK_CHKERRQ(ierr);

    // Indicate that the solver is initialized.
    d_is_initialized = true;

    IBAMR_TIMER_STOP(t_initialize_solver_state);
    return;
}// initializeSolverState
void INSStaggeredCenteredConvectiveOperator::applyConvectiveOperator(const int U_idx,
                                                                     const int N_idx)
{
    IBAMR_TIMER_START(t_apply_convective_operator);
#if !defined(NDEBUG)
    if (!d_is_initialized)
    {
        TBOX_ERROR(
            "INSStaggeredCenteredConvectiveOperator::applyConvectiveOperator():\n"
            << "  operator must be initialized prior to call to applyConvectiveOperator\n");
    }
    TBOX_ASSERT(U_idx == d_u_idx);
#endif

    // Fill ghost cell values for all components.
    static const bool homogeneous_bc = false;
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent
    InterpolationTransactionComponent;
    std::vector<InterpolationTransactionComponent> transaction_comps(1);
    transaction_comps[0] = InterpolationTransactionComponent(d_U_scratch_idx,
                                                             U_idx,
                                                             "CONSERVATIVE_LINEAR_REFINE",
                                                             false,
                                                             "CONSERVATIVE_COARSEN",
                                                             d_bdry_extrap_type,
                                                             false,
                                                             d_bc_coefs);
    d_hier_bdry_fill->resetTransactionComponents(transaction_comps);
    d_hier_bdry_fill->setHomogeneousBc(homogeneous_bc);
    StaggeredStokesPhysicalBoundaryHelper::setupBcCoefObjects(
        d_bc_coefs, NULL, d_U_scratch_idx, -1, homogeneous_bc);
    d_hier_bdry_fill->fillData(d_solution_time);
    StaggeredStokesPhysicalBoundaryHelper::resetBcCoefObjects(d_bc_coefs, NULL);
    //  d_bc_helper->enforceDivergenceFreeConditionAtBoundary(d_U_scratch_idx);
    d_hier_bdry_fill->resetTransactionComponents(d_transaction_comps);

    // Compute the convective derivative.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        for (PatchLevel<NDIM>::Iterator p(level); p; p++)
        {
            Pointer<Patch<NDIM> > patch = level->getPatch(p());

            const Pointer<CartesianPatchGeometry<NDIM> > patch_geom =
                patch->getPatchGeometry();
            const double* const dx = patch_geom->getDx();

            const Box<NDIM>& patch_box = patch->getBox();
            const IntVector<NDIM>& patch_lower = patch_box.lower();
            const IntVector<NDIM>& patch_upper = patch_box.upper();

            Pointer<SideData<NDIM, double> > N_data = patch->getPatchData(N_idx);
            Pointer<SideData<NDIM, double> > U_data = patch->getPatchData(d_U_scratch_idx);

            const IntVector<NDIM>& N_data_gcw = N_data->getGhostCellWidth();
            const IntVector<NDIM>& U_data_gcw = U_data->getGhostCellWidth();

            switch (d_difference_form)
            {
            case CONSERVATIVE:
                NAVIER_STOKES_STAGGERED_DIV_DERIVATIVE_FC(dx,
#if (NDIM == 2)
                                                          patch_lower(0),
                                                          patch_upper(0),
                                                          patch_lower(1),
                                                          patch_upper(1),
                                                          U_data_gcw(0),
                                                          U_data_gcw(1),
                                                          U_data->getPointer(0),
                                                          U_data->getPointer(1),
                                                          N_data_gcw(0),
                                                          N_data_gcw(1),
                                                          N_data->getPointer(0),
                                                          N_data->getPointer(1)
#endif
#if (NDIM == 3)
                                                          patch_lower(0),
                                                          patch_upper(0),
                                                          patch_lower(1),
                                                          patch_upper(1),
                                                          patch_lower(2),
                                                          patch_upper(2),
                                                          U_data_gcw(0),
                                                          U_data_gcw(1),
                                                          U_data_gcw(2),
                                                          U_data->getPointer(0),
                                                          U_data->getPointer(1),
                                                          U_data->getPointer(2),
                                                          N_data_gcw(0),
                                                          N_data_gcw(1),
                                                          N_data_gcw(2),
                                                          N_data->getPointer(0),
                                                          N_data->getPointer(1),
                                                          N_data->getPointer(2)
#endif
                                                          );
                break;
            case ADVECTIVE:
                NAVIER_STOKES_STAGGERED_ADV_DERIVATIVE_FC(dx,
#if (NDIM == 2)
                                                          patch_lower(0),
                                                          patch_upper(0),
                                                          patch_lower(1),
                                                          patch_upper(1),
                                                          U_data_gcw(0),
                                                          U_data_gcw(1),
                                                          U_data->getPointer(0),
                                                          U_data->getPointer(1),
                                                          N_data_gcw(0),
                                                          N_data_gcw(1),
                                                          N_data->getPointer(0),
                                                          N_data->getPointer(1)
#endif
#if (NDIM == 3)
                                                          patch_lower(0),
                                                          patch_upper(0),
                                                          patch_lower(1),
                                                          patch_upper(1),
                                                          patch_lower(2),
                                                          patch_upper(2),
                                                          U_data_gcw(0),
                                                          U_data_gcw(1),
                                                          U_data_gcw(2),
                                                          U_data->getPointer(0),
                                                          U_data->getPointer(1),
                                                          U_data->getPointer(2),
                                                          N_data_gcw(0),
                                                          N_data_gcw(1),
                                                          N_data_gcw(2),
                                                          N_data->getPointer(0),
                                                          N_data->getPointer(1),
                                                          N_data->getPointer(2)
#endif
                                                          );
                break;
            case SKEW_SYMMETRIC:
                NAVIER_STOKES_STAGGERED_SKEW_SYM_DERIVATIVE_FC(dx,
#if (NDIM == 2)
                                                               patch_lower(0),
                                                               patch_upper(0),
                                                               patch_lower(1),
                                                               patch_upper(1),
                                                               U_data_gcw(0),
                                                               U_data_gcw(1),
                                                               U_data->getPointer(0),
                                                               U_data->getPointer(1),
                                                               N_data_gcw(0),
                                                               N_data_gcw(1),
                                                               N_data->getPointer(0),
                                                               N_data->getPointer(1)
#endif
#if (NDIM == 3)
                                                               patch_lower(0),
                                                               patch_upper(0),
                                                               patch_lower(1),
                                                               patch_upper(1),
                                                               patch_lower(2),
                                                               patch_upper(2),
                                                               U_data_gcw(0),
                                                               U_data_gcw(1),
                                                               U_data_gcw(2),
                                                               U_data->getPointer(0),
                                                               U_data->getPointer(1),
                                                               U_data->getPointer(2),
                                                               N_data_gcw(0),
                                                               N_data_gcw(1),
                                                               N_data_gcw(2),
                                                               N_data->getPointer(0),
                                                               N_data->getPointer(1),
                                                               N_data->getPointer(2)
#endif
                                                               );
                break;
            default:
                TBOX_ERROR(
                    "INSStaggeredCenteredConvectiveOperator::applyConvectiveOperator():\n"
                    << "  unsupported differencing form: "
                    << enum_to_string<ConvectiveDifferencingType>(d_difference_form) << " \n"
                    << "  valid choices are: ADVECTIVE, CONSERVATIVE, SKEW_SYMMETRIC\n");
            }
        }
    }

    IBAMR_TIMER_STOP(t_apply_convective_operator);
    return;
} // applyConvectiveOperator
Beispiel #25
0
void
HierarchyProjector::resetHierarchyConfiguration(
    const Pointer<BasePatchHierarchy<NDIM> > hierarchy,
    const int coarsest_level,
    const int finest_level)
{
    IBAMR_TIMER_START(t_reset_hierarchy_configuration);

#ifdef DEBUG_CHECK_ASSERTIONS
    TBOX_ASSERT(!hierarchy.isNull());
    TBOX_ASSERT((coarsest_level >= 0)
                && (coarsest_level <= finest_level)
                && (finest_level <= hierarchy->getFinestLevelNumber()));
    for (int ln = 0; ln <= finest_level; ++ln)
    {
        TBOX_ASSERT(!(hierarchy->getPatchLevel(ln)).isNull());
    }
#endif
    const int finest_hier_level = hierarchy->getFinestLevelNumber();

    // Reset the Hierarchy data operations for the new hierarchy configuration.
    d_hier_cc_data_ops->setPatchHierarchy(hierarchy);
    d_hier_cc_data_ops->resetLevels(0, finest_hier_level);

    d_hier_fc_data_ops->setPatchHierarchy(hierarchy);
    d_hier_fc_data_ops->resetLevels(0, finest_hier_level);

    d_hier_sc_data_ops->setPatchHierarchy(hierarchy);
    d_hier_sc_data_ops->resetLevels(0, finest_hier_level);

    // Reset the Hierarchy math operations for the new configuration.
    if (d_is_managing_hier_math_ops)
    {
        d_hier_math_ops->setPatchHierarchy(hierarchy);
        d_hier_math_ops->resetLevels(0, finest_hier_level);
    }
    else if (d_hier_math_ops.isNull())
    {
        d_hier_math_ops = new HierarchyMathOps(d_object_name+"::HierarchyMathOps", hierarchy);
        d_is_managing_hier_math_ops = true;
    }

    // Get the cell weights data.
    d_wgt_var = d_hier_math_ops->getCellWeightVariable();
    d_wgt_idx = d_hier_math_ops->getCellWeightPatchDescriptorIndex();

    // Get the volume of the physical domain.
    d_volume = d_hier_math_ops->getVolumeOfPhysicalDomain();

    // Reset the solution and rhs vectors.
    d_sol_vec = new SAMRAIVectorReal<NDIM,double>(
        d_object_name+"::sol_vec", d_hierarchy, 0, finest_hier_level);
    d_sol_vec->addComponent(d_sol_var,d_sol_idx,d_wgt_idx,d_hier_cc_data_ops);

    d_rhs_vec = new SAMRAIVectorReal<NDIM,double>(
        d_object_name+"::rhs_vec", d_hierarchy, 0, finest_hier_level);
    d_rhs_vec->addComponent(d_rhs_var,d_rhs_idx,d_wgt_idx,d_hier_cc_data_ops);

    // (Re)-initialize the Poisson solver.
    d_laplace_op->setHierarchyMathOps(d_hier_math_ops);
    d_poisson_fac_op->setResetLevels(coarsest_level, finest_level);
    d_poisson_solver->initializeSolverState(*d_sol_vec,*d_rhs_vec);

    // Initialize the interpolation operators.
    Pointer<VariableFillPattern<NDIM> > cc_fill_pattern = new CellNoCornersFillPattern(CELLG, false, false, true);
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;

    InterpolationTransactionComponent P_transaction_comp(d_P_idx, DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY, NULL, cc_fill_pattern);
    d_P_hier_bdry_fill_op = new HierarchyGhostCellInterpolation();
    d_P_hier_bdry_fill_op->initializeOperatorState(P_transaction_comp, d_hierarchy);

    InterpolationTransactionComponent Phi_transaction_comp(d_F_idx, DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY, d_Phi_bc_coef, cc_fill_pattern);
    d_Phi_hier_bdry_fill_op = new HierarchyGhostCellInterpolation();
    d_Phi_hier_bdry_fill_op->initializeOperatorState(Phi_transaction_comp, d_hierarchy);

    // (Re)build refine communication schedules.  These are created for all
    // levels in the hierarchy.
    d_fc_velocity_rscheds.resize(finest_hier_level+1);
    d_sc_velocity_rscheds.resize(finest_hier_level+1);
    for (int ln = coarsest_level; ln <= finest_hier_level; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = hierarchy->getPatchLevel(ln);
        d_fc_velocity_rscheds[ln] = d_fc_velocity_ralg->createSchedule(
            level, ln-1, hierarchy, d_fc_velocity_rstrategy);
        d_sc_velocity_rscheds[ln] = d_sc_velocity_ralg->createSchedule(
            level, ln-1, hierarchy, d_sc_velocity_rstrategy);
    }

    IBAMR_TIMER_STOP(t_reset_hierarchy_configuration);
    return;
}// resetHierarchyConfiguration
void StaggeredStokesOperator::initializeOperatorState(const SAMRAIVectorReal<NDIM, double>& in,
                                                      const SAMRAIVectorReal<NDIM, double>& out)
{
    IBAMR_TIMER_START(t_initialize_operator_state);

    // Deallocate the operator state if the operator is already initialized.
    if (d_is_initialized) deallocateOperatorState();

    // Setup solution and rhs vectors.
    d_x = in.cloneVector(in.getName());
    d_b = out.cloneVector(out.getName());
    d_x->allocateVectorData();

    // Setup the interpolation transaction information.
    d_U_fill_pattern = new SideNoCornersFillPattern(SIDEG, false, false, true);
    d_P_fill_pattern = new CellNoCornersFillPattern(CELLG, false, false, true);
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;
    d_transaction_comps.resize(2);
    d_transaction_comps[0] = InterpolationTransactionComponent(d_x->getComponentDescriptorIndex(0),
                                                               in.getComponentDescriptorIndex(0),
                                                               DATA_REFINE_TYPE,
                                                               USE_CF_INTERPOLATION,
                                                               DATA_COARSEN_TYPE,
                                                               BDRY_EXTRAP_TYPE,
                                                               CONSISTENT_TYPE_2_BDRY,
                                                               d_U_bc_coefs,
                                                               d_U_fill_pattern);
    d_transaction_comps[1] = InterpolationTransactionComponent(in.getComponentDescriptorIndex(1),
                                                               DATA_REFINE_TYPE,
                                                               USE_CF_INTERPOLATION,
                                                               DATA_COARSEN_TYPE,
                                                               BDRY_EXTRAP_TYPE,
                                                               CONSISTENT_TYPE_2_BDRY,
                                                               d_P_bc_coef,
                                                               d_P_fill_pattern);

    // Initialize the interpolation operators.
    d_hier_bdry_fill = new HierarchyGhostCellInterpolation();
    d_hier_bdry_fill->initializeOperatorState(d_transaction_comps, d_x->getPatchHierarchy());

    // Initialize hierarchy math ops object.
    if (!d_hier_math_ops_external)
    {
        d_hier_math_ops = new HierarchyMathOps(d_object_name + "::HierarchyMathOps",
                                               in.getPatchHierarchy(),
                                               in.getCoarsestLevelNumber(),
                                               in.getFinestLevelNumber());
    }
#if !defined(NDEBUG)
    else
    {
        TBOX_ASSERT(d_hier_math_ops);
    }
#endif

    // Indicate the operator is initialized.
    d_is_initialized = true;

    IBAMR_TIMER_STOP(t_initialize_operator_state);
    return;
} // initializeOperatorState
void
INSStaggeredUpwindConvectiveOperator::applyConvectiveOperator(const int U_idx, const int N_idx)
{
    IBAMR_TIMER_START(t_apply_convective_operator);
#if !defined(NDEBUG)
    if (!d_is_initialized)
    {
        TBOX_ERROR("INSStaggeredUpwindConvectiveOperator::applyConvectiveOperator():\n"
                   << "  operator must be initialized prior to call to applyConvectiveOperator\n");
    }
    TBOX_ASSERT(U_idx == d_u_idx);
#endif

    // Allocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->allocatePatchData(d_U_scratch_idx);
    }

    // Fill ghost cell values for all components.
    static const bool homogeneous_bc = false;
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;
    std::vector<InterpolationTransactionComponent> transaction_comps(1);
    transaction_comps[0] = InterpolationTransactionComponent(d_U_scratch_idx,
                                                             U_idx,
                                                             "CONSTANT_REFINE",
                                                             false,
                                                             "CONSERVATIVE_COARSEN",
                                                             d_bdry_extrap_type,
                                                             false,
                                                             d_bc_coefs);
    d_hier_bdry_fill->resetTransactionComponents(transaction_comps);
    d_hier_bdry_fill->setHomogeneousBc(homogeneous_bc);
    StaggeredStokesPhysicalBoundaryHelper::setupBcCoefObjects(d_bc_coefs, NULL, d_U_scratch_idx, -1, homogeneous_bc);
    d_hier_bdry_fill->fillData(d_solution_time);
    StaggeredStokesPhysicalBoundaryHelper::resetBcCoefObjects(d_bc_coefs, NULL);
    d_hier_bdry_fill->resetTransactionComponents(d_transaction_comps);

    // Compute the convective derivative.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        for (PatchLevel<NDIM>::Iterator p(level); p; p++)
        {
            Pointer<Patch<NDIM> > patch = level->getPatch(p());

            const Pointer<CartesianPatchGeometry<NDIM> > patch_geom = patch->getPatchGeometry();
            const double* const dx = patch_geom->getDx();

            const Box<NDIM>& patch_box = patch->getBox();
            const IntVector<NDIM>& patch_lower = patch_box.lower();
            const IntVector<NDIM>& patch_upper = patch_box.upper();

            Pointer<SideData<NDIM, double> > N_data = patch->getPatchData(N_idx);
            Pointer<SideData<NDIM, double> > U_data = patch->getPatchData(d_U_scratch_idx);

            const IntVector<NDIM> ghosts = IntVector<NDIM>(1);
            boost::array<Box<NDIM>, NDIM> side_boxes;
            boost::array<Pointer<FaceData<NDIM, double> >, NDIM> U_adv_data;
            boost::array<Pointer<FaceData<NDIM, double> >, NDIM> U_half_data;
            for (unsigned int axis = 0; axis < NDIM; ++axis)
            {
                side_boxes[axis] = SideGeometry<NDIM>::toSideBox(patch_box, axis);
                U_adv_data[axis] = new FaceData<NDIM, double>(side_boxes[axis], 1, ghosts);
                U_half_data[axis] = new FaceData<NDIM, double>(side_boxes[axis], 1, ghosts);
            }
#if (NDIM == 2)
            NAVIER_STOKES_INTERP_COMPS_FC(patch_lower(0),
                                          patch_upper(0),
                                          patch_lower(1),
                                          patch_upper(1),
                                          U_data->getGhostCellWidth()(0),
                                          U_data->getGhostCellWidth()(1),
                                          U_data->getPointer(0),
                                          U_data->getPointer(1),
                                          side_boxes[0].lower(0),
                                          side_boxes[0].upper(0),
                                          side_boxes[0].lower(1),
                                          side_boxes[0].upper(1),
                                          U_adv_data[0]->getGhostCellWidth()(0),
                                          U_adv_data[0]->getGhostCellWidth()(1),
                                          U_adv_data[0]->getPointer(0),
                                          U_adv_data[0]->getPointer(1),
                                          side_boxes[1].lower(0),
                                          side_boxes[1].upper(0),
                                          side_boxes[1].lower(1),
                                          side_boxes[1].upper(1),
                                          U_adv_data[1]->getGhostCellWidth()(0),
                                          U_adv_data[1]->getGhostCellWidth()(1),
                                          U_adv_data[1]->getPointer(0),
                                          U_adv_data[1]->getPointer(1));
#endif
#if (NDIM == 3)
            NAVIER_STOKES_INTERP_COMPS_FC(patch_lower(0),
                                          patch_upper(0),
                                          patch_lower(1),
                                          patch_upper(1),
                                          patch_lower(2),
                                          patch_upper(2),
                                          U_data->getGhostCellWidth()(0),
                                          U_data->getGhostCellWidth()(1),
                                          U_data->getGhostCellWidth()(2),
                                          U_data->getPointer(0),
                                          U_data->getPointer(1),
                                          U_data->getPointer(2),
                                          side_boxes[0].lower(0),
                                          side_boxes[0].upper(0),
                                          side_boxes[0].lower(1),
                                          side_boxes[0].upper(1),
                                          side_boxes[0].lower(2),
                                          side_boxes[0].upper(2),
                                          U_adv_data[0]->getGhostCellWidth()(0),
                                          U_adv_data[0]->getGhostCellWidth()(1),
                                          U_adv_data[0]->getGhostCellWidth()(2),
                                          U_adv_data[0]->getPointer(0),
                                          U_adv_data[0]->getPointer(1),
                                          U_adv_data[0]->getPointer(2),
                                          side_boxes[1].lower(0),
                                          side_boxes[1].upper(0),
                                          side_boxes[1].lower(1),
                                          side_boxes[1].upper(1),
                                          side_boxes[1].lower(2),
                                          side_boxes[1].upper(2),
                                          U_adv_data[1]->getGhostCellWidth()(0),
                                          U_adv_data[1]->getGhostCellWidth()(1),
                                          U_adv_data[1]->getGhostCellWidth()(2),
                                          U_adv_data[1]->getPointer(0),
                                          U_adv_data[1]->getPointer(1),
                                          U_adv_data[1]->getPointer(2),
                                          side_boxes[2].lower(0),
                                          side_boxes[2].upper(0),
                                          side_boxes[2].lower(1),
                                          side_boxes[2].upper(1),
                                          side_boxes[2].lower(2),
                                          side_boxes[2].upper(2),
                                          U_adv_data[2]->getGhostCellWidth()(0),
                                          U_adv_data[2]->getGhostCellWidth()(1),
                                          U_adv_data[2]->getGhostCellWidth()(2),
                                          U_adv_data[2]->getPointer(0),
                                          U_adv_data[2]->getPointer(1),
                                          U_adv_data[2]->getPointer(2));
#endif
            for (unsigned int axis = 0; axis < NDIM; ++axis)
            {
                const ArrayData<NDIM, double>& U_array_data = U_data->getArrayData(axis);
                for (unsigned int d = 0; d < NDIM; ++d)
                {
                    for (FaceIterator<NDIM> ic(side_boxes[axis], d); ic; ic++)
                    {
                        const FaceIndex<NDIM>& i = ic();
                        const double u_ADV = (*U_adv_data[axis])(i);
                        const double U_lower = U_array_data(i.toCell(0), 0);
                        const double U_upper = U_array_data(i.toCell(1), 0);
                        (*U_half_data[axis])(i) =
                            (u_ADV > 1.0e-8) ? U_lower : (u_ADV < 1.0e-8) ? U_upper : 0.5 * (U_lower + U_upper);
                    }
                }
            }
            for (unsigned int axis = 0; axis < NDIM; ++axis)
            {
                switch (d_difference_form)
                {
                case CONSERVATIVE:
#if (NDIM == 2)
                    CONVECT_DERIVATIVE_FC(dx,
                                          side_boxes[axis].lower(0),
                                          side_boxes[axis].upper(0),
                                          side_boxes[axis].lower(1),
                                          side_boxes[axis].upper(1),
                                          U_adv_data[axis]->getGhostCellWidth()(0),
                                          U_adv_data[axis]->getGhostCellWidth()(1),
                                          U_half_data[axis]->getGhostCellWidth()(0),
                                          U_half_data[axis]->getGhostCellWidth()(1),
                                          U_adv_data[axis]->getPointer(0),
                                          U_adv_data[axis]->getPointer(1),
                                          U_half_data[axis]->getPointer(0),
                                          U_half_data[axis]->getPointer(1),
                                          N_data->getGhostCellWidth()(0),
                                          N_data->getGhostCellWidth()(1),
                                          N_data->getPointer(axis));
#endif
#if (NDIM == 3)
                    CONVECT_DERIVATIVE_FC(dx,
                                          side_boxes[axis].lower(0),
                                          side_boxes[axis].upper(0),
                                          side_boxes[axis].lower(1),
                                          side_boxes[axis].upper(1),
                                          side_boxes[axis].lower(2),
                                          side_boxes[axis].upper(2),
                                          U_adv_data[axis]->getGhostCellWidth()(0),
                                          U_adv_data[axis]->getGhostCellWidth()(1),
                                          U_adv_data[axis]->getGhostCellWidth()(2),
                                          U_half_data[axis]->getGhostCellWidth()(0),
                                          U_half_data[axis]->getGhostCellWidth()(1),
                                          U_half_data[axis]->getGhostCellWidth()(2),
                                          U_adv_data[axis]->getPointer(0),
                                          U_adv_data[axis]->getPointer(1),
                                          U_adv_data[axis]->getPointer(2),
                                          U_half_data[axis]->getPointer(0),
                                          U_half_data[axis]->getPointer(1),
                                          U_half_data[axis]->getPointer(2),
                                          N_data->getGhostCellWidth()(0),
                                          N_data->getGhostCellWidth()(1),
                                          N_data->getGhostCellWidth()(2),
                                          N_data->getPointer(axis));
#endif
                    break;
                case ADVECTIVE:
#if (NDIM == 2)
                    ADVECT_DERIVATIVE_FC(dx,
                                         side_boxes[axis].lower(0),
                                         side_boxes[axis].upper(0),
                                         side_boxes[axis].lower(1),
                                         side_boxes[axis].upper(1),
                                         U_adv_data[axis]->getGhostCellWidth()(0),
                                         U_adv_data[axis]->getGhostCellWidth()(1),
                                         U_half_data[axis]->getGhostCellWidth()(0),
                                         U_half_data[axis]->getGhostCellWidth()(1),
                                         U_adv_data[axis]->getPointer(0),
                                         U_adv_data[axis]->getPointer(1),
                                         U_half_data[axis]->getPointer(0),
                                         U_half_data[axis]->getPointer(1),
                                         N_data->getGhostCellWidth()(0),
                                         N_data->getGhostCellWidth()(1),
                                         N_data->getPointer(axis));
#endif
#if (NDIM == 3)
                    ADVECT_DERIVATIVE_FC(dx,
                                         side_boxes[axis].lower(0),
                                         side_boxes[axis].upper(0),
                                         side_boxes[axis].lower(1),
                                         side_boxes[axis].upper(1),
                                         side_boxes[axis].lower(2),
                                         side_boxes[axis].upper(2),
                                         U_adv_data[axis]->getGhostCellWidth()(0),
                                         U_adv_data[axis]->getGhostCellWidth()(1),
                                         U_adv_data[axis]->getGhostCellWidth()(2),
                                         U_half_data[axis]->getGhostCellWidth()(0),
                                         U_half_data[axis]->getGhostCellWidth()(1),
                                         U_half_data[axis]->getGhostCellWidth()(2),
                                         U_adv_data[axis]->getPointer(0),
                                         U_adv_data[axis]->getPointer(1),
                                         U_adv_data[axis]->getPointer(2),
                                         U_half_data[axis]->getPointer(0),
                                         U_half_data[axis]->getPointer(1),
                                         U_half_data[axis]->getPointer(2),
                                         N_data->getGhostCellWidth()(0),
                                         N_data->getGhostCellWidth()(1),
                                         N_data->getGhostCellWidth()(2),
                                         N_data->getPointer(axis));
#endif
                    break;
                case SKEW_SYMMETRIC:
#if (NDIM == 2)
                    SKEW_SYM_DERIVATIVE_FC(dx,
                                           side_boxes[axis].lower(0),
                                           side_boxes[axis].upper(0),
                                           side_boxes[axis].lower(1),
                                           side_boxes[axis].upper(1),
                                           U_adv_data[axis]->getGhostCellWidth()(0),
                                           U_adv_data[axis]->getGhostCellWidth()(1),
                                           U_half_data[axis]->getGhostCellWidth()(0),
                                           U_half_data[axis]->getGhostCellWidth()(1),
                                           U_adv_data[axis]->getPointer(0),
                                           U_adv_data[axis]->getPointer(1),
                                           U_half_data[axis]->getPointer(0),
                                           U_half_data[axis]->getPointer(1),
                                           N_data->getGhostCellWidth()(0),
                                           N_data->getGhostCellWidth()(1),
                                           N_data->getPointer(axis));
#endif
#if (NDIM == 3)
                    SKEW_SYM_DERIVATIVE_FC(dx,
                                           side_boxes[axis].lower(0),
                                           side_boxes[axis].upper(0),
                                           side_boxes[axis].lower(1),
                                           side_boxes[axis].upper(1),
                                           side_boxes[axis].lower(2),
                                           side_boxes[axis].upper(2),
                                           U_adv_data[axis]->getGhostCellWidth()(0),
                                           U_adv_data[axis]->getGhostCellWidth()(1),
                                           U_adv_data[axis]->getGhostCellWidth()(2),
                                           U_half_data[axis]->getGhostCellWidth()(0),
                                           U_half_data[axis]->getGhostCellWidth()(1),
                                           U_half_data[axis]->getGhostCellWidth()(2),
                                           U_adv_data[axis]->getPointer(0),
                                           U_adv_data[axis]->getPointer(1),
                                           U_adv_data[axis]->getPointer(2),
                                           U_half_data[axis]->getPointer(0),
                                           U_half_data[axis]->getPointer(1),
                                           U_half_data[axis]->getPointer(2),
                                           N_data->getGhostCellWidth()(0),
                                           N_data->getGhostCellWidth()(1),
                                           N_data->getGhostCellWidth()(2),
                                           N_data->getPointer(axis));
#endif
                    break;
                default:
                    TBOX_ERROR("INSStaggeredUpwindConvectiveOperator::applyConvectiveOperator():\n"
                               << "  unsupported differencing form: "
                               << enum_to_string<ConvectiveDifferencingType>(d_difference_form)
                               << " \n"
                               << "  valid choices are: ADVECTIVE, CONSERVATIVE, SKEW_SYMMETRIC\n");
                }
            }
        }
    }

    // Deallocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->deallocatePatchData(d_U_scratch_idx);
    }

    IBAMR_TIMER_STOP(t_apply_convective_operator);
    return;
} // applyConvectiveOperator
Beispiel #28
0
void
HierarchyProjector::projectHierarchy(
    const double rho,
    const double dt,
    const double time,
    const ProjectionMethodType& projection_type,
    const int u_idx,
    const Pointer<SideVariable<NDIM,double> >& /*u_var*/,
    const int P_idx,
    const Pointer<CellVariable<NDIM,double> >& /*P_var*/,
    const int Phi_idx,
    const Pointer<CellVariable<NDIM,double> >& Phi_var,
    const int grad_Phi_idx,
    const Pointer<SideVariable<NDIM,double> >& grad_Phi_var,
    const int w_idx,
    const Pointer<SideVariable<NDIM,double> >& w_var,
    const int Q_idx,
    const Pointer<CellVariable<NDIM,double> >& Q_var)
{
    IBAMR_TIMER_START(t_project_hierarchy);

    const int coarsest_ln = 0;
    const int finest_ln = d_hierarchy->getFinestLevelNumber();

    // Allocate temporary data.
    ComponentSelector scratch_idxs;
    scratch_idxs.setFlag(d_F_idx);
    scratch_idxs.setFlag(d_P_idx);
    scratch_idxs.setFlag(d_w_sc_idx);
    for (int ln = coarsest_ln; ln <= finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->allocatePatchData(scratch_idxs, time);
    }

    // Fill the pressure data if we are using a pressure-increment projection.
    if (projection_type == PRESSURE_INCREMENT)
    {
        d_hier_cc_data_ops->copyData(d_P_idx, P_idx);
        d_P_hier_bdry_fill_op->fillData(time);
    }

    // Fill the intermediate velocity data.
    d_hier_sc_data_ops->copyData(d_w_sc_idx, w_idx);
    for (int ln = coarsest_ln; ln <= finest_ln; ++ln)
    {
        d_sc_velocity_rscheds[ln]->fillData(time);
    }

    // Setup the boundary coefficient specification object.
    //
    // NOTE: These boundary coefficients are also used by the linear operator
    // and by the FAC preconditioner objects associated with this class.
    d_Phi_bc_coef->setProblemCoefs(rho, dt);
    d_Phi_bc_coef->setCurrentPressurePatchDataIndex(d_P_idx);
    d_Phi_bc_coef->setPressurePhysicalBcCoef(d_P_bc_coef);
    d_Phi_bc_coef->setProjectionType(projection_type);
    d_Phi_bc_coef->setIntermediateVelocityPatchDataIndex(d_w_sc_idx);
    d_Phi_bc_coef->setVelocityPhysicalBcCoefs(d_u_bc_coefs);

    // Setup the linear operator.
    d_laplace_op->setTime(time);
    d_laplace_op->setHierarchyMathOps(d_hier_math_ops);

    // Setup the preconditioner.
    d_poisson_fac_op->setTime(time);

    // Compute F = (rho/dt)*(Q - div w).
    const bool w_cf_bdry_synch = true;
    d_hier_math_ops->div(
        d_F_idx, d_F_var,   // dst
        -rho/dt,            // alpha
        w_idx, w_var,       // src1
        d_no_fill_op,       // src1_bdry_fill
        time,               // src1_bdry_fill_time
        w_cf_bdry_synch,    // src1_cf_bdry_synch
        +rho/dt,            // beta
        Q_idx, Q_var     ); // src2

    // Solve -div grad Phi = F = (rho/dt)*(Q - div w).
    SAMRAIVectorReal<NDIM,double> sol_vec(
        d_object_name+"::sol_vec", d_hierarchy, coarsest_ln, finest_ln);
    sol_vec.addComponent(Phi_var, Phi_idx, d_wgt_idx, d_hier_cc_data_ops);

    SAMRAIVectorReal<NDIM,double> rhs_vec(
        d_object_name+"::rhs_vec", d_hierarchy, coarsest_ln, finest_ln);
    rhs_vec.addComponent(d_F_var, d_F_idx, d_wgt_idx, d_hier_cc_data_ops);

    d_poisson_solver->solveSystem(sol_vec,rhs_vec);

    if (d_do_log) plog << "HierarchyProjector::projectHierarchy(): linear solve number of iterations = " << d_poisson_solver->getNumIterations() << "\n";
    if (d_do_log) plog << "HierarchyProjector::projectHierarchy(): linear solve residual norm        = " << d_poisson_solver->getResidualNorm()  << "\n";

    if (d_poisson_solver->getNumIterations() == d_poisson_solver->getMaxIterations())
    {
        pout << d_object_name << "::projectHierarchy():"
             <<"  WARNING: linear solver iterations == max iterations\n";
    }

    // Setup the interpolation transaction information.
    Pointer<VariableFillPattern<NDIM> > cc_fill_pattern = new CellNoCornersFillPattern(CELLG, false, false, true);
    typedef HierarchyGhostCellInterpolation::InterpolationTransactionComponent InterpolationTransactionComponent;
    InterpolationTransactionComponent Phi_transaction_comp(Phi_idx, DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY, d_Phi_bc_coef, cc_fill_pattern);
    d_Phi_hier_bdry_fill_op->resetTransactionComponent(Phi_transaction_comp);

    // Fill the physical boundary conditions for Phi.
    d_Phi_bc_coef->setHomogeneousBc(false);
    d_Phi_hier_bdry_fill_op->setHomogeneousBc(false);
    d_Phi_hier_bdry_fill_op->fillData(time);
    d_Phi_bc_coef->setHomogeneousBc(true);

    // Set u = w - (dt/rho)*grad Phi.
    const bool grad_Phi_cf_bdry_synch = true;
    d_hier_math_ops->grad(
        grad_Phi_idx, grad_Phi_var, // dst
        grad_Phi_cf_bdry_synch,     // dst_cf_bdry_synch
        1.0,                        // alpha
        Phi_idx, Phi_var,           // src
        d_no_fill_op,               // src_bdry_fill
        time              );        // src_bdry_fill_time
    d_hier_sc_data_ops->axpy(u_idx, -dt/rho, grad_Phi_idx, w_idx);

    // Deallocate scratch data.
    for (int ln = coarsest_ln; ln <= finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->deallocatePatchData(scratch_idxs);
    }

    // Reset the transaction components.
    InterpolationTransactionComponent d_P_transaction_comp(d_P_idx, DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY, NULL, cc_fill_pattern);
    d_P_hier_bdry_fill_op->resetTransactionComponent(d_P_transaction_comp);

    InterpolationTransactionComponent d_Phi_transaction_comp(d_F_idx, DATA_COARSEN_TYPE, BDRY_EXTRAP_TYPE, CONSISTENT_TYPE_2_BDRY, d_Phi_bc_coef, cc_fill_pattern);
    d_Phi_hier_bdry_fill_op->resetTransactionComponent(d_Phi_transaction_comp);

    IBAMR_TIMER_STOP(t_project_hierarchy);
    return;
}// projectHierarchy
bool
StaggeredStokesProjectionPreconditioner::solveSystem(SAMRAIVectorReal<NDIM, double>& x,
                                                     SAMRAIVectorReal<NDIM, double>& b)
{
    IBAMR_TIMER_START(t_solve_system);

    // Initialize the solver (if necessary).
    const bool deallocate_at_completion = !d_is_initialized;
    if (!d_is_initialized) initializeSolverState(x, b);

    // Determine whether we are solving a steady-state problem.
    const bool steady_state =
        d_U_problem_coefs.cIsZero() ||
        (d_U_problem_coefs.cIsConstant() && MathUtilities<double>::equalEps(d_U_problem_coefs.getCConstant(), 0.0));

    // Get the vector components.
    const int F_U_idx = b.getComponentDescriptorIndex(0);
    const int F_P_idx = b.getComponentDescriptorIndex(1);

    const Pointer<Variable<NDIM> >& F_U_var = b.getComponentVariable(0);
    const Pointer<Variable<NDIM> >& F_P_var = b.getComponentVariable(1);

    Pointer<SideVariable<NDIM, double> > F_U_sc_var = F_U_var;
    Pointer<CellVariable<NDIM, double> > F_P_cc_var = F_P_var;

    const int U_idx = x.getComponentDescriptorIndex(0);
    const int P_idx = x.getComponentDescriptorIndex(1);

    const Pointer<Variable<NDIM> >& U_var = x.getComponentVariable(0);
    const Pointer<Variable<NDIM> >& P_var = x.getComponentVariable(1);

    Pointer<SideVariable<NDIM, double> > U_sc_var = U_var;
    Pointer<CellVariable<NDIM, double> > P_cc_var = P_var;

    // Setup the component solver vectors.
    Pointer<SAMRAIVectorReal<NDIM, double> > F_U_vec;
    F_U_vec = new SAMRAIVectorReal<NDIM, double>(d_object_name + "::F_U", d_hierarchy, d_coarsest_ln, d_finest_ln);
    F_U_vec->addComponent(F_U_sc_var, F_U_idx, d_velocity_wgt_idx, d_velocity_data_ops);

    Pointer<SAMRAIVectorReal<NDIM, double> > U_vec;
    U_vec = new SAMRAIVectorReal<NDIM, double>(d_object_name + "::U", d_hierarchy, d_coarsest_ln, d_finest_ln);
    U_vec->addComponent(U_sc_var, U_idx, d_velocity_wgt_idx, d_velocity_data_ops);

    Pointer<SAMRAIVectorReal<NDIM, double> > Phi_scratch_vec;
    Phi_scratch_vec =
        new SAMRAIVectorReal<NDIM, double>(d_object_name + "::Phi_scratch", d_hierarchy, d_coarsest_ln, d_finest_ln);
    Phi_scratch_vec->addComponent(d_Phi_var, d_Phi_scratch_idx, d_pressure_wgt_idx, d_pressure_data_ops);

    Pointer<SAMRAIVectorReal<NDIM, double> > F_Phi_vec;
    F_Phi_vec = new SAMRAIVectorReal<NDIM, double>(d_object_name + "::F_Phi", d_hierarchy, d_coarsest_ln, d_finest_ln);
    F_Phi_vec->addComponent(d_F_Phi_var, d_F_Phi_idx, d_pressure_wgt_idx, d_pressure_data_ops);

    Pointer<SAMRAIVectorReal<NDIM, double> > P_vec;
    P_vec = new SAMRAIVectorReal<NDIM, double>(d_object_name + "::P", d_hierarchy, d_coarsest_ln, d_finest_ln);
    P_vec->addComponent(P_cc_var, P_idx, d_pressure_wgt_idx, d_pressure_data_ops);

    // Allocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->allocatePatchData(d_Phi_scratch_idx);
        level->allocatePatchData(d_F_Phi_idx);
    }

    // (1) Solve the velocity sub-problem for an initial approximation to U.
    //
    // U^* := inv(rho/dt - K*mu*L) F_U
    //
    // An approximate Helmholtz solver is used.
    d_velocity_solver->setHomogeneousBc(true);
    LinearSolver* p_velocity_solver = dynamic_cast<LinearSolver*>(d_velocity_solver.getPointer());
    if (p_velocity_solver) p_velocity_solver->setInitialGuessNonzero(false);
    d_velocity_solver->solveSystem(*U_vec, *F_U_vec);

    // (2) Solve the pressure sub-problem.
    //
    // We treat two cases:
    //
    // (i) rho/dt = 0.
    //
    // In this case,
    //
    //    U - U^* + G Phi = 0
    //    -D U = F_P
    //
    // so that
    //
    //    Phi := inv(-L_p) * F_Phi = inv(-L_p) * (-F_P - D U^*)
    //    P   := -K*mu*F_Phi
    //
    // in which L_p = D*G.
    //
    // (ii) rho/dt != 0.
    //
    // In this case,
    //
    //    rho (U - U^*) + G Phi = 0
    //    -D U = F_P
    //
    // so that
    //
    //    Phi := inv(-L_rho) * F_phi = inv(-L_rho) * (-F_P - D U^*)
    //    P   := (1/dt - K*mu*L_rho)*Phi = (1/dt) Phi - K*mu*F_phi
    //
    // in which L_rho = D*(1/rho)*G.
    //
    // Approximate Poisson solvers are used in both cases.
    d_hier_math_ops->div(d_F_Phi_idx,
                         d_F_Phi_var,
                         -1.0,
                         U_idx,
                         U_sc_var,
                         d_no_fill_op,
                         d_new_time,
                         /*cf_bdry_synch*/ true,
                         -1.0,
                         F_P_idx,
                         F_P_cc_var);
    d_pressure_solver->setHomogeneousBc(true);
    LinearSolver* p_pressure_solver = dynamic_cast<LinearSolver*>(d_pressure_solver.getPointer());
    p_pressure_solver->setInitialGuessNonzero(false);
    d_pressure_solver->solveSystem(*Phi_scratch_vec, *F_Phi_vec);
    if (steady_state)
    {
        d_pressure_data_ops->scale(P_idx, -d_U_problem_coefs.getDConstant(), d_F_Phi_idx);
    }
    else
    {
        d_pressure_data_ops->linearSum(
            P_idx, 1.0 / getDt(), d_Phi_scratch_idx, -d_U_problem_coefs.getDConstant(), d_F_Phi_idx);
    }

    // (3) Evaluate U in terms of U^* and Phi.
    //
    // We treat two cases:
    //
    // (i) rho = 0.  In this case,
    //
    //    U = U^* - G Phi
    //
    // (ii) rho != 0.  In this case,
    //
    //    U = U^* - (1.0/rho) G Phi
    double coef;
    if (steady_state)
    {
        coef = -1.0;
    }
    else
    {
        coef = d_P_problem_coefs.getDConstant();
    }
    d_hier_math_ops->grad(U_idx,
                          U_sc_var,
                          /*cf_bdry_synch*/ true,
                          coef,
                          d_Phi_scratch_idx,
                          d_Phi_var,
                          d_Phi_bdry_fill_op,
                          d_pressure_solver->getSolutionTime(),
                          1.0,
                          U_idx,
                          U_sc_var);

    // Account for nullspace vectors.
    correctNullspace(U_vec, P_vec);

    // Deallocate scratch data.
    for (int ln = d_coarsest_ln; ln <= d_finest_ln; ++ln)
    {
        Pointer<PatchLevel<NDIM> > level = d_hierarchy->getPatchLevel(ln);
        level->deallocatePatchData(d_Phi_scratch_idx);
        level->deallocatePatchData(d_F_Phi_idx);
    }

    // Deallocate the solver (if necessary).
    if (deallocate_at_completion) deallocateSolverState();

    IBAMR_TIMER_STOP(t_solve_system);
    return true;
} // solveSystem