void JESPExhaustivePlanner::Plan()
{
    if(DEBUG_EXJESP) {
        cout << "\n---------------------------------"<<endl;
        cout << "Exhaustive JESP  - Plan() started"<<endl;
        cout << "---------------------------------"<<endl;
    }
    double v_best = -DBL_MAX;
    JointPolicyPureVector *jpol = new JointPolicyPureVector(this);
    JointPolicyPureVector *best = new JointPolicyPureVector(this);
    jpol->RandomInitialization();

    if(DEBUG_EXJESP) {
        cout << "joint policy randomly initialized to:";
        jpol->Print();
    }

    int stop = 0;
    size_t nr_non_improving_agents = 0;
    while(nr_non_improving_agents < GetDPOMDPD()->GetNrAgents()
            && stop++ < 1000)
    {
        int agentI = GetNextAgentIndex();
        double v = ExhaustiveBestResponse(jpol, agentI);
        if(v > v_best + 1e-9)
        {
            (*best) = (*jpol);
            if(DEBUG_EXHBR)
            {
                cout << "Plan: new best policy:"<<endl;
                best->Print();
            }
            v_best = v;
            nr_non_improving_agents = 0;
        }
        else
            nr_non_improving_agents++;
    }
    _m_foundPolicy = JPPV_sharedPtr(best);
    _m_expectedRewardFoundPolicy=v_best;


    if(DEBUG_EXJESP) {
        cout << "Exhaustive JESP  - resulting policy:"<<endl;
        cout << "------------------------------------"<<endl;
        best->Print();
    }
}
/* this is the high-level pseudo-code for what happens:
    start with a horizon 0 joint policy - i.e. specifying 0 actions
    JPolValPool.push( <jpol,val=0.0> )    
    do
        ppi = <pol,val> = JPolValPool.GetNext()

        //poolOfNextPolicies     = {<pol,vals>} 
        //isLowerBound  = bool   - whether the vals are lower bounds to the 
        //                optimal value (i.e. value for the optimal policy)
        <poolOfNextPolicies, isLowerBound> = ConstructAndValuateNextPolicies(ppi)
        if(isLowerBound)
            Prune( JPolValPool, max(lowerBound) )

        poolOfNextPolicies = SelectPoliciesToProcessFurther(poolOfNextPolicies);
        JPolValPool.insert(poolOfNextPolicies)
      
    while !empty JPolValPool
*/
void GeneralizedMAAStarPlanner::Plan()    
{
    if( _m_foundPolicy != 0)
        delete _m_foundPolicy;

    StartTimer("GMAA::Plan");

    //stuff for timing (if used)
    tms ts_start;   //the time struct
    clock_t tck_start;  //ticks
    tck_start = times(&ts_start);
    //the intermediate timing stream
    ofstream & its = *_m_intermediateResultFile;

    //this counter maintains the maximum policy pool size.
    _m_maxJPolPoolSize = 0;
    double maxLB = -DBL_MAX;
    PartialJointPolicyDiscretePure * bestJPol = NewJPol();
    bestJPol->SetPastReward(-DBL_MAX);
#if DEBUG_GMAA_POLS    
    cout << "GMAA initialized with empty policy:"<<endl;
    bestJPol->Print();
#endif                 
    PartialPolicyPoolInterface * pp_p = NewPP(); 
    pp_p->Init( GetThisFromMostDerivedPU() ); //initialize with empty joint policy
    do
    {
        StartTimer("GMAA::Plan::iteration");
        if(_m_saveIntermediateTiming)
            SaveTimers(_m_intermediateTimingFilename);

        if(_m_verboseness >= 2) {
            cout << "\n---------------------------------------------------\n";
            cout << "-->>Start of new GMAA iteration, polpool size="<<
                pp_p->Size()<<"<<--"<<endl;
        }

        PartialPolicyPoolItemInterface* ppi = pp_p->Select();
        PartialJointPolicyDiscretePure * jpol_sel =  ppi->GetJPol();
        double v_sel = ppi->GetValue();
        size_t depth_sel = jpol_sel->GetDepth();
        if(_m_verboseness >= 3) {
            cout << "Select returned the following policy to expand:\n";
            ppi->GetJPol()->Print();
            cout << "of depth="<< depth_sel << " and heur. val="<<v_sel<<endl;
        }
        
        if( (v_sel + _m_slack) < maxLB) //the highest upperbound < the best lower
        {
            //TODO: 
            //  1)if JPolValPool is no priority queue, this should be changed.
            //  2)this is only necessary, because PRUNE (see todo below) is nyi
            if(_m_verboseness >= 0)
                cout<<"highest upper < best found lower bound, stopping\n";
            break;
        }

        //poolOfNextPolicies     = {<pol,vals>} 
        //isLowerBound  = bool   - whether the vals are lower bounds to the 
        //                optimal value (i.e. value for the optimal policy)
        //<poolOfNextPolicies,isLowerBound>=ConstructAndValuateNextPolicies(ppi)

        PartialPolicyPoolInterface * poolOfNextPolicies = NewPP();
        bool are_LBs = ConstructAndValuateNextPolicies(ppi, poolOfNextPolicies);

        //Clean up ppi
        if(pp_p->Size() > 0) //should always be true
        {
            pp_p->Pop();
            delete ppi;
        }
        else //should not happen
            throw E("GeneralizedMAAStarPlanner.cpp:policy pool empty? - should not happen?");

#if DEBUG_GMAA4        
        if(DEBUG_GMAA4){
            cout << "\n>>>The next policies found, poolOfNextPolicies:"<<endl;
            PartialPolicyPoolInterface* pp_copy = NewPP();
            *pp_copy = *poolOfNextPolicies;
            while(! pp_copy->Empty())
            {
                PartialPolicyPoolItemInterface* it = pp_copy->Select();
                it->Print();
                cout << endl;
                pp_copy->Pop();
            }
        }
#endif        

        //if(isLowerBound)
        //    Prune( JPolValPool, max(lowerBound) )
        if(are_LBs && poolOfNextPolicies->Size() > 0)
        {
            PartialPolicyPoolItemInterface* bestRanked_ppi = poolOfNextPolicies->
                GetBestRanked();
            poolOfNextPolicies->PopBestRanked();
            double bestNextVal = bestRanked_ppi->GetValue();
            if(bestNextVal > maxLB) //new best lowerbound (and policy) found
            {
                maxLB = bestNextVal;
                *bestJPol = *(bestRanked_ppi->GetJPol());
                if(_m_verboseness >= 2) {
                    cout << "new bestJPol (and max. lowerbound) found!" << endl;
                    cout << "Its value v="
                         << bestNextVal <<" - "
                         << bestRanked_ppi->GetJPol()->SoftPrintBrief() << endl;
                }
                if(_m_verboseness >= 3) 
                    cout << "new bestJPol->SoftPrint():"<<bestJPol->SoftPrint();

                //if we maintain the internal timings...
                if(_m_intermediateResultFile != 0)
                {
                    tms ts_cur;
                    clock_t tck_cur;
                    tck_cur = times(&ts_cur);
                    clock_t diff = tck_cur - tck_start;
                    its << diff << "\t" <<  maxLB << endl;
                }
                // prune JPolValPool
                pp_p->Prune(maxLB - _m_slack );
            }
            delete bestRanked_ppi;

        }
        SelectPoliciesToProcessFurther(poolOfNextPolicies, are_LBs, maxLB - _m_slack);
        pp_p->Union(poolOfNextPolicies);

        delete poolOfNextPolicies;

        if( _m_maxJPolPoolSize < pp_p->Size())
            _m_maxJPolPoolSize = pp_p->Size();
        
        StopTimer("GMAA::Plan::iteration");
        if(_m_verboseness >= 2) { 
            cout << "\nGMAA::Plan::iteration ending, best policy found so far:";
            cout << endl << bestJPol->SoftPrintBrief()  <<endl;
            if(_m_verboseness >= 3)  
                cout << endl << bestJPol->SoftPrint()  <<endl;
        }

    } 
    while(! pp_p->Empty() ); //<- end do...while
    //we don't want to do any conversions here... takes (sometimes too much)
    //time...
    _m_foundPolicy=bestJPol;  //->ToJointPolicyPureVector());


    _m_expectedRewardFoundPolicy=maxLB;
    if(_m_verboseness >= 1) {
        cout << "\nGMAA::Plan ending, best policy found: ";
        cout << bestJPol->SoftPrintBrief() << " = " <<endl;
        if(_m_verboseness >= 3) 
            cout << _m_foundPolicy->SoftPrint() << endl;
        cout << endl;
#if 0
    JointPolicyPureVector* jppv = _m_foundPolicy->ToJointPolicyPureVector();
    jppv->Print();
#endif                 
    }
    if(_m_verboseness >= 2)    
        cout << "\n\n ";
    if(_m_verboseness >= 0)    
        cout << "GMAA::Plan GMAA ENDED"<<endl;
    if(_m_verboseness >= 2)    
        cout << "\n\n ";

    delete pp_p;
    
    StopTimer("GMAA::Plan");
}