void ROCKSAMPLE::DisplayBeliefs(const BELIEF_STATE &belief,
std::ostream &ostr) const {
std::vector<double> prob(NumRocks);
for (int i = 0; i < belief.GetNumSamples(); ++i) {
const ROCKSAMPLE_STATE &rockstate =
safe_cast<const ROCKSAMPLE_STATE &>(*belief.GetSample(i));
for (int j = 0; j < NumRocks; ++j) {
if (!rockstate.Rocks[j].Collected) {
prob[j] += rockstate.Rocks[j].Valuable;
}
}
}
ostr << "#Belief: ";
for (int j = 0; j < NumRocks; ++j) {
prob[j] /= belief.GetNumSamples();
if (prob[j] > 0.0) {
ostr << "#" << j << "(" << prob[j] << ") ";
}
}
ostr << std::endl;
}
void POCMAN::DisplayBeliefs(const BELIEF_STATE& beliefState,
ostream& ostr) const
{
GRID<int> counts(Maze.GetXSize(), Maze.GetYSize());
counts.SetAllValues(0);
for (int i = 0; i < beliefState.GetNumSamples(); i++)
{
const POCMAN_STATE* pocstate =
safe_cast<const POCMAN_STATE*>(
beliefState.GetSample(i));
for (int g = 0; g < NumGhosts; g++)
counts(pocstate->GhostPos[g])++;
}
for (int y = Maze.GetYSize() - 1; y >= 0; y--)
{
for (int x = 0; x < Maze.GetXSize(); x++)
{
ostr.width(6);
ostr.precision(2);
ostr << fixed << (double) counts(x, y) / beliefState.GetNumSamples();
}
ostr << endl;
}
}
bool MCTS::Update(int action, int observation, double reward) {
History.Add(action, observation);
BELIEF_STATE beliefs;
// Find matching vnode from the rest of the tree
QNODE &qnode = Root->Child(action);
VNODE *vnode = qnode.Child(observation);
if( vnode ) {
if( Params.Verbose >= 1 )
cout << "Matched " << vnode->Beliefs().GetNumSamples() << " states" << endl;
beliefs.Copy(vnode->Beliefs(), Simulator);
} else {
if( Params.Verbose >= 1 )
cout << "No matching node found" << endl;
}
// Generate transformed states to avoid particle deprivation
if( Params.UseTransforms )
AddTransforms(Root, beliefs);
// If we still have no particles, fail
if( beliefs.Empty() && (!vnode || vnode->Beliefs().Empty()))
return false;
if( Params.Verbose >= 1 )
Simulator.DisplayBeliefs(beliefs, cout);
// Find a state to initialise prior (only requires fully observed state)
const STATE *state = 0;
if( vnode && !vnode->Beliefs().Empty())
state = vnode->Beliefs().GetSample(0);
else
state = beliefs.GetSample(0);
// Delete old tree and create new root
VNODE::Free(Root, Simulator);
VNODE *newRoot = ExpandNode(state);
newRoot->Beliefs() = beliefs;
Root = newRoot;
return true;
}
void ROOMS::DisplayBeliefs(const BELIEF_STATE &belief, std::ostream &ostr) const {
unordered_map<COORD, int> m;
for (int i = 0; i < belief.GetNumSamples(); ++i) {
const ROOMS_STATE &state =
safe_cast<const ROOMS_STATE &>(*belief.GetSample(i));
m[state.AgentPos] += 1;
}
vector<pair<double, const COORD *>> sorted;
for (unordered_map<COORD, int>::iterator it = m.begin(); it != m.end();
++it) {
double p = double(it->second) / double(belief.GetNumSamples());
sorted.push_back(make_pair(p, &(it->first)));
}
sort(sorted.begin(), sorted.end(), greater<pair<double, const COORD *>>());
ostr << "#Belief: ";
for (uint i = 0; i < sorted.size(); ++i) {
ostr << "#" << *(sorted[i].second) << " (" << sorted[i].first << ") ";
}
ostr << std::endl;
}
void GHOSTGAME::DisplayBeliefs(const BELIEF_STATE& beliefState,
ostream& ostr) const
{
const GHOSTGAME_STATE* ghostGameState;
int numGhosts = 0;
if (beliefState.GetNumSamples() > 0)
{
ghostGameState = safe_cast<const GHOSTGAME_STATE*>(beliefState.GetSample(0));
numGhosts = ghostGameState->GhostLocs.size();
}
vector<int> targetCounts(numGhosts, 0);
for (int i = 0; i < beliefState.GetNumSamples(); i++)
{
ghostGameState = safe_cast<const GHOSTGAME_STATE*>(beliefState.GetSample(i));
targetCounts[ghostGameState->TargetGhost]++;
}
for(unsigned int i = 0; i < targetCounts.size(); ++i)
{
ostr << "Ghost " << i << ": " << (targetCounts[i] / beliefState.GetNumSamples()) << endl;
}
}
void FlatMCTS::ParticleFilter(BELIEF_STATE &beliefs) // unweighted particle filter
{
int attempts = 0, added = 0;
int max_attempts = (Params.NumStartStates - beliefs.GetNumSamples()) * 10;
int realObs = History.Back().Observation;
int stepObs;
double stepReward;
if (Params.Verbose >= 1) {
cout << "MCTS::ParticleFilter: last step belief size "
<< Root->Beliefs().GetNumSamples() << ", current belief size "
<< beliefs.GetNumSamples() << endl;
}
while (beliefs.GetNumSamples() < Params.NumStartStates &&
attempts < max_attempts) {
STATE *state = Root->Beliefs().CreateSample(Simulator);
Simulator.Step(*state, History.Back().Action, stepObs, stepReward);
if (Params.ThompsonSampling) {
Root->Child(History.Back().Action).Update(stepObs, stepReward);
}
if (stepObs == realObs) {
beliefs.AddSample(state);
added++;
} else {
Simulator.FreeState(state);
}
attempts++;
}
if (Params.Verbose >= 1) {
cout << "MCTS::ParticleFilter: Created " << added
<< " local transformations out of " << attempts << " attempts" << endl;
}
}
void FlatMCTS::AddTransforms(BELIEF_STATE &beliefs) {
int attempts = 0, added = 0;
if (Params.Verbose >= 1) {
cout << "MCTS::AddTransforms: last step belief size "
<< Root->Beliefs().GetNumSamples() << ", current belief size "
<< beliefs.GetNumSamples() << endl;
}
// Local transformations of state that are consistent with history
while (added < Params.NumTransforms && attempts < Params.MaxAttempts) {
STATE *transform = CreateTransform();
if (transform) {
beliefs.AddSample(transform);
added++;
}
attempts++;
}
if (Params.Verbose >= 1) {
cout << "MCTS::AddTransforms: Created " << added
<< " local transformations out of " << attempts << " attempts" << endl;
}
}
void MCTS::AddTransforms(VNODE *root, BELIEF_STATE &beliefs) {
int attempts = 0, added = 0;
// Local transformations of state that are consistent with history
while (added < Params.NumTransforms && attempts < Params.MaxAttempts) {
STATE *transform = CreateTransform();
if( transform ) {
beliefs.AddSample(transform);
added++;
}
attempts++;
}
if( Params.Verbose >= 1 ) {
cout << "Created " << added << " local transformations out of " << attempts << " attempts" << endl;
}
}
bool FlatMCTS::Update(int action, int observation, STATE &state) {
History.Add(action, observation);
if (Simulator.mFullyObservable) { // running an MDP in fact in cases of hplanning
// Delete old tree and create new root
VNODE::Free(Root, Simulator);
Root = ExpandNode(&state, History);
STATE *sample = Simulator.Copy(state);
Root->Beliefs().AddSample(sample);
if (Params.Verbose >= 1) Simulator.DisplayBeliefs(Root->Beliefs(), cout);
return true;
}
else {
BELIEF_STATE beliefs;
// Find matching vnode from the rest of the tree
QNODE &qnode = Root->Child(action);
VNODE *vnode = qnode.Child(observation);
if (vnode) {
if (Params.Verbose >= 1)
cout << "Matched " << vnode->Beliefs().GetNumSamples() << " states"
<< endl;
beliefs.Copy(vnode->Beliefs(), Simulator);
} else {
if (Params.Verbose >= 1) cout << "No matching node found" << endl;
}
if (Params.Verbose >= 1) Simulator.DisplayBeliefs(beliefs, cout);
if (Params.UseParticleFilter) {
ParticleFilter(beliefs);
if (Params.Verbose >= 1) Simulator.DisplayBeliefs(beliefs, cout);
}
// Generate transformed states to avoid particle deprivation
if (Params.UseTransforms) {
AddTransforms(beliefs);
if (Params.Verbose >= 1) Simulator.DisplayBeliefs(beliefs, cout);
}
// If we still have no particles, fail
if (beliefs.Empty() && (!vnode || vnode->Beliefs().Empty()))
return false;
// Find a state to initialise prior (only requires fully observed state)
const STATE *sample = 0;
if (vnode && !vnode->Beliefs().Empty())
sample = vnode->Beliefs().GetSample();
else
sample = beliefs.GetSample();
if (vnode && Params.ReuseTree) {
int size1 = VNODE::GetNumAllocated();
VNODE::Free(Root, Simulator, vnode);
int size2 = VNODE::GetNumAllocated();
assert(size2 < size1);
Root = vnode;
Root->Beliefs().Free(Simulator);
} else { // Delete old tree and create new root
VNODE::Free(Root, Simulator);
Root = ExpandNode(sample, History);
}
Root->Beliefs() = beliefs;
return true;
}
}