Direction MaxOfFewSearch::move(GameState& state, int ms
#ifdef DEBUG
, std::list<Direction>& moveSequence
#endif
)
{
Direction bestMove = DIR_END;
double bestValue = -std::numeric_limits<double>::infinity();
for(int i = 0; i < k_ || bestMove == DIR_END; ++i)
{
Direction direction = rand() % DIR_END;
if(!state.canMove(direction))
continue;
state.move(direction);
double value = heuristic_(state);
state.undo(direction);
if(value > bestValue || bestMove == DIR_END)
{
bestMove = direction;
bestValue = value;
}
}
#ifdef DEBUG
moveSequence.clear();
moveSequence.push_front(bestMove);
#endif
return bestMove;
}
/** This is the default implementation for extending and backing up a history sequence. */
SearchStatus BasicSearchStrategy::extendAndBackup(HistorySequence *sequence, long maximumDepth) {
// We extend from the last entry in the sequence.
HistoryEntry *firstEntry = sequence->getLastEntry();
long firstEntryId = firstEntry->getId();
HistoryEntry *currentEntry = firstEntry;
BeliefNode *currentNode = currentEntry->getAssociatedBeliefNode();
SearchStatus status = SearchStatus::UNINITIALIZED;
std::unique_ptr<StepGenerator> generator = factory_->createGenerator(status, currentEntry,
currentEntry->getState(), currentNode->getHistoricalData());
if (status == SearchStatus::UNINITIALIZED) {
// Failure to initialize => return this status.
return status;
}
// We check for some invalid possibilities in order to deal with them more cleanly.
Model *model = solver_->getModel();
if (model->isTerminal(*currentEntry->getState())) {
debug::show_message("WARNING: Attempted to continue sequence"
" from a terminal state.");
return SearchStatus::ERROR;
} else if (currentEntry->getAction() != nullptr) {
debug::show_message("ERROR: The last in the sequence already has an action!?");
return SearchStatus::ERROR;
} else if (currentEntry->immediateReward_ != 0) {
debug::show_message("ERROR: The last in the sequence has a nonzero reward!?");
return SearchStatus::ERROR;
}
while (true) {
if (currentNode->getDepth() >= maximumDepth) {
// We've hit the depth limit, so we can't generate any more steps in the sequence.
status = SearchStatus::OUT_OF_STEPS;
break;
}
// Step the search forward.
Model::StepResult result = generator->getStep(currentEntry, currentEntry->getState(),
currentNode->getHistoricalData());
// Null action => stop the search.
if (result.action == nullptr) {
break;
}
// Set the parameters of the current history entry using the ones we got from the result.
currentEntry->immediateReward_ = result.reward;
currentEntry->action_ = std::move(result.action);
currentEntry->transitionParameters_ = std::move(result.transitionParameters);
currentEntry->observation_ = std::move(result.observation);
// Create the child belief node, and set the current node to be that node.
BeliefNode *nextNode = currentNode->createOrGetChild(*currentEntry->action_,
*currentEntry->observation_);
currentNode = nextNode;
// Now we create a new history entry and step the history forward.
StateInfo *nextStateInfo = solver_->getStatePool()->createOrGetInfo(*result.nextState);
currentEntry = sequence->addEntry();
// Register the new history entry with its state, and with its associated belief node.
currentEntry->registerState(nextStateInfo);
currentEntry->registerNode(currentNode);
if (result.isTerminal) {
// Terminal state => search complete.
status = SearchStatus::FINISHED;
break;
}
}
// OUT_OF_STEPS => must calculated a heuristic estimate.
if (status == SearchStatus::OUT_OF_STEPS) {
currentEntry->immediateReward_ = heuristic_(currentEntry, currentEntry->getState(),
currentNode->getHistoricalData());
status = SearchStatus::FINISHED;
}
if (status == SearchStatus::FINISHED) {
// Now that we're finished, we back up the sequence.
if (firstEntryId > 0 && currentEntry->getId() > firstEntryId) {
// If we've extended a previously terminating sequence, we have to add a continuation.
solver_->updateEstimate(firstEntry->getAssociatedBeliefNode(), 0, +1);
}
// We only do a partial backup along the newly generated part of the sequence.
solver_->updateSequence(sequence, +1, firstEntryId);
} else {
// This shouldn't happen => print out an error message.
if (status == SearchStatus::UNINITIALIZED) {
debug::show_message("ERROR: Search algorithm could not initialize.");
} else if (status == SearchStatus::INITIAL) {
debug::show_message("ERROR: Search algorithm initialized but did not run!?");
} else if (status == SearchStatus::ERROR) {
debug::show_message("ERROR: Error in search algorithm!");
} else {
debug::show_message("ERROR: Invalid search status.");
}
}
// Finally, we just return the status.
return status;
}
