void BestFirstSearch::expand_tree(TreeNode* start_node) {
// If the root is terminal, we will not expand any of its children; deal with this
// appropriately
if (start_node->is_terminal) {
set_terminal_root(start_node);
return;
}
int num_simulated_steps = 0;
bool explore = true;
int max_nodes_per_frame = max_sim_steps_per_frame / sim_steps_per_node;
clear_queues();
if (!start_node->v_children.empty()) {
start_node->updateTreeNode();
num_simulated_steps += reuse_branch(start_node);
std::cout << "Num_reused_steps: " << num_simulated_steps << std::endl;
num_simulated_steps = 0;
//COMMENT LINES BELOW, AND UNCOMMENT ABOVE TO WORKSHOP STYLE. ALSO CHANGE FN NOVEL 2ND QUEUE
//reset_branch( start_node );
//q_exploration->push(start_node);
update_novelty_table(start_node->state);
} else {
q_exploration->push(start_node);
update_novelty_table(start_node->state);
}
m_expanded_nodes = 0;
m_generated_nodes = 0;
m_exp_count_novelty1 = 0;
m_exp_count_novelty2 = 0;
m_gen_count_novelty1 = 0;
m_gen_count_novelty2 = 0;
m_pruned_nodes = 0;
while (!(q_exploration->empty() && q_exploitation->empty())) {
// Pop a node to expand
TreeNode* curr_node;
if (q_exploration->empty() && q_exploitation->empty())
break;
if (explore) {
if (q_exploration->empty()) {
explore = false;
continue;
}
curr_node = q_exploration->top();
q_exploration->pop();
explore = false;
} else {
if (q_exploitation->empty()) {
explore = true;
continue;
}
curr_node = q_exploitation->top();
q_exploitation->pop();
explore = true;
}
if (curr_node->depth() > m_max_depth)
m_max_depth = curr_node->depth();
/**
* check nodes that have been expanded by other queue
*/
if (curr_node->already_expanded)
continue;
/**
* check if subtree is bigger than max_budget of nodes
*/
if (curr_node->num_nodes_reusable > max_nodes_per_frame) {
continue;
}
//if(curr_node->novelty != 1)
// std::cout << curr_node->depth() << " " << curr_node->novelty << " " << curr_node->fn << " " << std::endl;
if (curr_node->depth() > m_reward_horizon - 1)
continue;
num_simulated_steps += expand_node(curr_node);
// std::cout << "q_exploration size: "<< q_exploration.size() << std::endl;
// std::cout << "q_exploitation size: "<< q_exploitation.size() << std::endl;
// Stop once we have simulated a maximum number of steps
if (num_simulated_steps >= max_sim_steps_per_frame) {
break;
}
}
std::cout << "\tExpanded so far: " << m_expanded_nodes << std::endl;
std::cout << "\tExpanded Novelty 1: " << m_exp_count_novelty1 << std::endl;
std::cout << "\tExpanded Novelty 2: " << m_exp_count_novelty2 << std::endl;
std::cout << "\tPruned so far: " << m_pruned_nodes << std::endl;
std::cout << "\tGenerated so far: " << m_generated_nodes << std::endl;
std::cout << "\tGenerated Novelty 1: " << m_gen_count_novelty1 << std::endl;
std::cout << "\tGenerated Novelty 2: " << m_gen_count_novelty2 << std::endl;
if (q_exploration->empty() && q_exploitation->empty())
std::cout << "Search Space Exhausted!" << std::endl;
std::cout << "q_exploration size: " << q_exploration->size() << std::endl;
std::cout << "q_exploitation size: " << q_exploitation->size() << std::endl;
update_branch_return(start_node);
}
/* *********************************************************************
Expands the tree from the given node until i_max_sim_steps_per_frame
is reached
******************************************************************* */
void UniformCostSearch::expand_tree(TreeNode* start_node) {
// If the root is terminal, we will not expand any of its children; deal with this
// appropriately
if (start_node->is_terminal) {
set_terminal_root(start_node);
return;
}
std::priority_queue<TreeNode*, std::vector<TreeNode*>, TreeNodeComparer > q;
q.push(start_node);
int num_simulated_steps = 0;
int num_actions = available_actions.size();
m_expanded_nodes = 0;
m_generated_nodes = 0;
m_pruned_nodes = 0;
while(!q.empty()) {
// Pop a node to expand
TreeNode* curr_node = q.top();
q.pop();
bool leaf_node = (curr_node->v_children.empty());
m_expanded_nodes++;
// Expand all of its children (simulates the result)
for (int a = 0; a < num_actions; a++) {
Action act = available_actions[a];
TreeNode * child;
// If re-expanding an internal node, don't creates new nodes
if (leaf_node) {
m_generated_nodes++;
child = new TreeNode(curr_node,
curr_node->state,
this,
act,
sim_steps_per_node);
// Miquel: Let's fix C = 10000
child->fn += ( m_max_reward - child->accumulated_reward ); // Miquel: add this to obtain Hector's BFS + m_max_reward * (720 - child->depth()) ;
// TODO: UniformCostSearch needs to be split into two classes
// the new one encapsulating the novelty-based search algorithm
if (child->depth() > m_max_depth ) m_max_depth = child->depth();
num_simulated_steps += child->num_simulated_steps;
curr_node->v_children.push_back(child);
}
else {
child = curr_node->v_children[a];
if ( !child->is_terminal )
num_simulated_steps += child->num_simulated_steps;
}
// Don't expand duplicate nodes, or terminal nodes
if (!child->is_terminal) {
if (! (ignore_duplicates && test_duplicate(child)) )
q.push(child);
}
}
// Stop once we have simulated a maximum number of steps
if (num_simulated_steps >= max_sim_steps_per_frame) {
break;
}
}
if (q.empty()) std::cout << "Search Space Exhausted!" << std::endl;
update_branch_return(start_node);
}
/* *********************************************************************
Expands the tree from the given node until i_max_sim_steps_per_frame
is reached
******************************************************************* */
void BreadthFirstSearch::expand_tree(TreeNode* start_node) {
// If the root is terminal, we will not expand any of its children; deal with this
// appropriately
if (start_node->is_terminal) {
set_terminal_root(start_node);
return;
}
queue<TreeNode*> q;
q.push(start_node);
int num_simulated_steps = 0;
int num_actions = available_actions.size();
m_expanded_nodes = 0;
m_generated_nodes = 0;
m_pruned_nodes = 0;
while(!q.empty()) {
// Pop a node to expand
TreeNode* curr_node = q.front();
q.pop();
bool leaf_node = (curr_node->v_children.empty());
m_expanded_nodes++;
// Expand all of its children (simulates the result)
if(m_randomize_successor)
std::random_shuffle ( available_actions.begin(), available_actions.end() );
if(leaf_node){
curr_node->v_children.resize( num_actions );
curr_node->available_actions = available_actions;
}
for (int a = 0; a < num_actions; a++) {
Action act = available_actions[a];
TreeNode * child;
// If re-expanding an internal node, don't creates new nodes
if (leaf_node) {
m_generated_nodes++;
child = new TreeNode(curr_node,
curr_node->state,
this,
act,
sim_steps_per_node);
// TODO: BreadthFirstSearch needs to be split into two classes
// the new one encapsulating the novelty-based search algorithm
if (child->depth() > m_max_depth ) m_max_depth = child->depth();
num_simulated_steps += child->num_simulated_steps;
curr_node->v_children[a] = child;
}
else {
child = curr_node->v_children[a];
if ( !child->is_terminal )
num_simulated_steps += child->num_simulated_steps;
}
// Don't expand duplicate nodes, or terminal nodes
if (!child->is_terminal) {
if (! (ignore_duplicates && test_duplicate(child)) )
q.push(child);
}
}
// Stop once we have simulated a maximum number of steps
if (num_simulated_steps >= max_sim_steps_per_frame) {
break;
}
}
if (q.empty()) std::cout << "Search Space Exhausted!" << std::endl;
update_branch_return(start_node);
}
