bool does_value_change(ALEInterface &ale,
const vector<Action> &possible_actions,
unsigned int addr) {
ALEState s0 = ale.cloneSystemState();
ale.environment->oneStepAct(possible_actions.at(0), PLAYER_B_NOOP);
// printf("initial X: %d\n", ale.getRAM().get(addr));
const byte_t x0 = ale.getRAM().get(addr);
bool controllable = false;
for(size_t i=1; !controllable && i<possible_actions.size(); i++) {
ale.restoreSystemState(s0);
ale.environment->oneStepAct(possible_actions.at(i), PLAYER_B_NOOP);
// printf("X: %zu %d\n", i, ale.getRAM().get(addr));
const byte_t xi = ale.getRAM().get(addr);
if(x0 != xi) {
controllable = true;
}
}
ale.restoreSystemState(s0);
ale.environment->processRAM();
ale.environment->processScreen();
return controllable;
}
static hexq::Reward
move_to_the(ALEInterface &ale, DisplayScreen *display, const Action action, const hexq::Reward discount_rate, hexq::MontezumaOptionsMdp &mdp, size_t &elapsed_time, hexq::Reward &nophi_reward, hexq::Reward &phi_reward, vector<pair<hexq::Reward,hexq::State> > &all_steps) {
const vector<Action> *axis_actions;
unsigned int unchanging_addr, changing_addr;
if(action == PLAYER_A_LEFT || action == PLAYER_A_RIGHT) {
axis_actions = &vertical_actions;
unchanging_addr = ADDR_Y;
changing_addr = ADDR_X;
} else {
axis_actions = &horizontal_actions;
unchanging_addr = ADDR_X;
changing_addr = ADDR_Y;
}
const bool initial_cannot_change_axis =
!does_value_change(ale, *axis_actions, unchanging_addr);
hexq::State prev_s = mdp.StateUniqueID();
phi_reward = mdp.ComputeState(ale.act(action), nophi_reward);
vector<pair<pair<hexq::Reward, hexq::Reward>, ALEState> > frames;
frames.push_back(make_pair(make_pair(phi_reward, nophi_reward), ale.cloneSystemState()));
all_steps.push_back(make_pair(phi_reward, prev_s));
byte_t prev_changing = ale.getRAM().get(changing_addr);
const int initial_lives = ale.lives();
int n_frames_unchanged;
bool controllable = true;
bool lost_life = false;
for(size_t max_n_iterations=0; !lost_life && max_n_iterations<MAX_FRAMES; max_n_iterations++) {
hexq::Reward nophi_r;
prev_s = mdp.StateUniqueID();
hexq::Reward reward = mdp.ComputeState(ale.act(action), nophi_r);
frames.push_back(make_pair(make_pair(reward, nophi_r), ale.cloneSystemState()));
all_steps.push_back(make_pair(reward, prev_s));
DISPLAY(display);
controllable = !(ale.getRAM().get(0xd8) != 0x00 || ale.getRAM().get(0xd6) != 0xff);
if(!controllable)break;
bool stop_for_axis_change = initial_cannot_change_axis &&
does_value_change(ale, *axis_actions, unchanging_addr);
if(stop_for_axis_change) {
// printf("Break because axis change possibility %d\n", ABHG++);
break;
}
// printf("X: %d Y: %d\n", ale.getRAM().get(ADDR_X), ale.getRAM().get(ADDR_Y));
byte_t new_changing = ale.getRAM().get(changing_addr);
if(new_changing == prev_changing && controllable) {
n_frames_unchanged++;
if(n_frames_unchanged >= NOT_MOVING_FRAMES) {
// printf("Break because not moving %d\n", ABHG++);
break;
}
} else {
n_frames_unchanged = 0;
prev_changing = new_changing;
}
lost_life = ale.lives() < initial_lives;
}
if((lost_life || !controllable) && frames.size() > N_BACK_FRAMES) {
size_t new_size = frames.size() - N_BACK_FRAMES;
frames.resize(new_size);
all_steps.resize(new_size);
printf("went back\n");
ale.restoreSystemState(frames.rbegin()->second);
ale.environment->processRAM();
ale.environment->processScreen();
DISPLAY(display);
hexq::Reward r;
(void)mdp.ComputeState(0, r);
}
hexq::Reward discount = 1.;
hexq::Reward total_reward = 0;
phi_reward = nophi_reward = 0;
for(size_t i=0; i<frames.size(); i++) {
total_reward += discount*frames.at(i).first.first;
discount *= discount_rate;
nophi_reward += frames.at(i).first.second;
phi_reward += frames.at(i).first.first;
}
elapsed_time += frames.size();
return total_reward;
}