void Learner::act(ALEInterface& ale, int action, vector<float> &reward, vector<vector<vector<float> > > &learnedOptions){
float r_alg = 0.0, r_real = 0.0;
FRam.clear();
ramFeatures.getCompleteFeatureVector(ale.getRAM(), FRam);
if(action < numBasicActions){
r_real = ale.act(actions[action]);
}
else{
int option_idx = action - numBasicActions;
r_real = playOption(ale, option_idx, learnedOptions);
}
FnextRam.clear();
ramFeatures.getCompleteFeatureVector(ale.getRAM(), FnextRam);
updateTransitionVector(FRam, FnextRam);
for(int i = 0; i < transitions.size(); i++){
transitions[i] = (transitions[i] - mean[i])/std[i];
r_alg += eigVector[i] * transitions[i];
}
reward[0] = r_alg;
reward[1] = r_real;
}
void TrueOnlineSarsaLearner::evaluatePolicy(ALEInterface& ale, Features *features){
double reward = 0;
double cumReward = 0;
double prevCumReward = 0;
//Repeat (for each episode):
for(int episode = 0; episode < numEpisodesEval; episode++){
//Repeat(for each step of episode) until game is over:
for(int step = 0; !ale.game_over() && step < episodeLength; step++){
//Get state and features active on that state:
F.clear();
features->getActiveFeaturesIndices(ale.getScreen(), ale.getRAM(), F);
updateQValues(F, Q); //Update Q-values for each possible action
currentAction = epsilonGreedy(Q);
//Take action, observe reward and next state:
reward = 0;
for(int i = 0; i < numStepsPerAction && !ale.game_over() ; i++){
reward += ale.act(actions[currentAction]);
}
cumReward += reward;
}
ale.reset_game();
sanityCheck();
printf("%d, %f, %f \n", episode + 1, (double)cumReward/(episode + 1.0), cumReward-prevCumReward);
prevCumReward = cumReward;
}
}
int actUpdatingAvg(ALEInterface& ale, RAMFeatures *ram, BPROFeatures *features, int nextAction,
vector<vector<vector<float> > > &w, Parameters param, int totalNumFrames, int gameId,
vector<bool> &F, vector<bool> &Fprev){
int reward = 0;
//If the selected action was one of the primitive actions
if(nextAction < NUM_ACTIONS){
for(int i = 0; i < FRAME_SKIP && totalNumFrames + ale.getEpisodeFrameNumber() < MAX_NUM_FRAMES; i++){
reward += ale.act((Action) nextAction);
Fprev.swap(F);
F.clear();
ram->getCompleteFeatureVector(ale.getRAM(), F);
F.pop_back();
updateAverage(Fprev, F, ale.getEpisodeFrameNumber(), param, gameId);
}
}
//If the selected action was one of the options
else{
int currentAction;
vector<int> Fbpro; //Set of features active
vector<float> Q(NUM_ACTIONS, 0.0); //Q(a) entries
int option = nextAction - NUM_ACTIONS;
while(rand()%1000 > 1000 * PROB_TERMINATION && !ale.game_over() && totalNumFrames + ale.getEpisodeFrameNumber() < MAX_NUM_FRAMES){
//Get state and features active on that state:
Fbpro.clear();
features->getActiveFeaturesIndices(ale.getScreen(), Fbpro);
updateQValues(Fbpro, Q, w, option); //Update Q-values for each possible action
currentAction = epsilonGreedy(Q);
//Take action, observe reward and next state:
reward += ale.act((Action) currentAction);
Fprev.swap(F);
F.clear();
ram->getCompleteFeatureVector(ale.getRAM(), F);
F.pop_back();
updateAverage(Fprev, F, ale.getEpisodeFrameNumber(), param, gameId);
}
}
return reward;
}
bool does_value_change(ALEInterface &ale,
const vector<Action> &possible_actions,
unsigned int addr) {
ALEState s0 = ale.cloneSystemState();
ale.environment->oneStepAct(possible_actions.at(0), PLAYER_B_NOOP);
// printf("initial X: %d\n", ale.getRAM().get(addr));
const byte_t x0 = ale.getRAM().get(addr);
bool controllable = false;
for(size_t i=1; !controllable && i<possible_actions.size(); i++) {
ale.restoreSystemState(s0);
ale.environment->oneStepAct(possible_actions.at(i), PLAYER_B_NOOP);
// printf("X: %zu %d\n", i, ale.getRAM().get(addr));
const byte_t xi = ale.getRAM().get(addr);
if(x0 != xi) {
controllable = true;
}
}
ale.restoreSystemState(s0);
ale.environment->processRAM();
ale.environment->processScreen();
return controllable;
}
void TrueOnlineSarsaLearner::learnPolicy(ALEInterface& ale, Features *features){
struct timeval tvBegin, tvEnd, tvDiff;
vector<double> reward;
double elapsedTime;
double norm_a;
double q_old, delta_q;
double cumReward = 0, prevCumReward = 0;
unsigned int maxFeatVectorNorm = 1;
sawFirstReward = 0; firstReward = 1.0;
//Repeat (for each episode):
for(int episode = 0; episode < numEpisodesLearn; episode++){
for(unsigned int a = 0; a < nonZeroElig.size(); a++){
for(unsigned int i = 0; i < nonZeroElig[a].size(); i++){
int idx = nonZeroElig[a][i];
e[a][idx] = 0.0;
}
nonZeroElig[a].clear();
}
//We have to clean the traces every episode:
for(unsigned int i = 0; i < e.size(); i++){
for(unsigned int j = 0; j < e[i].size(); j++){
e[i][j] = 0.0;
}
}
F.clear();
features->getActiveFeaturesIndices(ale.getScreen(), ale.getRAM(), F);
updateQValues(F, Q);
currentAction = epsilonGreedy(Q);
q_old = Q[currentAction];
//Repeat(for each step of episode) until game is over:
gettimeofday(&tvBegin, NULL);
frame = 0;
while(frame < episodeLength && !ale.game_over()){
reward.clear();
reward.push_back(0.0);
reward.push_back(0.0);
updateQValues(F, Q);
sanityCheck();
//Take action, observe reward and next state:
act(ale, currentAction, reward);
cumReward += reward[1];
if(!ale.game_over()){
//Obtain active features in the new state:
Fnext.clear();
features->getActiveFeaturesIndices(ale.getScreen(), ale.getRAM(), Fnext);
updateQValues(Fnext, Qnext); //Update Q-values for the new active features
nextAction = epsilonGreedy(Qnext);
}
else{
nextAction = 0;
for(unsigned int i = 0; i < Qnext.size(); i++){
Qnext[i] = 0;
}
}
//To ensure the learning rate will never increase along
//the time, Marc used such approach in his JAIR paper
if (F.size() > maxFeatVectorNorm){
maxFeatVectorNorm = F.size();
}
norm_a = alpha/maxFeatVectorNorm;
delta_q = Q[currentAction] - q_old;
q_old = Qnext[nextAction];
delta = reward[0] + gamma * Qnext[nextAction] - Q[currentAction];
//e <- e + [1 - alpha * e^T phi(S,A)] phi(S,A)
updateTrace(currentAction, norm_a);
//theta <- theta + alpha * delta * e + alpha * delta_q (e - phi(S,A))
updateWeights(currentAction, norm_a, delta_q);
//e <- gamma * lambda * e
decayTrace();
F = Fnext;
currentAction = nextAction;
}
ale.reset_game();
gettimeofday(&tvEnd, NULL);
timeval_subtract(&tvDiff, &tvEnd, &tvBegin);
elapsedTime = double(tvDiff.tv_sec) + double(tvDiff.tv_usec)/1000000.0;
double fps = double(frame)/elapsedTime;
printf("episode: %d,\t%.0f points,\tavg. return: %.1f,\t%d frames,\t%.0f fps\n",
episode + 1, (cumReward-prevCumReward), (double)cumReward/(episode + 1.0), frame, fps);
prevCumReward = cumReward;
}
}
static hexq::Reward
move_to_the(ALEInterface &ale, DisplayScreen *display, const Action action, const hexq::Reward discount_rate, hexq::MontezumaOptionsMdp &mdp, size_t &elapsed_time, hexq::Reward &nophi_reward, hexq::Reward &phi_reward, vector<pair<hexq::Reward,hexq::State> > &all_steps) {
const vector<Action> *axis_actions;
unsigned int unchanging_addr, changing_addr;
if(action == PLAYER_A_LEFT || action == PLAYER_A_RIGHT) {
axis_actions = &vertical_actions;
unchanging_addr = ADDR_Y;
changing_addr = ADDR_X;
} else {
axis_actions = &horizontal_actions;
unchanging_addr = ADDR_X;
changing_addr = ADDR_Y;
}
const bool initial_cannot_change_axis =
!does_value_change(ale, *axis_actions, unchanging_addr);
hexq::State prev_s = mdp.StateUniqueID();
phi_reward = mdp.ComputeState(ale.act(action), nophi_reward);
vector<pair<pair<hexq::Reward, hexq::Reward>, ALEState> > frames;
frames.push_back(make_pair(make_pair(phi_reward, nophi_reward), ale.cloneSystemState()));
all_steps.push_back(make_pair(phi_reward, prev_s));
byte_t prev_changing = ale.getRAM().get(changing_addr);
const int initial_lives = ale.lives();
int n_frames_unchanged;
bool controllable = true;
bool lost_life = false;
for(size_t max_n_iterations=0; !lost_life && max_n_iterations<MAX_FRAMES; max_n_iterations++) {
hexq::Reward nophi_r;
prev_s = mdp.StateUniqueID();
hexq::Reward reward = mdp.ComputeState(ale.act(action), nophi_r);
frames.push_back(make_pair(make_pair(reward, nophi_r), ale.cloneSystemState()));
all_steps.push_back(make_pair(reward, prev_s));
DISPLAY(display);
controllable = !(ale.getRAM().get(0xd8) != 0x00 || ale.getRAM().get(0xd6) != 0xff);
if(!controllable)break;
bool stop_for_axis_change = initial_cannot_change_axis &&
does_value_change(ale, *axis_actions, unchanging_addr);
if(stop_for_axis_change) {
// printf("Break because axis change possibility %d\n", ABHG++);
break;
}
// printf("X: %d Y: %d\n", ale.getRAM().get(ADDR_X), ale.getRAM().get(ADDR_Y));
byte_t new_changing = ale.getRAM().get(changing_addr);
if(new_changing == prev_changing && controllable) {
n_frames_unchanged++;
if(n_frames_unchanged >= NOT_MOVING_FRAMES) {
// printf("Break because not moving %d\n", ABHG++);
break;
}
} else {
n_frames_unchanged = 0;
prev_changing = new_changing;
}
lost_life = ale.lives() < initial_lives;
}
if((lost_life || !controllable) && frames.size() > N_BACK_FRAMES) {
size_t new_size = frames.size() - N_BACK_FRAMES;
frames.resize(new_size);
all_steps.resize(new_size);
printf("went back\n");
ale.restoreSystemState(frames.rbegin()->second);
ale.environment->processRAM();
ale.environment->processScreen();
DISPLAY(display);
hexq::Reward r;
(void)mdp.ComputeState(0, r);
}
hexq::Reward discount = 1.;
hexq::Reward total_reward = 0;
phi_reward = nophi_reward = 0;
for(size_t i=0; i<frames.size(); i++) {
total_reward += discount*frames.at(i).first.first;
discount *= discount_rate;
nophi_reward += frames.at(i).first.second;
phi_reward += frames.at(i).first.first;
}
elapsed_time += frames.size();
return total_reward;
}