/* ***************************************************************************
* This is a temporary method, used for preparing a demo video.
* It should be pretty much ignored!
* ***************************************************************************/
Action SearchAgent::tmp_prepare_demo_vid(void) {
if (i_frame_counter < 1250) {
return PLAYER_A_NOOP;
}
if (i_frame_counter < 1300) {
return PLAYER_A_DOWN;
}
str_curr_state = save_state();
// initilize the screen_matrix
if (pm_sim_scr_matrix == NULL) {
pm_sim_scr_matrix = new IntMatrix;
assert(i_screen_height > 0);
assert(i_screen_width > 0);
for (int i = 0; i < i_screen_height; i++) {
IntVect row;
for (int j = 0; j < i_screen_width; j++) {
row.push_back(-1);
}
pm_sim_scr_matrix->push_back(row);
}
}
char buffer [50];
ostringstream filename;
MediaSource& mediasrc = p_osystem->console().mediaSource();
for (int a = 0; a < i_num_actions; a++) {
load_state(str_curr_state);
GameController::apply_action(p_sim_event_obj, PLAYER_A_NOOP, PLAYER_B_NOOP);
p_osystem->myTimingInfo.start = p_osystem->getTicks();
mediasrc.update();
Action curr_act = (*p_game_settings->pv_possible_actions)[a];
filename.str("");
filename << "action_" << action_to_string(curr_act) << "_00.png";
p_osystem->p_export_screen->save_png(pm_sim_scr_matrix, filename.str());
for (int i = 0; i < 15; i++) {
GameController::apply_action(p_sim_event_obj, curr_act, PLAYER_B_NOOP);
p_osystem->myTimingInfo.start = p_osystem->getTicks();
mediasrc.update();
copy_simulated_framebuffer();
sprintf (buffer, "%02d", i + 1);
filename.str("");
filename << "action_" << action_to_string(curr_act) << "_"
<< buffer << ".png";
p_osystem->p_export_screen->save_png(pm_sim_scr_matrix, filename.str());
}
}
end_game();
return UNDEFINED;
}
Action DQN::SelectAction(const InputFrames& last_frames, const double epsilon) {
assert(epsilon >= 0.0 && epsilon <= 1.0);
auto action = SelectActionGreedily(last_frames).first;
if (std::uniform_real_distribution<>(0.0, 1.0)(random_engine) < epsilon) {
// Select randomly
const auto random_idx =
std::uniform_int_distribution<int>(0, legal_actions_.size() - 1)(random_engine);
action = legal_actions_[random_idx];
std::cout << action_to_string(action) << " (random)";
} else {
std::cout << action_to_string(action) << " (greedy)";
}
std::cout << " epsilon:" << epsilon << std::endl;
return action;
}
static int do_kobject_uevent(struct kobject *kobj, enum kobject_action action,
struct attribute *attr, int gfp_mask)
{
char *path;
char *attrpath;
char *signal;
int len;
int rc = -ENOMEM;
path = kobject_get_path(kobj, gfp_mask);
if (!path)
return -ENOMEM;
signal = action_to_string(action);
if (!signal)
return -EINVAL;
if (attr) {
len = strlen(path);
len += strlen(attr->name) + 2;
attrpath = kmalloc(len, gfp_mask);
if (!attrpath)
goto exit;
sprintf(attrpath, "%s/%s", path, attr->name);
rc = send_uevent(signal, attrpath, NULL, gfp_mask);
kfree(attrpath);
} else
rc = send_uevent(signal, path, NULL, gfp_mask);
exit:
kfree(path);
return rc;
}
/* *********************************************************************
This method is called when the game ends. The superclass
implementation takes care of counting number of episodes, and
saving the reward history. Here we check to see if we need to
swtch to the next action, and also update the episode counter and
v_act_epis_rewards
******************************************************************** */
void ActionSummaryAgent::on_end_of_game(void) {
v_act_epis_rewards.push_back(f_episode_reward);
i_act_episode_counter++;
if (i_act_episode_counter > i_max_episode_per_act) {
// Save the average reward for this action
float average_reward = get_vector_average( &v_act_epis_rewards, 0,
v_act_epis_rewards.size());
int ind;
if (i_curr_action_index == -1) {
ind = RANDOM;
} else {
ind = (*pv_possible_actions)[i_curr_action_index];
}
v_results_sum[ind] = average_reward;
// Export the rewards
string filename = "reward_per_episode__act_" + str_current_action + ".txt";
export_vector(&v_act_epis_rewards, filename);
// switch to the next action
i_curr_action_index++;
if (i_curr_action_index == i_num_actions) {
// we are done
save_results_summary();
end_game();
}
str_current_action = action_to_string(
(*pv_possible_actions)[i_curr_action_index]);
cout << " ** Switching to action:" << str_current_action << " **" << endl;
i_act_episode_counter = 0;
v_act_epis_rewards.clear();
}
PlayerAgent::on_end_of_game();
}
void UniformCostSearch::print_frame_data( int frame_number, float elapsed, Action curr_action, std::ostream& output )
{
output << "frame=" << frame_number;
output << ",expanded=" << expanded_nodes();
output << ",generated=" << generated_nodes();
output << ",pruned=" << pruned();
output << ",depth_tree=" << max_depth();
output << ",tree_size=" << num_nodes();
output << ",best_action=" << action_to_string( curr_action );
output << ",branch_reward=" << get_root_value();
output << ",elapsed=" << elapsed << std::endl;
}
/* *********************************************************************
Saves the summary results to a text file
******************************************************************** */
void ActionSummaryAgent::save_results_summary(void) {
ofstream file;
file.open("actions_summary.txt");
file << "action,average_reward_over_"
<< i_max_episode_per_act << "_episodes" << endl;
file << "RANDOM," << v_results_sum[RANDOM] << endl;
for (unsigned int a = 0; a < i_num_actions; a++) {
file << action_to_string((*pv_possible_actions)[a]) << ","
<< v_results_sum[(*pv_possible_actions)[a]] << endl;
}
file.close();
}
std::string PrintQValues(
const std::vector<float>& q_values, const ActionVect& actions) {
assert(!q_values.empty());
assert(!actions.empty());
assert(q_values.size() == actions.size());
std::ostringstream actions_buf;
std::ostringstream q_values_buf;
for (auto i = 0; i < q_values.size(); ++i) {
const auto a_str =
boost::algorithm::replace_all_copy(
action_to_string(actions[i]), "PLAYER_A_", "");
const auto q_str = std::to_string(q_values[i]);
const auto column_size = std::max(a_str.size(), q_str.size()) + 1;
actions_buf.width(column_size);
actions_buf << a_str;
q_values_buf.width(column_size);
q_values_buf << q_str;
}
actions_buf << std::endl;
q_values_buf << std::endl;
return actions_buf.str() + q_values_buf.str();
}
static int do_vzevent_send(int event, char *msg, int len)
{
struct sk_buff *skb;
char *buf, *action;
int alen;
action = action_to_string(event);
if (!action)
return -EINVAL;
alen = strlen(action);
skb = alloc_skb(len + 1 + alen, GFP_KERNEL);
if (!skb)
return -ENOMEM;
buf = skb_put(skb, len + 1 + alen);
memcpy(buf, action, alen);
buf[alen] = '@';
memcpy(buf + alen + 1, msg, len);
(void)netlink_broadcast(vzev_sock, skb, 0, VZ_EVGRP_ALL, GFP_KERNEL);
return 0;
}
/**
* kobject_hotplug - notify userspace by executing /sbin/hotplug
*
* @action: action that is happening (usually "ADD" or "REMOVE")
* @kobj: struct kobject that the action is happening to
*/
void kobject_hotplug(struct kobject *kobj, enum kobject_action action)
{
char *argv [3];
char **envp = NULL;
char *buffer = NULL;
char *seq_buff;
char *scratch;
int i = 0;
int retval;
char *kobj_path = NULL;
char *name = NULL;
char *action_string;
#ifdef CONFIG_MOT_FEAT_HOTPLUG_FILTER
char *filter = NULL;
#endif
u64 seq;
struct kobject *top_kobj = kobj;
struct kset *kset;
static struct kset_hotplug_ops null_hotplug_ops;
struct kset_hotplug_ops *hotplug_ops = &null_hotplug_ops;
/* If this kobj does not belong to a kset,
try to find a parent that does. */
if (!top_kobj->kset && top_kobj->parent) {
do {
top_kobj = top_kobj->parent;
} while (!top_kobj->kset && top_kobj->parent);
}
if (top_kobj->kset)
kset = top_kobj->kset;
else
return;
if (kset->hotplug_ops)
hotplug_ops = kset->hotplug_ops;
/* If the kset has a filter operation, call it.
Skip the event, if the filter returns zero. */
if (hotplug_ops->filter) {
if (!hotplug_ops->filter(kset, kobj))
return;
}
pr_debug ("%s\n", __FUNCTION__);
action_string = action_to_string(action);
if (!action_string)
return;
envp = kmalloc(NUM_ENVP * sizeof (char *), GFP_KERNEL);
if (!envp)
return;
memset (envp, 0x00, NUM_ENVP * sizeof (char *));
buffer = kmalloc(BUFFER_SIZE, GFP_KERNEL);
if (!buffer)
goto exit;
if (hotplug_ops->name)
name = hotplug_ops->name(kset, kobj);
if (name == NULL)
name = kset->kobj.name;
#ifdef CONFIG_MOT_FEAT_HOTPLUG_FILTER
if (hotplug_filters) {
for ( filter = hotplug_filters; *filter != '\0'; ) {
if (!strcmp(filter, name))
break;
filter += strlen(filter) + 1;
}
if (*filter == '\0')
goto exit;
} else {
/* default state is to filter all events */
goto exit;
}
#endif
argv [0] = hotplug_path;
argv [1] = name;
argv [2] = NULL;
/* minimal command environment */
envp [i++] = "HOME=/";
envp [i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
scratch = buffer;
envp [i++] = scratch;
scratch += sprintf(scratch, "ACTION=%s", action_string) + 1;
kobj_path = kobject_get_path(kobj, GFP_KERNEL);
if (!kobj_path)
goto exit;
envp [i++] = scratch;
scratch += sprintf (scratch, "DEVPATH=%s", kobj_path) + 1;
envp [i++] = scratch;
scratch += sprintf(scratch, "SUBSYSTEM=%s", name) + 1;
/* reserve space for the sequence,
* put the real one in after the hotplug call */
envp[i++] = seq_buff = scratch;
scratch += strlen("SEQNUM=18446744073709551616") + 1;
if (hotplug_ops->hotplug) {
/* have the kset specific function add its stuff */
retval = hotplug_ops->hotplug (kset, kobj,
&envp[i], NUM_ENVP - i, scratch,
BUFFER_SIZE - (scratch - buffer));
if (retval) {
pr_debug ("%s - hotplug() returned %d\n",
__FUNCTION__, retval);
goto exit;
}
}
spin_lock(&sequence_lock);
seq = ++hotplug_seqnum;
spin_unlock(&sequence_lock);
sprintf(seq_buff, "SEQNUM=%llu", (unsigned long long)seq);
pr_debug ("%s: %s %s seq=%llu %s %s %s %s %s\n",
__FUNCTION__, argv[0], argv[1], (unsigned long long)seq,
envp[0], envp[1], envp[2], envp[3], envp[4]);
send_uevent(action_string, kobj_path, envp, GFP_KERNEL);
if (!hotplug_path[0])
goto exit;
retval = call_usermodehelper (argv[0], argv, envp, 0);
if (retval)
pr_debug ("%s - call_usermodehelper returned %d\n",
__FUNCTION__, retval);
exit:
kfree(kobj_path);
kfree(buffer);
kfree(envp);
return;
}
/**
* kobject_uevent_env - send an uevent with environmental data
*
* @action: action that is happening (usually KOBJ_MOVE)
* @kobj: struct kobject that the action is happening to
* @envp_ext: pointer to environmental data
*
* Returns 0 if kobject_uevent() is completed with success or the
* corresponding error when it fails.
*/
int kobject_uevent_env(struct kobject *kobj, enum kobject_action action,
char *envp_ext[])
{
char **envp;
char *buffer;
char *scratch;
const char *action_string;
const char *devpath = NULL;
const char *subsystem;
struct kobject *top_kobj;
struct kset *kset;
struct kset_uevent_ops *uevent_ops;
u64 seq;
char *seq_buff;
int i = 0;
int retval = 0;
int j;
pr_debug("%s\n", __FUNCTION__);
action_string = action_to_string(action);
if (!action_string) {
pr_debug("kobject attempted to send uevent without action_string!\n");
return -EINVAL;
}
/* search the kset we belong to */
top_kobj = kobj;
while (!top_kobj->kset && top_kobj->parent) {
top_kobj = top_kobj->parent;
}
if (!top_kobj->kset) {
pr_debug("kobject attempted to send uevent without kset!\n");
return -EINVAL;
}
kset = top_kobj->kset;
uevent_ops = kset->uevent_ops;
/* skip the event, if the filter returns zero. */
if (uevent_ops && uevent_ops->filter)
if (!uevent_ops->filter(kset, kobj)) {
pr_debug("kobject filter function caused the event to drop!\n");
return 0;
}
/* originating subsystem */
if (uevent_ops && uevent_ops->name)
subsystem = uevent_ops->name(kset, kobj);
else
subsystem = kobject_name(&kset->kobj);
if (!subsystem) {
pr_debug("unset subsytem caused the event to drop!\n");
return 0;
}
/* environment index */
envp = kzalloc(NUM_ENVP * sizeof (char *), GFP_KERNEL);
if (!envp)
return -ENOMEM;
/* environment values */
buffer = kmalloc(BUFFER_SIZE, GFP_KERNEL);
if (!buffer) {
retval = -ENOMEM;
goto exit;
}
/* complete object path */
devpath = kobject_get_path(kobj, GFP_KERNEL);
if (!devpath) {
retval = -ENOENT;
goto exit;
}
/* event environemnt for helper process only */
envp[i++] = "HOME=/";
envp[i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
/* default keys */
scratch = buffer;
envp [i++] = scratch;
scratch += sprintf(scratch, "ACTION=%s", action_string) + 1;
envp [i++] = scratch;
scratch += sprintf (scratch, "DEVPATH=%s", devpath) + 1;
envp [i++] = scratch;
scratch += sprintf(scratch, "SUBSYSTEM=%s", subsystem) + 1;
for (j = 0; envp_ext && envp_ext[j]; j++)
envp[i++] = envp_ext[j];
/* just reserve the space, overwrite it after kset call has returned */
envp[i++] = seq_buff = scratch;
scratch += strlen("SEQNUM=18446744073709551616") + 1;
/* let the kset specific function add its stuff */
if (uevent_ops && uevent_ops->uevent) {
retval = uevent_ops->uevent(kset, kobj,
&envp[i], NUM_ENVP - i, scratch,
BUFFER_SIZE - (scratch - buffer));
if (retval) {
pr_debug ("%s - uevent() returned %d\n",
__FUNCTION__, retval);
goto exit;
}
}
/* we will send an event, request a new sequence number */
spin_lock(&sequence_lock);
seq = ++uevent_seqnum;
spin_unlock(&sequence_lock);
sprintf(seq_buff, "SEQNUM=%llu", (unsigned long long)seq);
#if defined(CONFIG_NET)
/* send netlink message */
if (uevent_sock) {
struct sk_buff *skb;
size_t len;
/* allocate message with the maximum possible size */
len = strlen(action_string) + strlen(devpath) + 2;
skb = alloc_skb(len + BUFFER_SIZE, GFP_KERNEL);
if (skb) {
/* add header */
scratch = skb_put(skb, len);
sprintf(scratch, "%s@%s", action_string, devpath);
/* copy keys to our continuous event payload buffer */
for (i = 2; envp[i]; i++) {
len = strlen(envp[i]) + 1;
scratch = skb_put(skb, len);
strcpy(scratch, envp[i]);
}
NETLINK_CB(skb).dst_group = 1;
netlink_broadcast(uevent_sock, skb, 0, 1, GFP_KERNEL);
}
}
#endif
/* call uevent_helper, usually only enabled during early boot */
if (uevent_helper[0]) {
char *argv [3];
argv [0] = uevent_helper;
argv [1] = (char *)subsystem;
argv [2] = NULL;
call_usermodehelper (argv[0], argv, envp, 0);
}
exit:
kfree(devpath);
kfree(buffer);
kfree(envp);
return retval;
}
static int do_sieve_error(int ret,
sieve_interp_t *interp,
void *script_context,
void *message_context,
strarray_t *imapflags,
action_list_t *actions,
notify_list_t *notify_list,
/* notify_action_t *notify_action,*/
int lastaction,
int implicit_keep,
char *actions_string,
const char *errmsg
)
{
if (ret != SIEVE_OK) {
if (lastaction == -1) /* we never executed an action */
snprintf(actions_string+strlen(actions_string),
ACTIONS_STRING_LEN-strlen(actions_string),
"script execution failed: %s\n",
errmsg ? errmsg : sieve_errstr(ret));
else
snprintf(actions_string+strlen(actions_string),
ACTIONS_STRING_LEN-strlen(actions_string),
"%s action failed: %s\n",
action_to_string(lastaction),
errmsg ? errmsg : sieve_errstr(ret));
}
/* Process notify actions */
if (interp->notify && notify_list)
{
notify_list_t *n = notify_list;
int notify_ret = SIEVE_OK;
while (n != NULL)
{
if (n->isactive)
{
lastaction = ACTION_NOTIFY;
notify_ret = send_notify_callback(interp,
message_context,
script_context,n,
actions_string, &errmsg);
ret |= notify_ret;
}
n = n->next;
}
if (notify_list) free_notify_list(notify_list);
notify_list = NULL; /* don't try any notifications again */
if (notify_ret != SIEVE_OK)
return do_sieve_error(ret, interp,
script_context, message_context,
imapflags, actions, notify_list, lastaction,
implicit_keep, actions_string, errmsg);
}
if ((ret != SIEVE_OK) && interp->execute_err) {
char buf[ERR_BUF_SIZE];
if (lastaction == -1) /* we never executed an action */
snprintf(buf, ERR_BUF_SIZE,
"%s", errmsg ? errmsg : sieve_errstr(ret));
else {
if (interp->lastitem) {
snprintf(buf, ERR_BUF_SIZE, "%s (%s): %s",
action_to_string(lastaction), interp->lastitem,
errmsg ? errmsg : sieve_errstr(ret));
}
else {
snprintf(buf, ERR_BUF_SIZE, "%s: %s",
action_to_string(lastaction),
errmsg ? errmsg : sieve_errstr(ret));
}
}
ret |= interp->execute_err(buf, interp->interp_context,
script_context, message_context);
}
if (implicit_keep) {
sieve_keep_context_t keep_context;
int keep_ret;
keep_context.imapflags = imapflags;
lastaction = ACTION_KEEP;
keep_ret = interp->keep(&keep_context, interp->interp_context,
script_context, message_context, &errmsg);
ret |= keep_ret;
if (keep_ret == SIEVE_OK)
snprintf(actions_string+strlen(actions_string),
ACTIONS_STRING_LEN-strlen(actions_string),
"Kept\n");
else {
implicit_keep = 0; /* don't try an implicit keep again */
return do_sieve_error(ret, interp,
script_context, message_context,
imapflags, actions, notify_list, lastaction,
implicit_keep, actions_string, errmsg);
}
}
if (actions)
free_action_list(actions);
return ret;
}
void handle_cr4_event(ikgt_event_info_t *event_info)
{
uint64_t new_cr4_value;
uint64_t cur_cr4_value;
uint64_t diff;
ikgt_cpu_event_info_t *cpuinfo;
ikgt_vmcs_guest_state_reg_id_t operand_reg_id;
ikgt_status_t status;
int i, tmp, str_count;
policy_entry_t *entry;
policy_cr4_ctx ctx;
char log_entry_message[LOG_MESSAGE_SIZE];
char access[MAX_ACCESS_BUF_SIZE], action[MAX_ACTION_BUF_SIZE];
event_info->response = IKGT_EVENT_RESPONSE_ALLOW;
g_cr4_count++;
cpuinfo = (ikgt_cpu_event_info_t *) (event_info->event_specific_data);
if (IKGT_CPU_REG_UNKNOWN == cpuinfo->operand_reg) {
ikgt_printf("Error, cpuinfo->operand_reg=IKGT_CPU_REG_UNKNOWN\n");
return;
}
status = read_guest_reg(VMCS_GUEST_STATE_CR4, &cur_cr4_value);
if (IKGT_STATUS_SUCCESS != status) {
return;
}
/* get the VMCS reg ID for the operand */
status = get_vmcs_guest_reg_id(cpuinfo->operand_reg, &operand_reg_id);
if (IKGT_STATUS_SUCCESS != status) {
return;
}
/* read the guest register from VMCS
* new_cr4_value contains the new value to be written to cr4
*/
status = read_guest_reg(operand_reg_id, &new_cr4_value);
if (IKGT_STATUS_SUCCESS != status) {
return;
}
diff = cur_cr4_value ^ new_cr4_value;
if (0 == diff)
return;
ctx.event_info = event_info;
ctx.new_cr4_value = new_cr4_value;
ctx.cur_cr4_value = cur_cr4_value;
ctx.diff = diff;
ctx.log = FALSE;
for (i = 0; i < POLICY_MAX_ENTRIES; i++) {
entry = policy_get_entry_by_index(i);
if ((POLICY_GET_RESOURCE_ID(entry) == RESOURCE_ID_UNKNOWN) || !IS_CR4_ENTRY(entry))
continue;
process_cr4_policy(entry, &ctx);
}
if (ctx.new_cr4_value == cur_cr4_value) {
event_info->response = IKGT_EVENT_RESPONSE_REDIRECT;
}
if (ctx.log) {
tmp = mon_sprintf_s(access, MAX_ACCESS_BUF_SIZE, "write");
action_to_string(&event_info->response, action);
str_count = mon_sprintf_s(log_entry_message, LOG_MESSAGE_SIZE, "resource-name=CR4, access=%s, value=0x%016llx, RIP=0x%016llx, action=%s",
access, new_cr4_value, event_info->vmcs_guest_state.ia32_reg_rip, action);
log_event(log_entry_message, event_info->thread_id);
}
/* If response is skip then return */
if (event_info->response == IKGT_EVENT_RESPONSE_REDIRECT)
return;
status = write_guest_reg(operand_reg_id, ctx.new_cr4_value);
if (IKGT_STATUS_SUCCESS != status) {
ikgt_printf("error, write_guest_reg(%u)=%u\n", operand_reg_id, status);
}
event_info->response = IKGT_EVENT_RESPONSE_ALLOW;
}
/* *********************************************************************
The main method. given a 2D array of the color indecies on the
screen, and the content of the consoel RAM this method
will decides the next action based on the desired algorithm.
The implementation in the superclass takes care of restarting the
game at the end, pressing the first action (if defined), and
countin frames and episodes. It should be called from all
overriden functions
******************************************************************** */
Action PlayerAgent::agent_step( const IntMatrix* screen_matrix,
const IntVect* console_ram,
int frame_number) {
i_frame_counter = frame_number;
if (i_max_num_frames != -1 &&
i_frame_counter > i_max_num_frames) {
cout << "Max number of frames: " << i_max_num_frames
<< " reached" << endl;
end_game();
}
pm_curr_screen_matrix = screen_matrix;
pv_curr_console_ram = console_ram;
// Export the Screen
if ( i_export_screen_frq != 0 &&
(i_frame_counter % i_export_screen_frq == 0) &&
(i_frame_counter >= i_export_screen_after) &&
(i_export_screen_before == -1 ||
i_frame_counter <= i_export_screen_before)) {
export_screen(pm_curr_screen_matrix);
}
f_curr_reward = p_game_settings->get_reward(screen_matrix, console_ram);
pv_reward_per_frame->push_back(f_curr_reward);
if (f_curr_reward > 0) {
b_reward_on_this_frame = true;
} else {
b_reward_on_this_frame = false;
}
if (b_minus_one_zero_reward) {
// Use the -1/0 reward system, i.e. -1 reward on all state, 0 on goal
if (f_curr_reward > 0) {
f_curr_reward = 0.0;
} else {
f_curr_reward = -1.0;
}
}
if (e_episode_status == AGENT_ACTING ||
e_episode_status == ACTION_EXPLOR) {
f_episode_reward += f_curr_reward;
}
if ((b_export_score_screens == true) && b_reward_on_this_frame) {
ostringstream filename;
filename << "reward_screen__frame_" << i_frame_counter <<
"__reward_" << (int) f_curr_reward << ".png";
p_osystem->p_export_screen->save_png(screen_matrix, filename.str());
}
switch (e_episode_status) {
case INITIAL_DELAY:
// We are at the initial delay stage (i.e. do nothing for n frames,
// before even resetting the game)
if (i_initial_delay_counter > 0) {
i_initial_delay_counter--;
return choice <Action> (p_game_settings->pv_possible_actions);;
} else {
// switch to the next state: restarting the game and
// going through the restart delay
e_episode_status = RESTART_DELAY;
i_restart_delay_counter = p_game_settings->i_delay_after_restart;
return RESET;
}
break;
case RESTART_DELAY:
// We are at the restart delay stage (i.e. doing nothing for a
// number frames, after resettng the game)
if (i_restart_delay_counter > 0) {
i_restart_delay_counter--;
return PLAYER_A_NOOP;
} else {
// In the next state, we hand the game to the plaeyr agent
// and either return the FIRST_ACTION, or whatever
// the on_start_of_game() returns
i_episode_first_frame = i_frame_counter;
if (p_game_settings->e_first_action != UNDEFINED) {
e_episode_status = START_EPISODE;
cout << "performed first_action on fram #" << i_frame_counter << endl;
return p_game_settings->e_first_action;
} else {
if (i_init_act_explor_count > 0 &&
i_curr_expl_act_index < i_num_actions) {
e_episode_status = ACTION_EXPLOR;
cout << "Exploring action: " << i_curr_expl_act_index <<
"(" <<
action_to_string((*pv_possible_actions)[i_curr_expl_act_index])
<< ") for " << i_curr_act_frame_count <<
" more frames." << endl;
} else {
e_episode_status = AGENT_ACTING;
}
return on_start_of_game();
}
}
break;
case START_EPISODE: {
// We Are about to start a new episode
if (i_init_act_explor_count > 0 &&
i_curr_expl_act_index < i_num_actions) {
e_episode_status = ACTION_EXPLOR;
cout << "Exploring action: " << i_curr_expl_act_index <<
"(" <<
action_to_string((*pv_possible_actions)[i_curr_expl_act_index])
<< ") for " << i_curr_act_frame_count <<
" more frames." << endl;
} else {
e_episode_status = AGENT_ACTING;
}
return on_start_of_game();
break;
}
case ACTION_EXPLOR:
// We are trying each action for a number of frames
if (i_curr_act_frame_count == 0) {
// Goto the next action
i_curr_expl_act_index++;
i_curr_act_frame_count = i_init_act_explor_count;
if (i_curr_expl_act_index == i_num_actions) {
// We have tried all the actions
e_episode_status = AGENT_ACTING;
} else {
cout << "Exploring action: " << i_curr_expl_act_index
<< "("
<< action_to_string((*pv_possible_actions)[i_curr_expl_act_index])
<< ") for " << i_curr_act_frame_count << " frames." << endl;
}
} else {
i_curr_act_frame_count--;
}
// NOTE: no break here! we still have to check for the end of the
// game.
case AGENT_ACTING: {
// We are acting in the word.
// See if the game has eneded, and if so, restart the game
bool game_ended = p_game_settings->is_end_of_game(screen_matrix,
console_ram, i_frame_counter);
if (b_end_game_with_score && b_reward_on_this_frame) {
game_ended = true;
}
int num_frames_in_episode = i_frame_counter - i_episode_first_frame;
if (i_max_num_frames_per_episode != -1 &&
num_frames_in_episode > i_max_num_frames_per_episode) {
cout << "Reached maximum frames per episode. "<< endl;
p_game_settings->reset_state();
game_ended = true;
}
if (game_ended) {
// take care of end-of-game business
on_end_of_game();
// switch to the next state: restarting the game and
// going through the restart delay
e_episode_status = RESTART_DELAY;
i_restart_delay_counter=p_game_settings->i_delay_after_restart;
return RESET;
} else {
if (b_do_bg_detection) {
// Send screen for background detection
p_background_detect->get_new_screen(pm_curr_screen_matrix);
if (p_background_detect->is_bg_extraction_complete()) {
cout << "Background Detection Complete." << endl;
end_game();
}
}
if (b_do_class_disc) {
// Send screen fro background detection
p_class_dicovery->get_new_screen(pm_curr_screen_matrix,
i_frame_counter);
if (p_class_dicovery->is_class_discovery_complete()) {
cout << "Class Discovery Complete." << endl;
end_game();
}
}
return UNDEFINED; // let the subclass decide what to do
}
break;
}
default:
assert(false); // we should never reach here
}
assert(false); // we should never reach here
return UNDEFINED;
}
static gboolean
_dispatcher_call (DispatcherAction action,
gboolean blocking,
NMSettingsConnection *settings_connection,
NMConnection *applied_connection,
NMDevice *device,
NMConnectivityState connectivity_state,
const char *vpn_iface,
NMProxyConfig *vpn_proxy_config,
NMIP4Config *vpn_ip4_config,
NMIP6Config *vpn_ip6_config,
DispatcherFunc callback,
gpointer user_data,
guint *out_call_id)
{
GVariant *connection_dict;
GVariantBuilder connection_props;
GVariantBuilder device_props;
GVariantBuilder device_proxy_props;
GVariantBuilder device_ip4_props;
GVariantBuilder device_ip6_props;
GVariant *device_dhcp4_props = NULL;
GVariant *device_dhcp6_props = NULL;
GVariantBuilder vpn_proxy_props;
GVariantBuilder vpn_ip4_props;
GVariantBuilder vpn_ip6_props;
DispatchInfo *info = NULL;
gboolean success = FALSE;
GError *error = NULL;
static guint request_counter = 0;
guint reqid = ++request_counter;
if (!dispatcher_proxy)
return FALSE;
/* Wrapping protection */
if (G_UNLIKELY (!reqid))
reqid = ++request_counter;
g_assert (!blocking || (!callback && !user_data));
_ensure_requests ();
/* All actions except 'hostname' and 'connectivity-change' require a device */
if ( action == DISPATCHER_ACTION_HOSTNAME
|| action == DISPATCHER_ACTION_CONNECTIVITY_CHANGE) {
_LOGD ("(%u) dispatching action '%s'%s",
reqid, action_to_string (action),
blocking
? " (blocking)"
: (callback ? " (with callback)" : ""));
} else {
g_return_val_if_fail (NM_IS_DEVICE (device), FALSE);
_LOGD ("(%u) (%s) dispatching action '%s'%s",
reqid,
vpn_iface ? vpn_iface : nm_device_get_iface (device),
action_to_string (action),
blocking
? " (blocking)"
: (callback ? " (with callback)" : ""));
}
if (!_get_monitor_by_action(action)->has_scripts) {
if (blocking == FALSE && (out_call_id || callback)) {
info = g_malloc0 (sizeof (*info));
info->action = action;
info->request_id = reqid;
info->callback = callback;
info->user_data = user_data;
info->idle_id = g_idle_add (dispatcher_idle_cb, info);
_LOGD ("(%u) simulate request; no scripts in %s", reqid, _get_monitor_by_action(action)->dir);
} else
_LOGD ("(%u) ignoring request; no scripts in %s", reqid, _get_monitor_by_action(action)->dir);
success = TRUE;
goto done;
}
if (applied_connection)
connection_dict = nm_connection_to_dbus (applied_connection, NM_CONNECTION_SERIALIZE_NO_SECRETS);
else
connection_dict = g_variant_new_array (G_VARIANT_TYPE ("{sa{sv}}"), NULL, 0);
g_variant_builder_init (&connection_props, G_VARIANT_TYPE_VARDICT);
if (settings_connection) {
const char *connection_path;
const char *filename;
connection_path = nm_connection_get_path (NM_CONNECTION (settings_connection));
if (connection_path) {
g_variant_builder_add (&connection_props, "{sv}",
NMD_CONNECTION_PROPS_PATH,
g_variant_new_object_path (connection_path));
}
filename = nm_settings_connection_get_filename (settings_connection);
if (filename) {
g_variant_builder_add (&connection_props, "{sv}",
NMD_CONNECTION_PROPS_FILENAME,
g_variant_new_string (filename));
}
if (nm_settings_connection_get_nm_generated_assumed (settings_connection)) {
g_variant_builder_add (&connection_props, "{sv}",
NMD_CONNECTION_PROPS_EXTERNAL,
g_variant_new_boolean (TRUE));
}
}
g_variant_builder_init (&device_props, G_VARIANT_TYPE_VARDICT);
g_variant_builder_init (&device_proxy_props, G_VARIANT_TYPE_VARDICT);
g_variant_builder_init (&device_ip4_props, G_VARIANT_TYPE_VARDICT);
g_variant_builder_init (&device_ip6_props, G_VARIANT_TYPE_VARDICT);
g_variant_builder_init (&vpn_proxy_props, G_VARIANT_TYPE_VARDICT);
g_variant_builder_init (&vpn_ip4_props, G_VARIANT_TYPE_VARDICT);
g_variant_builder_init (&vpn_ip6_props, G_VARIANT_TYPE_VARDICT);
/* hostname and connectivity-change actions don't send device data */
if ( action != DISPATCHER_ACTION_HOSTNAME
&& action != DISPATCHER_ACTION_CONNECTIVITY_CHANGE) {
fill_device_props (device,
&device_props,
&device_proxy_props,
&device_ip4_props,
&device_ip6_props,
&device_dhcp4_props,
&device_dhcp6_props);
if (vpn_ip4_config || vpn_ip6_config) {
fill_vpn_props (vpn_proxy_config,
vpn_ip4_config,
vpn_ip6_config,
&vpn_proxy_props,
&vpn_ip4_props,
&vpn_ip6_props);
}
}
if (!device_dhcp4_props)
device_dhcp4_props = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0));
if (!device_dhcp6_props)
device_dhcp6_props = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0));
/* Send the action to the dispatcher */
if (blocking) {
GVariant *ret;
GVariantIter *results;
ret = _nm_dbus_proxy_call_sync (dispatcher_proxy, "Action",
g_variant_new ("(s@a{sa{sv}}a{sv}a{sv}a{sv}a{sv}a{sv}@a{sv}@a{sv}ssa{sv}a{sv}a{sv}b)",
action_to_string (action),
connection_dict,
&connection_props,
&device_props,
&device_proxy_props,
&device_ip4_props,
&device_ip6_props,
device_dhcp4_props,
device_dhcp6_props,
nm_connectivity_state_to_string (connectivity_state),
vpn_iface ? vpn_iface : "",
&vpn_proxy_props,
&vpn_ip4_props,
&vpn_ip6_props,
nm_logging_enabled (LOGL_DEBUG, LOGD_DISPATCH)),
G_VARIANT_TYPE ("(a(sus))"),
G_DBUS_CALL_FLAGS_NONE, CALL_TIMEOUT,
NULL, &error);
if (ret) {
g_variant_get (ret, "(a(sus))", &results);
dispatcher_results_process (reqid, action, results);
g_variant_iter_free (results);
g_variant_unref (ret);
success = TRUE;
} else {
g_dbus_error_strip_remote_error (error);
_LOGW ("(%u) failed: %s", reqid, error->message);
g_clear_error (&error);
success = FALSE;
}
} else {
info = g_malloc0 (sizeof (*info));
info->action = action;
info->request_id = reqid;
info->callback = callback;
info->user_data = user_data;
g_dbus_proxy_call (dispatcher_proxy, "Action",
g_variant_new ("(s@a{sa{sv}}a{sv}a{sv}a{sv}a{sv}a{sv}@a{sv}@a{sv}ssa{sv}a{sv}a{sv}b)",
action_to_string (action),
connection_dict,
&connection_props,
&device_props,
&device_proxy_props,
&device_ip4_props,
&device_ip6_props,
device_dhcp4_props,
device_dhcp6_props,
nm_connectivity_state_to_string (connectivity_state),
vpn_iface ? vpn_iface : "",
&vpn_proxy_props,
&vpn_ip4_props,
&vpn_ip6_props,
nm_logging_enabled (LOGL_DEBUG, LOGD_DISPATCH)),
G_DBUS_CALL_FLAGS_NONE, CALL_TIMEOUT,
NULL, dispatcher_done_cb, info);
success = TRUE;
}
g_variant_unref (device_dhcp4_props);
g_variant_unref (device_dhcp6_props);
done:
if (success && info) {
/* Track the request in case of cancelation */
g_hash_table_insert (requests, GUINT_TO_POINTER (info->request_id), info);
if (out_call_id)
*out_call_id = info->request_id;
} else if (out_call_id)
*out_call_id = 0;
return success;
}
/**
* kobject_hotplug - notify userspace by executing /sbin/hotplug
*
* @action: action that is happening (usually "ADD" or "REMOVE")
* @kobj: struct kobject that the action is happening to
*/
void kobject_hotplug(struct kobject *kobj, enum kobject_action action)
{
char *argv [3];
char **envp = NULL;
char *buffer = NULL;
char *scratch;
int i = 0;
int retval;
char *kobj_path = NULL;
char *name = NULL;
char *action_string;
u64 seq;
struct kobject *top_kobj = kobj;
struct kset *kset;
if (!top_kobj->kset && top_kobj->parent) {
do {
top_kobj = top_kobj->parent;
} while (!top_kobj->kset && top_kobj->parent);
}
if (top_kobj->kset && top_kobj->kset->hotplug_ops)
kset = top_kobj->kset;
else
return;
/* If the kset has a filter operation, call it.
Skip the event, if the filter returns zero. */
if (kset->hotplug_ops->filter) {
if (!kset->hotplug_ops->filter(kset, kobj))
return;
}
pr_debug ("%s\n", __FUNCTION__);
action_string = action_to_string(action);
if (!action_string)
return;
envp = kmalloc(NUM_ENVP * sizeof (char *), GFP_KERNEL);
if (!envp)
return;
memset (envp, 0x00, NUM_ENVP * sizeof (char *));
buffer = kmalloc(BUFFER_SIZE, GFP_KERNEL);
if (!buffer)
goto exit;
if (kset->hotplug_ops->name)
name = kset->hotplug_ops->name(kset, kobj);
if (name == NULL)
name = kset->kobj.name;
argv [0] = hotplug_path;
argv [1] = name;
argv [2] = NULL;
/* minimal command environment */
envp [i++] = "HOME=/";
envp [i++] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin";
scratch = buffer;
envp [i++] = scratch;
scratch += sprintf(scratch, "ACTION=%s", action_string) + 1;
kobj_path = kobject_get_path(kobj, GFP_KERNEL);
if (!kobj_path)
goto exit;
envp [i++] = scratch;
scratch += sprintf (scratch, "DEVPATH=%s", kobj_path) + 1;
spin_lock(&sequence_lock);
seq = ++hotplug_seqnum;
spin_unlock(&sequence_lock);
envp [i++] = scratch;
scratch += sprintf(scratch, "SEQNUM=%lld", (long long)seq) + 1;
envp [i++] = scratch;
scratch += sprintf(scratch, "SUBSYSTEM=%s", name) + 1;
if (kset->hotplug_ops->hotplug) {
/* have the kset specific function add its stuff */
retval = kset->hotplug_ops->hotplug (kset, kobj,
&envp[i], NUM_ENVP - i, scratch,
BUFFER_SIZE - (scratch - buffer));
if (retval) {
pr_debug ("%s - hotplug() returned %d\n",
__FUNCTION__, retval);
goto exit;
}
}
pr_debug ("%s: %s %s %s %s %s %s %s\n", __FUNCTION__, argv[0], argv[1],
envp[0], envp[1], envp[2], envp[3], envp[4]);
send_uevent(action_string, kobj_path, buffer, scratch - buffer, GFP_KERNEL);
if (!hotplug_path[0])
goto exit;
retval = call_usermodehelper (argv[0], argv, envp, 0);
if (retval)
pr_debug ("%s - call_usermodehelper returned %d\n",
__FUNCTION__, retval);
exit:
kfree(kobj_path);
kfree(buffer);
kfree(envp);
return;
}
void DQN::Update() {
std::cout << "iteration: " << current_iter_++ << std::endl;
// Sample transitions from replay memory
std::vector<int> transitions;
transitions.reserve(kMinibatchSize);
for (auto i = 0; i < kMinibatchSize; ++i) {
const auto random_transition_idx =
std::uniform_int_distribution<int>(0, replay_memory_.size() - 1)(
random_engine);
transitions.push_back(random_transition_idx);
}
// Compute target values: max_a Q(s',a)
std::vector<InputFrames> target_last_frames_batch;
for (const auto idx : transitions) {
const auto& transition = replay_memory_[idx];
if (!std::get<3>(transition)) {
// This is a terminal state
continue;
}
// Compute target value
InputFrames target_last_frames;
for (auto i = 0; i < kInputFrameCount - 1; ++i) {
target_last_frames[i] = std::get<0>(transition)[i + 1];
}
target_last_frames[kInputFrameCount - 1] = std::get<3>(transition).get();
target_last_frames_batch.push_back(target_last_frames);
}
const auto actions_and_values =
SelectActionGreedily(target_last_frames_batch);
FramesLayerInputData frames_input;
TargetLayerInputData target_input;
FilterLayerInputData filter_input;
std::fill(target_input.begin(), target_input.end(), 0.0f);
std::fill(filter_input.begin(), filter_input.end(), 0.0f);
auto target_value_idx = 0;
for (auto i = 0; i < kMinibatchSize; ++i) {
const auto& transition = replay_memory_[transitions[i]];
const auto action = std::get<1>(transition);
assert(static_cast<int>(action) < kOutputCount);
const auto reward = std::get<2>(transition);
assert(reward >= -1.0 && reward <= 1.0);
const auto target = std::get<3>(transition) ?
reward + gamma_ * actions_and_values[target_value_idx++].second :
reward;
assert(!std::isnan(target));
target_input[i * kOutputCount + static_cast<int>(action)] = target;
filter_input[i * kOutputCount + static_cast<int>(action)] = 1;
VLOG(1) << "filter:" << action_to_string(action) << " target:" << target;
for (auto j = 0; j < kInputFrameCount; ++j) {
const auto& frame_data = std::get<0>(transition)[j];
std::copy(
frame_data->begin(),
frame_data->end(),
frames_input.begin() + i * kInputDataSize +
j * kCroppedFrameDataSize);
}
}
InputDataIntoLayers(frames_input, target_input, filter_input);
solver_->Step(1);
// Log the first parameter of each hidden layer
VLOG(1) << "conv1:" <<
net_->layer_by_name("conv1_layer")->blobs().front()->data_at(1, 0, 0, 0);
VLOG(1) << "conv2:" <<
net_->layer_by_name("conv2_layer")->blobs().front()->data_at(1, 0, 0, 0);
VLOG(1) << "ip1:" <<
net_->layer_by_name("ip1_layer")->blobs().front()->data_at(1, 0, 0, 0);
VLOG(1) << "ip2:" <<
net_->layer_by_name("ip2_layer")->blobs().front()->data_at(1, 0, 0, 0);
}
