/**
* This function sets the height, width, x_start, y_start (upper left coords),
* x_end, and y_end (lower right) variables of the student's window. These
* values are used in operations such as mouse movement. returns false on
* failure and does not set variables.
*/
bool populateWindowData(std::string window_name, int& height, int& width,
int& x_start, int& x_end, int& y_start, int& y_end){
if(windowExists(window_name)) {
std::vector<int> height_vec, width_vec, x_start_vec, y_start_vec;
// getWindowData returns a vector of ints associated with the query term
// (e.g. 'Height')
height_vec = getWindowData("Height", window_name);
width_vec = getWindowData("Width", window_name);
//These two values represent the upper left corner
x_start_vec = getWindowData("Absolute upper-left X", window_name);
y_start_vec = getWindowData("Absolute upper-left Y", window_name);
//The lines below should never return false, as the Height, Width, etc.
//fields should always be populated if the window exists. The only way that
// I can see for these to fail would be if xdotool changes or the window
// fails/shuts in this block.
if(height_vec.size() > 0){ height = height_vec[0];} else{ return false; }
if(width_vec.size() > 0) { width = width_vec[0];} else{ return false; }
if(x_start_vec.size() > 0){ x_start= x_start_vec[0];} else{ return false; }
if(y_start_vec.size() > 0){ y_start= y_start_vec[0];} else{ return false; }
//These values represent the upper right corner
x_end = x_start+width;
y_end = y_start + height;
return true;
}
else
{
std::cout << "Attempted to populate window data using a nonexistent window"
<< std::endl;
return false;
}
}
//-------------------------------------------------------------------------------------------------
D2Watcher::D2Watcher(std::string title) : mWinName(title), mD2Hand(0) {
DWORD pID;
if ( windowExists() ) {
GetWindowThreadProcessId (mD2Hwnd, &pID);
mD2Hand = OpenProcess (PROCESS_VM_READ|PROCESS_VM_OPERATION, FALSE, pID);
if (!mD2Hand) return;
if ( !inGame() ) return;
mThisPlayers = readPlayerCount();
mLastPlayers = mThisPlayers;
}
}
/**
* Given a window name, this function takes a screenshot of it if it exists.
* uses screenshot_name to title the image.
*/
bool screenshot(std::string window_name, std::string screenshot_name){
if(windowExists(window_name)){
//if the window hasn't crashed, bring it into focus and screenshot it
std::string command = "wmctrl -R " + window_name + " && scrot "
+ screenshot_name + " -u";
system(command.c_str());
return true;
}
else{
std::cout << "Attempted to screenshot a closed window." << std::endl;
return false;
}
}
/**
* This function was written to allow users to type special keys.
* The function wraps the xdotool key command, which can multipress
* keys. (e.g. ctrl+alt+del)
*/
bool key(std::string toType, std::string window_name)
{
//get window focus then type the string toType.
std::string internal_command = "wmctrl -R " + window_name
+" && xdotool key " + toType;
//for number of presses requested, check that the window exists and that we
//have something to type.
if(windowExists(window_name) && toType != ""){
system(internal_command.c_str());
return true;
} else{
return false;
}
}
int minSubArrayLen(int s, vector<int>& nums) {
if (nums.empty() || s <= 0) return 0;
int l = 1, u = nums.size();
int res = nums.size()+1;
while (l <= u) {
int mid = l + (u-l)/2;
if (windowExists(mid, nums, s)) {
res = mid;
u = mid-1;
} else {
l = mid+1;
}
}
return res == nums.size()+1 ? 0 : res;
}
/**
* Moves the mouse to the upper left of the window associated with windowname
* if it exists.
*/
bool moveMouseToOrigin(std::string window_name){
//populate the window vals to get the center.
int height, width, x_start, x_end, y_start, y_end;
bool success = populateWindowData(window_name, height, width, x_start,x_end,
y_start,y_end);
//wait until the last moment to check window existence.
if(success&& windowExists(window_name)){
success = mouse_move(window_name, x_start, y_start, x_start, x_end, y_start, y_end, false);
return success;
}
else{
std::cout << "Attempted to move mouse to origin of nonexistent window"
<< std::endl;
return false;
}
}
/**
* Centers the mouse on the window associated with windowname if it exists.
*/
bool centerMouse(std::string window_name){
//populate the window vals to get the center.
int height, width, x_start, x_end, y_start, y_end;
bool success = populateWindowData(window_name, height, width, x_start,x_end,
y_start,y_end);
int x_middle = x_start + (width/2);
int y_middle = y_start+(height/2);
//wait until the last moment to check window existence.
if(success && windowExists(window_name)){
success = mouse_move(window_name, x_middle, y_middle, x_start, x_end, y_start, y_end, false);
return success;
}
else{
std::cout << "Attempted to center mouse on a nonexistent window"
<< std::endl;
return false;
}
}
/**
* This function uses xdotool to put the mouse button associated with the int
* button into the 'up' state. Checks to see if the window exists so that we
* don't click on anything that doesn't belong to us.
*/
bool mouseUp(std::string window_name, int button){
//only mouseup on buttons 1,2,3
if(button == 1 || button == 2 || button == 3){
//Only mouse up if the window exists (give the window focus and mouseup)
if(windowExists(window_name)){
std::string command = "wmctrl -R " + window_name
+ " && xdotool mouseup " + std::to_string(button);
system(command.c_str());
return true;
}
else{
std::cout << "Tried to mouse up on a nonexistent window" << std::endl;
return false;
}
}
else{
std::cout << "ERROR: tried to click mouse button " << button << std::endl;
return false;
}
}
/**
* This function uses xdotool to put the mouse button associated with int button
* into the 'down' state. Checks to see if the window exists so that we don't
* click on anything that doesn't belong to us.
*/
bool mouseDown(std::string window_name, int button){
//only mouse down button 1, 2, or 3.
if(button == 1 || button == 2 || button == 3){
//only mouse down if the window exists (bring into focus and mousedown)
if(windowExists(window_name)){
std::string command = "wmctrl -R " + window_name
+ " && xdotool mousedown " + std::to_string(button);
system(command.c_str());
return true;
}
else{
std::cout << "Tried to mouse down on a nonexistent window." << std::endl;
return false;
}
}
else{
std::cout << "ERROR: tried to click nonexistent mouse button "
<< button << std::endl;
return false;
}
}
/**
* This function moves the mouse to moved_mouse_x, moved_mouse_y, clamping
* between x_start x_end and y_start y_end.
* NOTE: EXPECTS MOVED VARIALBES ALREADY IN WINDOW COORDINATES
* This is done in the takeAction function
*/
bool mouse_move(std::string window_name, int moved_mouse_x, int moved_mouse_y,
int x_start, int x_end, int y_start, int y_end, bool no_clamp){
//only move the mouse if the window exists. (get focus and mousemove.)
if(windowExists(window_name)){
std::vector<int> current_mouse_position = getMouseLocation();
if(current_mouse_position.size() < 2){
return false;
}
//Explicit clamping.
if(moved_mouse_x > x_end || moved_mouse_x < x_start){
std::cout << "Attempted to move outside of the window bounds." << std::endl;
return false;
}
//Explicit clamping
if(moved_mouse_y > y_end || moved_mouse_y < y_start){
std::cout << "Attempted to move outside of the window bounds." << std::endl;
return false;
}
if(current_mouse_position[0] == moved_mouse_x && current_mouse_position[1] == moved_mouse_y){
std::cout << "mouse was already in position. Skipping movement." << std::endl;
}
else{
std::string command = "wmctrl -R " + window_name +
" && xdotool mousemove --sync " + std::to_string(moved_mouse_x) + " "
+ std::to_string(moved_mouse_y);
system(command.c_str());
}
return true;
}
else{
std::cout << "Attempted to move mouse on a nonexistent window."
<< std::endl;
return false;
}
}
/**
* Given the title of an xwininfo data field, returns its integer value.
* (basic fields are height, width, etc, for a full list,
* run xwininfo -name <valid window name> or xwininfo and then click a window.
* Should only be used for integer fields.)
*/
std::vector<int> getWindowData(std::string data_string,
std::string window_name){
//Test that the window is still active.
if(windowExists(window_name)){
//use xwininfo to get information about windowname, then grep the data
//string we're looking for
std::string command = "xwininfo -name \"" + window_name + "\" | grep \""
+ data_string +"\"";
std::string value_string = output_of_system_command(command.c_str());
//grep will return nothing if the field doesn't exist, so return nothing.
if(value_string == ""){
std::vector<int> empty;
return empty;
}
//if grep returned something, get all the ints in what it returned
return extractIntsFromString(value_string);
}
else{
//if the window doesn't exist, just return an empty vector (nonexistent
// windows should be handled in one timestep per execute() in execute.cpp
std::vector<int> empty;
return empty;
}
}
/**
* This function processes the 'type' action, which types a quoted string one
* character at a time an optional number of times with an optional delay
* between repetitions. Because of the delay, we need all parameters necessary
* for a call to execute.cpp's delayAndMemCheck.
*/
bool type(std::string toType, float delay, int presses, std::string window_name, int childPID,
float &elapsed, float& next_checkpoint, float seconds_to_run,
int& rss_memory, int allowed_rss_memory, int& memory_kill, int& time_kill,
std::ostream &logfile){
delay = delay * 1000000;
toType = "\"" + toType + "\"";
//get window focus then type the string toType.
std::string internal_command = "wmctrl -R " + window_name
+" && xdotool type " + toType;
bool killed = false;
//for number of presses requested, check that the window exists and that we
//have something to type.
for(int i = 0; i < presses; i++){
if(windowExists(window_name) && toType != ""){
system(internal_command.c_str());
}
else{
std::cout << "Attempted to type on nonexistent window" << std::endl;
return false;
}
//allow this to run so that delays occur as expected.
if(i != presses-1){
killed = delay_and_mem_check(delay, childPID, elapsed, next_checkpoint,
seconds_to_run, rss_memory, allowed_rss_memory, memory_kill,time_kill, logfile);
}
if(killed){
return false;
}
}
return true;
}
/**
* The 'delta' version of the click and drag command. This function moves an xy
* distance from a startpoint. This distance is 'wrapping', so if it is outside
* of the window, we mouseup, return to the start position, mousedown, and then
* move again. We give a one pixel border at each side of the window and clamp
* using that value to avoid accidental resizing.
*/
bool clickAndDragDelta(std::string window_name, nlohmann::json action){
//get the values of the student's window.
int x_start, x_end, y_start, y_end, mouse_button;
bool no_clamp = false;
bool success = populateClickAndDragValues(action, window_name, x_start,
x_end, y_start, y_end, mouse_button, no_clamp);
//if we can't populate the click and drag values, do nothing.
if(!success){
std::cout << "Could not populate the click and drag values."<< std::endl;
return false;
}
//Define the corners of our window. (We use vectors as 2d points.)
std::vector<int> upper_left, upper_right, lower_left, lower_right;
upper_left.push_back(x_start); upper_left.push_back(y_start);
upper_right.push_back(x_end); upper_right.push_back(y_start);
lower_left.push_back(x_start); lower_left.push_back(y_end);
lower_right.push_back(x_end); lower_right.push_back(y_end);
//delta version, 2 values movement x and movement y.
int amt_x_movement_remaining = action.value("x_distance", 0);
int amt_y_movement_remaining = action.value("y_distance", 0);
std::string start_location = action.value("start_location", "center");
//This shouldn't fail unless there isn't a mouse.
std::string mouse_location_string = output_of_system_command("xdotool getmouselocation");
std::vector<int> xy = extractIntsFromString(mouse_location_string);
//if the mouse isn't detected, fail.
if(xy.size() < 2){
std::cout << "Mouse coordinates couldn't be found. Mouse undetected."
<< std::endl;
return false;
}
//get the current mouse location
int start_mouse_x = xy[0];
int start_mouse_y = xy[1];
//clamp the mouse within the screen (and move in by a pixel).
clamp(start_mouse_x, x_start+1, x_end-1);
clamp(start_mouse_y, y_start+1, y_end-1);
//get the center of the window
int width = x_end - x_start;
int height = y_end - y_start;
int x_middle = x_start + (width/2);
int y_middle = y_start+(height/2);
//NOTE: check my arithmetic.
/**
* The process that this algorithm goes through is as follows:
* 1) Determine the slope of the dragged line and its distance.
* 2) while we have not traversed the necessary distance
* 3) project a line from the current mouse position towards the end
* position with length equal to the remaining distance.
* 4) Now that we have a line segment defined, find where/if it intersects
* any of the window's edges
* 5) if it does intersect, cut it off at the point of intersection, and only
* drag that far. Else, if it doesn't intersect, we can assume we are
* inside of the window, due to the clamp, and can drag.
* 6) update remaining distance and continue to loop.
*/
//rise / run
float slope=(float)amt_y_movement_remaining/(float)amt_x_movement_remaining;
float total_distance_needed = sqrt(pow(amt_x_movement_remaining, 2)
+ pow (amt_y_movement_remaining, 2));
//remaining distance needed.
float remaining_distance_needed = total_distance_needed;
int action_start_x = (start_location == "current") ? start_mouse_x : x_middle;
int action_start_y = (start_location == "current") ? start_mouse_y : y_middle;
std::cout << "start x " << start_mouse_x << " our x " << action_start_x;
std::cout << "start y " << start_mouse_y << " our y " << action_start_y;
//The functions called within this loop will not fire if the window doesn't
// exist. This check just short circuits to avoid additional printing.
while(remaining_distance_needed >= 1 && windowExists(window_name)){
int curr_x = action_start_x;
int curr_y = action_start_y;
int moved_mouse_x, moved_mouse_y;
//reset the mouse to the start location.
mouse_move(window_name, action_start_x, action_start_y, x_start, x_end, y_start, y_end,
false);
//determine how far we've come.
float fraction_of_distance_remaining = remaining_distance_needed
/ total_distance_needed;
//project in the direction of the move to find the end of our line segment.
float projected_x = action_start_x + (amt_x_movement_remaining * fraction_of_distance_remaining);
float projected_y = action_start_y + (amt_y_movement_remaining * fraction_of_distance_remaining);
//we are using vectors as 2d points.
std::vector<int> current_point, projected_point;
current_point.push_back(curr_x); current_point.push_back(curr_y);
projected_point.push_back(projected_x);
projected_point.push_back(projected_y);
std::vector<float> intersection_point;
intersection_point=getLineIntersectionPoint(current_point,projected_point,
upper_left, upper_right);
/**
* TODO make this block smaller.
* These if statements just test all edges of the window against our
* projected line.
*/
//found is just a quick short-circuit to keep the code from ballooning.
bool found = false;
if(intersection_point.size() != 0){ //TOP
std::cout << "intersected top" << std::endl;
moved_mouse_x = (int)intersection_point[0];
moved_mouse_y = (int)intersection_point[1];
found = true;
}
if(!found) //RIGHT
{
intersection_point = getLineIntersectionPoint(current_point,
projected_point, upper_right, lower_right);
if(intersection_point.size() != 0){
std::cout << "intersected right" << std::endl;
moved_mouse_x = (int)intersection_point[0];
moved_mouse_y = (int)intersection_point[1];
found = true;
}
}
if(!found) //BOTTOM
{
intersection_point = getLineIntersectionPoint(current_point,
projected_point, lower_right, lower_left);
if(intersection_point.size() != 0){
std::cout << "intersected bottom" << std::endl;
moved_mouse_x = (int)intersection_point[0];
moved_mouse_y = (int)intersection_point[1];
found = true;
}
}
if(!found) //LEFT
{
intersection_point = getLineIntersectionPoint(current_point,
projected_point, lower_left, upper_left);
if(intersection_point.size() != 0){
std::cout << "intersected left" << std::endl;
moved_mouse_x = (int)intersection_point[0];
moved_mouse_y = (int)intersection_point[1];
found = true;
}
}
//if we didn't intersect, we are inside of the box (guaranteed by clamp)
// so we can move freely.
if(!found)
{
std::cout << "No intersection at all"<< std::endl;
moved_mouse_x = projected_x;
moved_mouse_y = projected_y;
}
//the distance we can move
float distance_of_move = sqrt(pow(moved_mouse_x - action_start_x, 2)
+ pow (moved_mouse_y - action_start_y, 2));
//we are moving distance_of_move
remaining_distance_needed -= distance_of_move;
std::cout << "after the move, we had " << remaining_distance_needed
<< " distance left " << std::endl;
mouseDown(window_name,mouse_button); //click
mouse_move(window_name, moved_mouse_x, moved_mouse_y,x_start, x_end, //drag
y_start, y_end, false);
mouseUp(window_name,mouse_button); //release
} //end loop.
//to preserve backwards compatibility.
if(start_location != "current"){
//put the mouse back where we found it.
mouse_move(window_name, start_mouse_x, start_mouse_y, x_start, x_end, y_start, y_end,false);
}
return true;
}
/**
* The central routing function for for all actions. Takes in a vector of
* actions and the # of actions taken thus far. It then passes the current
* action to be taken through tests to see which function to route to.
* This function requires all parameters to for execute.cpp's delayAndMemCheck
* function.
*/
void takeAction(const std::vector<nlohmann::json>& actions, int& actions_taken,
std::string window_name, int childPID,
float &elapsed, float& next_checkpoint, float seconds_to_run,
int& rss_memory, int allowed_rss_memory, int& memory_kill, int& time_kill,
std::ostream &logfile){
//We get the window data at every step in case it has changed size.
//if we make it past this check, we'll assume an action has been taken.
if(!windowExists(window_name)){
return;
}
nlohmann::json action = actions[actions_taken];
// std::vector<std::string> vec = stringOrArrayOfStrings(action, "action");
// if (vec.size() == 0){
// std::cout << "ERROR: poorly formatted action (no 'action' type specified)" << std::endl;
// action_name = "INVALID: NAME MISSING";
// }
// else{
// std::string action_name = vec[0];
// }
std::string action_name = action.value("action", "ACTION_NOT_SPECIFIED");
std::cout <<"Taking action "<<actions_taken+1<<" of "<<actions.size() <<": "<< action_name<< std::endl;
float delay_time = 0;
bool success = false;
//DELAY
if(action_name == "delay"){
float time_in_secs = action.value("seconds", 0);
delay_time = delay(time_in_secs);
success = true;
}
//SCREENSHOT
else if(action_name == "screenshot"){
std::string screenshot_name = action["name"];
success = screenshot(window_name, screenshot_name);
}
//TYPE
else if(action_name == "type"){
float delay_in_secs = action["delay_in_seconds"];
int presses = action["presses"];
std::string string_to_type = action["string"];
success = type(string_to_type, delay_in_secs, presses, window_name,childPID,elapsed, next_checkpoint,
seconds_to_run, rss_memory, allowed_rss_memory, memory_kill, time_kill, logfile);
}
//KEY
else if(action_name == "key"){
std::string key_to_type = action["key_combination"];
success = key(key_to_type, window_name);
}
//CLICK AND DRAG
else if(action_name == "click and drag"){
success = clickAndDragAbsolute(window_name,action);
}
else if(action_name == "click and drag delta"){
success = clickAndDragDelta(window_name,action);
}
//CLICK
else if(action_name == "click"){
std::string mouse_button = action["mouse_button"];
if(mouse_button == "left"){
success = click(window_name, 1);
}
else if(mouse_button == "middle"){
success = click(window_name, 2);
}
else if(mouse_button == "right"){
success = click(window_name, 3);
}
else{
success = click(window_name, 1);
}
}
//MOUSE MOVE
else if(action_name == "move mouse"){
//TODO: implement later if deemed prudent.
bool no_clamp = false;
int moved_x = action["end_x"];
int moved_y = action["end_y"];
int height, width, x_start, x_end, y_start, y_end;
bool populated = populateWindowData(window_name, height, width, x_start, x_end, y_start, y_end);
if(populated){
moved_x += x_start;
moved_y += y_start;
success = mouse_move(window_name, moved_x, moved_y, x_start, x_end, y_start, y_end, no_clamp);
}
}
//CENTER
else if(action_name == "center"){
success = centerMouse(window_name);
}
//ORIGIN
else if(action_name == "origin"){
success = moveMouseToOrigin(window_name);
}
else if(action_name == "gif"){
std::string gif_name = action["name"];
float duration_in_seconds = action["seconds"];
int fps = action["frames_per_second"];
bool save_pngs = action["preserve_individual_frames"];
success = make_gif(window_name, gif_name, duration_in_seconds, fps, save_pngs,
childPID, elapsed, next_checkpoint, seconds_to_run,
rss_memory, allowed_rss_memory, memory_kill, time_kill,
logfile);
}
//BAD COMMAND
else{
std::cout << "ERROR: ill formatted action: " << actions[actions_taken] << std::endl;
}
if(success){
std::cout << "The action was successful" << std::endl;
actions_taken++;
}else{
std::cout << "The action was unsuccessful" << std::endl;
}
delay_and_mem_check(delay_time, childPID, elapsed, next_checkpoint,
seconds_to_run, rss_memory, allowed_rss_memory, memory_kill, time_kill,logfile);
}
