Beispiel #1
0
//--------------------------------------------------------------
//copy map (of GridWorld)to maze (of LPA*)
void copyDisplayMapToMaze(GridWorld &gWorld, LpaStar* lpa){
	for(int i=0; i < gWorld.getGridWorldRows(); i++){
	   for(int j=0; j < gWorld.getGridWorldCols(); j++){
			lpa->maze[i][j].type = gWorld.map[i][j].type;
			lpa->maze[i][j].x = gWorld.map[i][j].col;
			lpa->maze[i][j].y = gWorld.map[i][j].row;
			
		   //lpa->maze[i][j].g = gWorld.map[i][j].g;
			//lpa->maze[i][j].rhs = gWorld.map[i][j].rhs;
		}
	}
	
	vertex startV = gWorld.getStartVertex();
	vertex goalV = gWorld.getGoalVertex();
	
	//lpa->start->g = gWorld.map[startV.row][startV.col].g ;
	//lpa->start->rhs = gWorld.map[startV.row][startV.col].rhs ;
	lpa->start->x = gWorld.map[startV.row][startV.col].col;
	lpa->start->y = gWorld.map[startV.row][startV.col].row;
	
	//lpa->goal->g = gWorld.map[goalV.row][goalV.col].g;
	//lpa->goal->rhs = gWorld.map[goalV.row][goalV.col].rhs;
	lpa->goal->x = gWorld.map[goalV.row][goalV.col].col;
	lpa->goal->y = gWorld.map[goalV.row][goalV.col].row;
	
}
Beispiel #2
0
int main() {
    GridWorld world;

    // This is optional, and should make solving the model almost instantaneous.
    // Unfortunately, since our main model is so big, the copying process
    // still takes a lot of time. But at least that would be a one-time cost!
    std::cout << currentTimeString() << "- Copying model...!\n";
    AIToolbox::MDP::SparseModel model(world);
    std::cout << currentTimeString() << "- Init solver...!\n";

    // This is a method that solves MDPs completely. It has a couple of
    // parameters available.
    // The only non-optional parameter is the horizon of the solution; as in
    // how many steps should the solution look ahead in order to decide which
    // move to take. If we chose 1, for example, the tiger would only consider
    // cells next to it to decide where to move; this wouldn't probably be
    // what we want.
    // We want the tiger to think for infinite steps: this can be
    // approximated with a very high horizon, since in theory the final solution
    // will converge to a single policy anyway. Thus we put a very high number
    // as the horizon here.
    AIToolbox::MDP::ValueIteration<decltype(model)> solver(1000000);

    std::cout << currentTimeString() << "- Starting solver!\n";
    // This is where the magic happen. This could take around 10-20 minutes,
    // depending on your machine (most of the time is spent on this tutorial's
    // code, however, since it is a pretty inefficient implementation).
    // But you can play with it and make it better!
    //
    // If you are using the Sparse Model though, it is instantaneous since
    // Eigen is very very efficient in computing the values we need!
    auto solution = solver(model);

    std::cout << currentTimeString() << "- Problem solved? " << std::get<0>(solution) << "\n";

    AIToolbox::MDP::Policy policy(world.getS(), world.getA(), std::get<1>(solution));

    std::cout << "Printing best actions when prey is in (5,5):\n\n";
    for ( int y = 10; y >= 0; --y ) {
        for ( int x = 0; x < 11; ++x ) {
            std::cout << policy.sampleAction( encodeState(CoordType{{x, y, 5, 5}}) ) << " ";
        }
        std::cout << "\n";
    }

    std::cout << "\nSaving policy to file for later usage...\n";
    {
        // You can load up this policy again using ifstreams.
        // You will not need to solve the model again ever, and you
        // can embed the policy into any application you want!
        std::ofstream output("policy.txt");
        output << policy;
    }

    return 0;
}
Beispiel #3
0
int main() {
    GridWorld world;

    // This is a method that solves MDPs completely. It has a couple of
    // parameters available, but in our case the defaults are perfectly
    // fine.
    AIToolbox::MDP::ValueIteration solver;

    std::cout << "Starting solver!\n";
    // This is where the magic happen. This could take around 10-20 minutes,
    // depending on your machine (most of the time is spent on this tutorial's
    // code, however, since it is a pretty inefficient implementation).
    // But you can play with it and make it better!
    auto solution = solver(world);

    std::cout << "Problem solved? " << std::get<0>(solution) << "\n";

    AIToolbox::MDP::Policy policy(world.getS(), world.getA(), std::get<1>(solution));

    std::cout << "Printing best actions when prey is in (5,5):\n\n";
    for ( int y = 10; y >= 0; --y ) {
        for ( int x = 0; x < 11; ++x ) {
            std::cout << policy.sampleAction( encodeState(CoordType{{x, y, 5, 5}}) ) << " ";
        }
        std::cout << "\n";
    }

    std::cout << "\nSaving policy to file for later usage...\n";
    {
        // You can load up this policy again using ifstreams.
        // You will not need to solve the model again ever, and you
        // can embed the policy into any application you want!
        std::ofstream output("policy.txt");
        output << policy;
    }

    return 0;
}
Beispiel #4
0
//--------------------------------------------------------------
//copy maze (from LPA*) to map (of GridWorld)
void copyMazeToDisplayMap(GridWorld &gWorld, LpaStar* lpa){
	for(int i=0; i < gWorld.getGridWorldRows(); i++){
	   for(int j=0; j < gWorld.getGridWorldCols(); j++){
			gWorld.map[i][j].type = lpa->maze[i][j].type;
		   gWorld.map[i][j].h = lpa->maze[i][j].h;
			gWorld.map[i][j].g = lpa->maze[i][j].g;
			gWorld.map[i][j].rhs = lpa->maze[i][j].rhs;
			gWorld.map[i][j].row = lpa->maze[i][j].y;
			gWorld.map[i][j].col = lpa->maze[i][j].x;
			
			for(int k=0; k < 2; k++){
			  gWorld.map[i][j].key[k] = lpa->maze[i][j].key[k];			  
			}
			
			
		}
	}
	gWorld.map[lpa->start->y][lpa->start->x].h = lpa->start->h;
	gWorld.map[lpa->start->y][lpa->start->x].g = lpa->start->g;
	gWorld.map[lpa->start->y][lpa->start->x].rhs = lpa->start->rhs;
	gWorld.map[lpa->start->y][lpa->start->x].row = lpa->start->y;
	gWorld.map[lpa->start->y][lpa->start->x].col = lpa->start->x;
	for(int k=0; k < 2; k++){
			  gWorld.map[lpa->start->y][lpa->start->x].key[k] = lpa->start->key[k];			  
	}
	
	
	gWorld.map[lpa->goal->y][lpa->goal->x].h = lpa->goal->h;
	gWorld.map[lpa->goal->y][lpa->goal->x].g = lpa->goal->g;
	gWorld.map[lpa->goal->y][lpa->goal->x].rhs = lpa->goal->rhs;
	gWorld.map[lpa->goal->y][lpa->goal->x].row = lpa->goal->y;
	gWorld.map[lpa->goal->y][lpa->goal->x].col = lpa->goal->x;
	for(int k=0; k < 2; k++){
			  gWorld.map[lpa->goal->y][lpa->goal->x].key[k] = lpa->goal->key[k];			  
	}
	
}
Beispiel #5
0
void testReinforcement()
{
    GridWorld g;
    TDLearning(g);
    g.debug();
}
Beispiel #6
0
inline AIToolbox::MDP::Model makeCliffProblem(const GridWorld & grid) {
    using namespace AIToolbox::MDP;

    size_t S = grid.getSizeX() * grid.getSizeY() + 2, A = 4;

    AIToolbox::Table3D transitions(boost::extents[S][A][S]);
    AIToolbox::Table3D rewards(boost::extents[S][A][S]);

    double failReward = -100.0, stepReward = -1.0, winReward = 0.0;

    // Default all transitions within the grid to be deterministic,
    // and give negative reward. Remember that the actual cliff is
    // under the grid.
    for ( size_t s = 0; s < S-2; ++s ) {
        for ( size_t a = 0; a < A; ++a ) {
            auto cell = grid(s);
            cell.setAdjacent((Direction)a);
            transitions[s][a][cell] = 1.0;
            rewards[s][a][cell] = stepReward;
        }
    }
    // Attach start and goal states
    size_t start = S - 2, goal = S - 1;
    size_t upStart = (grid.getSizeY() - 1) * grid.getSizeX();
    size_t upGoal  = S - 3;

    // Fix start
    transitions[start][UP   ][upStart] =  1.0;
    rewards    [start][UP   ][upStart] = stepReward;
    transitions[start][LEFT ][start  ] =  1.0;
    rewards    [start][LEFT ][start  ] = stepReward;
    transitions[start][DOWN ][start  ] =  1.0;
    rewards    [start][DOWN ][start  ] = stepReward;
    transitions[start][RIGHT][start  ] =  1.0;
    rewards    [start][RIGHT][start  ] = failReward; // This goes into the cliff

    // Fix down for upStart
    transitions[upStart][DOWN][upStart] = 0.0;
    rewards    [upStart][DOWN][upStart] = 0.0;
    transitions[upStart][DOWN][start  ] = 1.0;
    rewards    [upStart][DOWN][start  ] = stepReward;

    // Fix goal (self absorbing)
    transitions[goal][UP   ][goal] = 1.0;
    transitions[goal][LEFT ][goal] = 1.0;
    transitions[goal][DOWN ][goal] = 1.0;
    transitions[goal][RIGHT][goal] = 1.0;

    // Fix upGoal
    transitions[upGoal][DOWN][upGoal] = 0.0;
    rewards    [upGoal][DOWN][upGoal] = 0.0;
    transitions[upGoal][DOWN][goal  ] = 1.0;
    rewards    [upGoal][DOWN][goal  ] = winReward; // Won!

    // Fix cliff edge
    for ( size_t s = upStart + 1; s < upGoal; ++s ) {
        transitions[s][DOWN][s    ] = 0.0;
        rewards    [s][DOWN][s    ] = 0.0;
        transitions[s][DOWN][start] = 1.0;
        rewards    [s][DOWN][start] = failReward; // This goes into the cliff
    }

    Model model(S, A, transitions, rewards, 1.0);

    return model;
}
Beispiel #7
0
void runSimulation(char *fileName){
	WorldBoundaryType worldBoundary; //duplicated in GridWorld
   DevBoundaryType deviceBoundary; //duplicated in GridWorld
	bool ANIMATE_MOUSE_FLAG=false;
	bool validCellSelected=false;
	static BOOL page=false;
	int mX, mY;
	float worldX, worldY;
	worldX=0.0f;
	worldY=0.0f;
	
	int action=-1;
	//-----------------------
	CellPosition p;
	int rowSelected, colSelected;
	//-----------------------
   rowSelected=-1;
	colSelected=-1;
	
	int mouseRadius=1;
		
	srand(time(NULL));  // Seed the random number generator
			
	//Initialise the world boundaries
   grid_world.initSystemOfCoordinates();
	grid_world.loadMapAndDisplay(fileName);
	grid_world.initialiseMapConnections();
	
	//----------------------------------------------------------------
	//LPA*
	lpa_star = new LpaStar(grid_world.getGridWorldRows(), grid_world.getGridWorldCols());
	vertex start = grid_world.getStartVertex();
	vertex goal = grid_world.getGoalVertex();
	
	cout << "(start.col = " << start.col << ", start.row = " << start.row << ")" << endl;
	cout << "(goal.col = " << goal.col << ", goal.row = " << goal.row << ")" << endl;
	
	lpa_star->initialise(start.col, start.row, goal.col, goal.row);
	
	//lpa_star->copyMazeToDisplayMap(grid_world);
	//copyMazeToDisplayMap(grid_world, lpa_star);
	copyDisplayMapToMaze(grid_world, lpa_star);
	//----------------------------------------------------------------
		
	worldBoundary = grid_world.getWorldBoundary();
	deviceBoundary = grid_world.getDeviceBoundary();
	GRIDWORLD_ROWS = grid_world.getGridWorldRows();
	GRIDWORLD_COLS = grid_world.getGridWorldCols();
	
	//setvisualpage(page);
	
	// keep running the program until the ESC key is pressed   
	while((GetAsyncKeyState(VK_ESCAPE)) == 0 ) {
			 setactivepage(page);
			 cleardevice();
	
		    action = getKey(); 
		
		    if(SHOW_MAP_DETAILS) 
				 grid_world.displayMapWithDetails();
			 else
			    grid_world.displayMap();
			 
			 switch(action){
				case 1: //Block selected cell
				 		if( rowSelected != -1 && colSelected != -1){
							grid_world.setMapTypeValue(rowSelected-1, colSelected-1, '1');
							grid_world.initialiseMapConnections(); 
							
							rowSelected=-1;
							colSelected=-1;
						}
						action = -1;
						break;
				
				case 105: 
					   grid_world.displayMapWithKeyDetails();
						break;
				
				case 106: 
					  
					   //~ algorithmSelection = ASTAR_ALGORITHM;
						break;
				
				case 107: 
					  
					   //~ algorithmSelection = LPASTAR_ALGORITHM;
						break;
				
				case 108: 
					  
					   //~ algorithmSelection = DSTAR_ALGORITHM;
						break;
				
				case 15:
					 
					   if( rowSelected != -1 && colSelected != -1){
							grid_world.displayVertexConnections(colSelected-1, rowSelected-1);
						   //cout << "display connections" << endl;
						   rowSelected=-1;
						   colSelected=-1;
					   } else {
							cout << "invalid new START vertex, please select a new START vertex first." << endl;
							break;
						}
						//--------------------------------------------
					   action = -1;
					    break;
						
				case 16:
					 
					   if(grid_world.isGridMapInitialised()){
							grid_world.displayMapConnections();
						   //cout << "display connections" << endl;
						   //~ rowSelected=-1;
						   //~ colSelected=-1;
					   } else {
							cout << "map has not been initialised yet." << endl;
							break;
						}
						//--------------------------------------------
					   action = -1;
					    break;		
				
				case 6: //set cell as new START vertex 
				   {
					   //--------------------------------------------
				      // retrieve current START vertex
				      vertex s = 	grid_world.getStartVertex();
				      if( (s.row != -1) && (s.col != -1) ){
							//set current START VERTEX to an ordinary TRAVERSABLE CELL
							grid_world.setMapTypeValue(s.row, s.col, '0'); 
							grid_world.initialiseMapConnections(); 
							//ok, proceed
						} else {
							cout << "invalid START vertex" << endl;
							break;
						}
				      //--------------------------------------------
						//set selected cell as the NEW START VERTEX
					   if( rowSelected != -1 && colSelected != -1){
						   grid_world.setMapTypeValue(rowSelected-1, colSelected-1, '6');
						   s.row = rowSelected-1;
							s.col = colSelected-1;
							grid_world.setStartVertex(s);
							
						   rowSelected=-1;
						   colSelected=-1;
					   } else {
							cout << "invalid new START vertex, please select a new START vertex first." << endl;
							break;
						}
						//--------------------------------------------
					   action = -1;
						break;
					}
				
				case 7: //set cell as new GOAL vertex 
				   {
					   //--------------------------------------------
				      // retrieve current GOAL vertex
				      vertex s = 	grid_world.getGoalVertex();
				      if( (s.row != -1) && (s.col != -1) ){
							//set current GOAL VERTEX to an ordinary TRAVERSABLE CELL
							grid_world.setMapTypeValue(s.row, s.col, '0'); 
							
							//ok, proceed
						} else {
							cout << "invalid GOAL vertex" << endl;
							action = -1;
							break;
						}
				      //--------------------------------------------
						//set selected cell as the NEW GOAL VERTEX
					   if( rowSelected != -1 && colSelected != -1){
						   grid_world.setMapTypeValue(rowSelected-1, colSelected-1, '7');
						   s.row = rowSelected-1;
							s.col = colSelected-1;
							grid_world.setGoalVertex(s);
							grid_world.initialiseMapConnections(); 
							
						   rowSelected=-1;
						   colSelected=-1;
					   } else {
							cout << "invalid new GOAL vertex, please select a new GOAL vertex first." << endl;
							action = -1;
							break;
						}
						//--------------------------------------------
					   action = -1;
						break;
					}
							
            case 109:					
					   copyDisplayMapToMaze(grid_world, lpa_star);
				      cout << "copied display map to algorithm's maze" << endl;
				      action = -1;
				      break;
				
				case 110:					
					   lpa_star->updateHValues();
					   copyMazeToDisplayMap(grid_world, lpa_star);
				      cout << "copied algorithm's maze to display map" << endl;
				      action = -1;
				      break;
				
				case 9: //display g-values only
					   grid_world.displayMapWithSelectedDetails(true, false, false, false);  //(bool display_g, bool display_rhs, bool display_h, bool display_key) 
				      action = -1;
						break;
            case 10: //display h-values only
					   grid_world.displayMapWithSelectedDetails(false, false, true, false);  //(bool display_g, bool display_rhs, bool display_h, bool display_key) 
				 		action = -1;
				      break;
				case 11: //display key-values only
					   lpa_star->updateAllKeyValues();
				      copyMazeToDisplayMap(grid_world, lpa_star);
					   grid_world.displayMapWithSelectedDetails(false, false, false, true);  //(bool display_g, bool display_rhs, bool display_h, bool display_key) 
						action = -1;
				      break;
				
				case 12: //make cell Traversable
			 
					 if( rowSelected != -1 && colSelected != -1){
						 grid_world.setMapTypeValue(rowSelected-1, colSelected-1, '0');
						 
						 rowSelected=-1;
						 colSelected=-1;
					 }
					 action = -1;
					 break; 
					 
				case 14: 
					   grid_world.displayMapWithPositionDetails();
						action = -1;
				      break;	 
					 
				 //~ default: //Display grid without details
					   //~ grid_world.displayMap();
						  
				  
				 
		    };
		
	   //----------------------------------------------------------------------------------------------------------------	  
		// Mouse handling
		//
			 if(mousedown()){
						 				
				ANIMATE_MOUSE_FLAG=true;
				 			 
				mX = mousecurrentx();
				mY = mousecurrenty();
				 
				//if the goal selected is within the playing field boundaries
				if(mX >= grid_world.getFieldX1() && mX <= grid_world.getGridMaxX() && mY >= grid_world.getFieldY1() && mY <= grid_world.getGridMaxY()){
					
					    circle(mX, mY, 3);
					    validCellSelected = true;
  	            
				} else {
					validCellSelected = false;
				}
			 } //end of mousedown()
			 //------------------------------------------------------------------------------------------------------------------
			 /////////////////////////////////////////////////////////////////////////////
			 						 
			 if(ANIMATE_MOUSE_FLAG){
					
				  //draw Cross-hair to mark Goal	    
				  setcolor(RED);
				  circle(mX, mY, 20);
				  line(mX,mY-20,mX,mY+20);
				  line(mX-20,mY,mX+20,mY);
				  //end of draw Cross-hair 
			 
				  // special effect to display concentric circles locating the target
					setcolor(YELLOW);
					
					if(mouseRadius < 40) {
						mouseRadius += 1;
					}
					circle(mX, mY, mouseRadius);
					//Sleep(50);
									
					if(mouseRadius >= 40) {
						ANIMATE_MOUSE_FLAG=false;
						mouseRadius=0;
					}
					//end of special effect
			  }

			 
			 /////////////////////////////////////////////////////////////////////////////
			  char info[80]; 
			  float wX, wY;
			  
			  wX = xWorld(worldBoundary,deviceBoundary,mX);
			  wY = yWorld(worldBoundary,deviceBoundary,mY);
			  sprintf(info,"x: %d, y: %d",mX, mY); 
			  drawInformationPanel(grid_world.getFieldX2(), grid_world.getFieldY1() + textheight("H")*2, info);
			  
			 
			  sprintf(info,"wX: %3.0f, wY: %3.0f",wX, wY); 
			  drawInformationPanel(grid_world.getFieldX2(),grid_world.getFieldY1() + textheight("H")*5, info);
			 ///////////////////////////////////////////////////////////////////////////// 
			 
			  //~ CellPosition p;
			  //~ int rowSelected, colSelected;
			  
			  if(validCellSelected) {
				  p=grid_world.getCellPosition_markCell(mX, mY);
				  rowSelected = p.row;
				  colSelected = p.col;
				  
				  sprintf(info,"row: %d, col: %d",rowSelected, colSelected); 
			     drawInformationPanel(grid_world.getFieldX2(),grid_world.getFieldY1() + textheight("H")*6, info);
				  
			  }
			  setvisualpage(page);
			  page = !page;  //switch to another page
	}
}
Beispiel #8
0
int main(int argc, char* argv[]) {
    clock_t begin_main = GetTime();
    clock_t end_main;
    bool is_verbose = false;
#ifdef OUTPUT
    is_verbose = true;
#endif
    float discount_factor = 0.0;  /* gamma */
    float max_reward = 0.0;
    float weight = 0.0;
    char algorithm = 0;
    char opt = 0;
    unsigned int NUM_ACTIONS = 4;
    unsigned int NUM_AGENTS = 2;
    GridWorld *environment = NULL;
    VariablesInfo* stateDescription = NULL;
    int next_option = 0;
    int num_options = 0;
    /* A string listing valid short options letters:
    	h = help
    	a = number of actions
    	t = number of agents
    	m = algorithm (C=CoLF, K = CKCoLF)
     	g = gamma
    	p = number of steps for each trial
    	e = environment (G = grid for now)
    */
    const char* const short_options = "h:a:o:w:g:e:";
    /* An array describing valid long options (see above). */
    const struct option long_options[] = {
        {"help", 0, NULL, 'h'},
        {"algorithm", 1, NULL, 'a'},
        {"option", 1, NULL, 'o'},
        {"weight", 1, NULL, 'w'},
        {"gamma", 1, NULL, 'g'},
        {"environment", 1, NULL, 'e'},
        {NULL, 0, NULL, 0}   /* Required at end of array.  */
    };
    srand(time(NULL));
    vector<VI*> agents;
    vector<string> agent_names;
    do {
        next_option = getopt_long (argc, argv, short_options, long_options, NULL);
        switch (next_option) {
        case 'h': {		/* -h or --help */
            PrintUsage("Help message",EXIT_OK);
        }
        case 'a': {		/* -m or --algorithm --> learning algorithm for self play (C, K)  */

            if (*optarg == 'C' || *optarg == 'F'||*optarg == 'M'||*optarg == 'T')
                algorithm = *optarg;
            else
                PrintUsage("Invalid algorithm, only \'C\' or \'F\' or \'M\' or \'T\' are allowed!",BAD_USAGE);
            //cout << "Algorithm: " << algorithm << endl;
            num_options++;
            break;
        }
        case 'o': {
            if (*optarg == 'u' || *optarg == 'e'||*optarg == 'r')
                opt = *optarg;
            else
                PrintUsage("Invalid algorithm option, only \'u\' or \'e\' or \'r\' are allowed!",BAD_USAGE);
            //cout << "Algorithm: " << algorithm << endl;
            num_options++;
            break;
        }
        case 'w':
            weight = atof(optarg);
            if (weight < 0 || weight > 1)
                PrintUsage("Invalid weight, it must be between 0 and 1", BAD_USAGE);
            //cout << "Gamma: " << discount_factor <<endl;;
            num_options++;
            break;

        case 'g': {		/* -g or --gamma --> discount factor */
            discount_factor = atof(optarg);
            if (discount_factor < 0 || discount_factor > 1)
                PrintUsage("Invalid discount_factor, it must be between 0 and 1", BAD_USAGE);
            //cout << "Gamma: " << discount_factor <<endl;;
            num_options++;
            break;
        }

        case 'e': {		/* -e or --environment --> environment choosen (G) */
            if ( *optarg == 'T') {
                if (NUM_AGENTS >= 8)
                    PrintUsage("Too many agents for the grid environment; maximum allowed = 7\n", BAD_USAGE);
                environment = new GridWorld(*optarg);
            }
            else if(*optarg == 'C'||*optarg == 'P'||*optarg == 'X'||*optarg == 'G'||*optarg == 'A')
                environment = new GridWorldWall(*optarg);
            else
                PrintUsage("Invalid environment", BAD_USAGE);
            //cout << "Environment: " << *optarg << endl;
            num_options++;
            break;
        }
        case '?': {		/* The user specified an invalid option.  */
            PrintUsage (string("Unknown option \'").append(optarg).append("\'"), BAD_USAGE);
        }
        case -1: {			/* Done with options.  */
            if (num_options != 5)
                PrintUsage("Mandatory options not present", BAD_USAGE);
            break;
        }
        default:    /* Something else unexpected.  */
            abort();
        }
    } while (next_option != -1);
    VariablesInfo* admissible_actions = new VariablesInfo();
    CE_VI* agent = NULL;
    AddAgentNames(&agent_names);

    AddAgentActions(NUM_ACTIONS, admissible_actions);
    // Set the intial state of grid environment with random position for all agents
    // Obviously is passed as argument to each agent...
    stateDescription = SetInitialState(NUM_AGENTS);
    environment->Init(stateDescription);

    //stateDescription = environment->GetStateDescription();

    //environment->AppendInit(AppendToInitialState(stateDescription,NUM_AGENTS));
    cout << "****" << endl;
    if (algorithm == 'C' || algorithm == 'F'|| algorithm == 'M'|| algorithm == 'T') {
        cout << "Creating VI..." <<endl << "algorithm: "<<algorithm<< "  option:" << opt<<endl;
        agent = new CE_VI(agent_names[0],
                          0,
                          NUM_AGENTS,
                          admissible_actions,
                          discount_factor,
                          max_reward,
                          is_verbose,
                          algorithm,
                          opt,
                          weight,
                          environment);
        if (agent == NULL) {
            cerr << "main: VI creation NULL!" << endl;
            exit(NO_MEM_AVAILABLE);
        }
    }
    else {
        cout << "The specified algorithm is not available" << endl;
        cout << "The only algorithms available are:" << endl;
        cout << "'C': CE-VI" << endl;
        cout << "'F': Friend-VI" << endl;
        cout << "'M': MiniMax-VI" << endl;
        cout << "'T': TB-VI" << endl;
        exit(1);
    }

    agent->SetTransitionModel(environment->GetTransitionModel());


    //FOR DEBUG
    /*vector <unsigned> vState;
    for (unsigned i= 0; i<81;i++)
    {
    	cout << "State " << i;
    	vState = environment->GetFromIndex(i,4,3);
    	cout <<": " << vState[0] << "," <<vState[1] << "," <<vState[2] << "," <<vState[3] <<endl;
    	cout << "<-state:" << environment->ComputeJointIndex(vState, 3) << endl;
    }*/

    agent->Execute(); // select and execute action
    //agent->BuildConvexHull();

    delete stateDescription;
    stateDescription = NULL;

    //test_environment(environment);
    //cout << "TOTAL PAYOFF --> G = " << endl << endl;
    /*for (i = 0; i < agents.size(); i++) {
    	cout << agents[i]->PrintQTable();
    }*/
    delete admissible_actions;
    delete environment;

    // perché diavolo segmenta?!?
    /*for (i = 0; i < agents.size(); i++) {
    	cout << "main: delete agent " << i << endl; flush(cout);
    	delete agents[i];
    }*/

    end_main = GetTime();
    cout << ">>> main.cpp: elapsed time = " << (double)(end_main-begin_main)/CLOCKS_PER_SEC << " seconds" <<endl;
    return 0;
}