int main (int argc, char** argv) {
// Set up random number generation
randomNumberGenerator.seed( time(0) );
// Get the world settings
if ( argc > 1 ) {
getSettingsFromFile(argv[1]);
} else {
worldSize = 64;
vectorLength = 1024;
alpha = 0.1;
lambda = 0.5;
discount = 0.9;
epsilon = 0.05;
}
eligibility.resize(vectorLength);
weights.resize(vectorLength);
hrrEngine.setVectorSize(vectorLength);
// Set up a log file
ofstream Log;
Log.open("results.log");
// Set up a final report file
ofstream FinalReport;
FinalReport.open("final.log");
// Initialize the world array with states
for (int i = 0; i < worldSize; i++) {
// Create a new state and add it to the world array
State newState(0, vectorLength, i);
world.push_back(newState);
}
// Initialize weight vector
for (double& weight : weights) {
weight = 0.0;
}
// Set up statistical variables
averageNumberOfSteps = 0;
maxNumberOfSteps = 0;
minNumberOfSteps = IntMax;
// Set up a location for the goal
goalLocation = randomNumberGenerator() % worldSize;
world[goalLocation].setReward(1.0);
// Main loop of program. Run episodes until task is learned
for (int i = 0; i < numberOfRuns; i++) {
// Set up a location for the agent
agentLocation = randomNumberGenerator() % worldSize;
// Initialize Episode statistical variables
int numberOfSteps = 0;
// Reset the eligibility vector
fill(eligibility.begin(), eligibility.end(), 0);
State* thisState;
// Movement through an episode
do {
// Set up a variable for the previous state
thisState = &world[agentLocation];
cout << "Goal Location: " << goalLocation << "\tCurrent State: " << thisState->isAt() << "\n";
// Update the eligibility of the current state
updateEligibility(thisState->getHRR());
// If we are at the goal, calculate the td error differently, since
// there are no future steps
if (thisState->isAt() == goalLocation) {
cout << "Goal reached in " << numberOfSteps << " steps.\n";
State G = *thisState;
HRR a = G.getHRR();
double TDError = r(G) - V(G);
cout << "\n\nr(G) = " << r(G) << "\tV(G) = " << V(G) << "\n\n";
cout << "TDError: " << TDError << "\n";
for ( int x = 0; x < weights.size(); x++ ) {
weights[x] += alpha * TDError * a[x];
}
break;
} else {
// Choose a movement for the agent
Move movement = chooseMovement( world[getLeftLocation( agentLocation )],
world[agentLocation],
world[getRightLocation( agentLocation )] );
// Perform the movement
switch (movement) {
case Left:
agentLocation = getLeftLocation(agentLocation);
break;
case Right:
agentLocation = getRightLocation(agentLocation);
break;
default:
agentLocation = getLeftLocation(agentLocation);
break;
}
State* nextState = &world[agentLocation];
// Update the weights
State s = *thisState;
State sPlus1 = *nextState;
HRR a = s.getHRR();
double TDError = ( r(s) + discount * V(sPlus1) ) - V(s);
cout << "r(s) = " << r(s) << "\tV(s+1) = " << V(sPlus1) << "\tV(s) = " << V(s) << "\n";
cout << "TDError: " << TDError << '\n';
cout << "Run: " << i << ", Step: " << numberOfSteps << ", Location: "
<< agentLocation << ", Goal: " << goalLocation << "\n";
for ( int x = 0; x < vectorLength; x++ ) {
weights[x] += alpha * TDError * a[x];
}
}
numberOfSteps++;
} while ( numberOfSteps <= 100 );
}
//cout << "The size of the hrrs is: " << hrrEngine.getVectorSize() << "\n";
//cout << "The size of the eligibility vector is: " << eligibility.size() << "\n";
//cout << "The size of the weight vector is: " << weights.size() << "\n";
cout << "Values of each state:\n";
for (int i = 0; i < worldSize; i++) {
cout << "\tV(" << i << ") = " << V(world[i]) << "\n";
}
return 0;
}
int main (int argc, char** argv) {
// Declare variables
int size;
char option;
string name = "";
string name2 = "";
ifstream fin;
vector<string> concepts = {};
string temp;
string filename;
HRR hrr;
HRR hrr2;
// Initiate program. Get size to use when working with vectors
cout << "***Welcome to the HRR Engine tester!***\n"
<< "What size vectors do you want to create? ";
cin >> size;
// Set up the HRR engine
HRREngine engine;
engine.setVectorSize(size);
do{
printMenu();
cin >> option;
switch (option){
case '1':
cout << "Please enter name of concept to search for: ";
cin >> name;
hrr = engine.query(name);
//engine.printHRRHorizontal(hrr);
break;
case '2':
cout << "Please enter name of concept to search for by representation: ";
cin >> name;
hrr = engine.query(name);
//cout << "Representation: ";
//engine.printHRRHorizontal(hrr);
cout << "\nRepresentation is most like: ";
cout << engine.query(hrr) << "\n\n";
break;
case '3':
// Get the name of the input file
cout << "Please input name of file to read from: ";
cin >> filename;
// Open input filestream
fin.open( filename );
// While there is more to read, read a concept and push it into concepts vector
while (fin >> temp){
concepts.push_back(temp);
}
// Encode each string in concepts vector and store in a map
engine.encodeConcepts( concepts );
break;
case '4':
// Get the names of the two vectors to combine
cout << "Input the first concept: ";
cin >> name;
cout << "Input the second concept: ";
cin >> name2;
name = engine.combineConcepts(name, name2);
cout << "The new concept is: " << name << "\n";
break;
case '5':
// Get the names of the two vectors to correlate
cout << "Input the complex concept: ";
cin >> name;
cout << "Input the base concept: ";
cin >> name2;
name = engine.extractConcept(name, name2);
cout << "The concept that is most similar to the extracted concept is: " << name << "\n";
break;
case '6':
engine.listAllConcepts();
break;
case '7':
engine.listAllConceptNames();
break;
case '0':
break;
default:
cout << "ERROR: Incorrect option entered\n";
break;
}
cout << "\n";
}while (!(option == '0'));
return 0;
}
// Get the value of a state
double V(State s) {
return hrrEngine.dot(s.getHRR(), weights);
}