int main() {
sf::RenderWindow window;
sf::ContextSettings glContextSettings;
glContextSettings.antialiasingLevel = 4;
window.create(sf::VideoMode(800, 600), "BIDInet", sf::Style::Default, glContextSettings);
window.setFramerateLimit(60);
window.setVerticalSyncEnabled(true);
std::mt19937 generator(time(nullptr));
/*sys::ComputeSystem cs;
cs.create(sys::ComputeSystem::_gpu);
sys::ComputeProgram program;
program.loadFromFile("resources/bidinet.cl", cs);
bidi::BIDInet bidinet;
std::vector<bidi::BIDInet::InputType> inputTypes(64, bidi::BIDInet::_state);
const int numStates = 3 + 3 + 2 + 2 + 1 + 2 + 2;
const int numActions = 3 + 3 + 2 + 2;
const int numQ = 8;
for (int i = 0; i < numStates; i++)
inputTypes[i] = bidi::BIDInet::_state;
for (int i = 0; i < numActions; i++)
inputTypes[numStates + i] = bidi::BIDInet::_action;
std::vector<bidi::BIDInet::LayerDesc> layerDescs(2);
layerDescs[0]._fbRadius = 16;
layerDescs[1]._width = 8;
layerDescs[1]._height = 8;
bidinet.createRandom(cs, program, 8, 8, inputTypes, layerDescs, -0.1f, 0.1f, 0.001f, 1.0f, generator);*/
// Physics
std::shared_ptr<b2World> world = std::make_shared<b2World>(b2Vec2(0.0f, -9.81f));
const float pixelsPerMeter = 256.0f;
const float groundWidth = 5000.0f;
const float groundHeight = 5.0f;
// Create ground
b2BodyDef groundBodyDef;
groundBodyDef.position.Set(0.0f, 0.0f);
b2Body* groundBody = world->CreateBody(&groundBodyDef);
b2PolygonShape groundBox;
groundBox.SetAsBox(groundWidth * 0.5f, groundHeight * 0.5f);
groundBody->CreateFixture(&groundBox, 0.0f);
sf::Texture skyTexture;
skyTexture.loadFromFile("resources/background1.png");
skyTexture.setSmooth(true);
sf::Texture floorTexture;
floorTexture.loadFromFile("resources/floor1.png");
floorTexture.setRepeated(true);
floorTexture.setSmooth(true);
Runner runner0;
runner0.createDefault(world, b2Vec2(0.0f, 2.762f), 0.0f, 1);
//Runner runner1;
//runner1.createDefault(world, b2Vec2(0.0f, 2.762f), 0.0f, 2);
//deep::FERL ferl;
const int recCount = 4;
const int clockCount = 4;
//ferl.createRandom(3 + 3 + 2 + 2 + 1 + 2 + 2 + recCount + clockCount, 3 + 3 + 2 + 2 + recCount, 32, 0.01f, generator);
//std::vector<float> prevAction(ferl.getNumAction(), 0.0f);
deep::CSRL prsdr;
const int inputCount = 3 + 3 + 2 + 2 + 1 + 2 + 2 + recCount + clockCount + 1;
const int outputCount = 3 + 3 + 2 + 2 + recCount;
std::vector<deep::CSRL::LayerDesc> layerDescs(2);
layerDescs[0]._width = 8;
layerDescs[0]._height = 8;
layerDescs[1]._width = 4;
layerDescs[1]._height = 4;
std::vector<sdr::IPRSDRRL::InputType> inputTypes(7 * 7, sdr::IPRSDRRL::_state);
prsdr.createRandom(7, 7, 8, layerDescs, -0.01f, 0.01f, 0.5f, generator);
//deep::SDRRL sdrrl;
//sdrrl.createRandom(inputCount, outputCount, 32, -0.01f, 0.01f, 0.0f, generator);
// ---------------------------- Game Loop -----------------------------
sf::View view = window.getDefaultView();
bool quit = false;
sf::Clock clock;
float dt = 0.017f;
int steps = 0;
do {
clock.restart();
// ----------------------------- Input -----------------------------
sf::Event windowEvent;
while (window.pollEvent(windowEvent))
{
switch (windowEvent.type)
{
case sf::Event::Closed:
quit = true;
break;
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
quit = true;
//bidinet.simStep(cs, 0.0f, 0.98f, 0.001f, 0.95f, 0.01f, 0.01f, generator);
const float maxRunnerBodyAngle = 0.3f;
const float runnerBodyAngleStab = 10.0f;
{
float reward;
if (sf::Keyboard::isKeyPressed(sf::Keyboard::K))
reward = -runner0._pBody->GetLinearVelocity().x;
else
reward = runner0._pBody->GetLinearVelocity().x;
std::vector<float> state;
runner0.getStateVector(state);
std::vector<float> action(3 + 3 + 2 + 2 + recCount);
for (int a = 0; a < recCount; a++)
state.push_back(prsdr.getPrediction(inputCount + outputCount - recCount + a));
for (int a = 0; a < clockCount; a++)
state.push_back(std::sin(steps / 60.0f * 2.0f * a * 2.0f * 3.141596f));
for (int i = 0; i < state.size(); i++)
prsdr.setInput(i, state[i]);
//sdrrl.simStep(reward, 0.1f, 0.99f, 16, 0.01f, 0.01f, 0.01f, 0.01f, 16, 0.05f, 0.98f, 0.05f, 0.01f, 0.01f, 4.0f, generator);
prsdr.simStep(reward, generator);
for (int i = 0; i < action.size(); i++)
action[i] = prsdr.getPrediction(inputCount + i) * 0.5f + 0.5f;
runner0.motorUpdate(action, 12.0f);
// Keep upright
if (std::abs(runner0._pBody->GetAngle()) > maxRunnerBodyAngle)
runner0._pBody->SetAngularVelocity(-runnerBodyAngleStab * runner0._pBody->GetAngle());
}
/*{
float reward;
if (sf::Keyboard::isKeyPressed(sf::Keyboard::K))
reward = -runner1._pBody->GetLinearVelocity().x;
else
reward = runner1._pBody->GetLinearVelocity().x;
std::vector<float> state;
runner1.getStateVector(state);
std::vector<float> action(3 + 3 + 2 + 2 + recCount);
for (int a = 0; a < recCount; a++)
state.push_back(prevAction[prevAction.size() - recCount + a]);
for (int a = 0; a < clockCount; a++)
state.push_back(std::sin(steps / 60.0f * 2.0f * a * 2.0f * 3.141596f));
// Bias
state.push_back(1.0f);
//ferl.step(state, action, reward, 0.5f, 0.99f, 0.98f, 0.05f, 16, 4, 0.05f, 0.01f, 0.05f, 600, 64, 0.01f, generator);
for (int i = 0; i < action.size(); i++)
action[i] = action[i] * 0.5f + 0.5f;
prevAction = action;
runner1.motorUpdate(action, 12.0f);
// Keep upright
if (std::abs(runner1._pBody->GetAngle()) > maxRunnerBodyAngle)
runner1._pBody->SetAngularVelocity(-runnerBodyAngleStab * runner1._pBody->GetAngle());
}*/
int subSteps = 1;
for (int ss = 0; ss < subSteps; ss++) {
world->ClearForces();
world->Step(1.0f / 60.0f / subSteps, 64, 64);
}
if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T) || steps % 200 == 1) {
// -------------------------------------------------------------------
//if (!sf::Keyboard::isKeyPressed(sf::Keyboard::B))
// view.setCenter(runner1._pBody->GetPosition().x * pixelsPerMeter, -runner1._pBody->GetPosition().y * pixelsPerMeter);
//else
view.setCenter(runner0._pBody->GetPosition().x * pixelsPerMeter, -runner0._pBody->GetPosition().y * pixelsPerMeter);
// Draw sky
sf::Sprite skySprite;
skySprite.setTexture(skyTexture);
window.setView(window.getDefaultView());
window.draw(skySprite);
window.setView(view);
sf::RectangleShape floorShape;
floorShape.setSize(sf::Vector2f(groundWidth * pixelsPerMeter, groundHeight * pixelsPerMeter));
floorShape.setTexture(&floorTexture);
floorShape.setTextureRect(sf::IntRect(0, 0, groundWidth * pixelsPerMeter, groundHeight * pixelsPerMeter));
floorShape.setOrigin(sf::Vector2f(groundWidth * pixelsPerMeter * 0.5f, groundHeight * pixelsPerMeter * 0.5f));
window.draw(floorShape);
//runner1.renderDefault(window, sf::Color::Blue, pixelsPerMeter);
runner0.renderDefault(window, sf::Color::Red, pixelsPerMeter);
/*sf::Image img;
img.create(sdrrl.getNumCells(), 1);
for (int i = 0; i < sdrrl.getNumCells(); i++) {
sf::Color c = sf::Color::Black;
c.r = c.g = c.b = 255.0f * (sdrrl.getCellState(i) > 0.0f ? 1.0f : 0.0f);
img.setPixel(i, 0, c);
}
float scale = 4.0f;
sf::Texture tex;
tex.loadFromImage(img);
sf::Sprite s;
s.setTexture(tex);
s.setScale(sf::Vector2f(scale, scale));
s.setPosition(sf::Vector2f(0.0f, window.getSize().y - scale * img.getSize().y));
window.setView(window.getDefaultView());
window.draw(s);*/
window.setView(view);
window.display();
}
else {
if (steps % 100 == 0)
std::cout << "Steps: " << steps << " Distance: " << runner0._pBody->GetPosition().x << std::endl;
}
//dt = clock.getElapsedTime().asSeconds();
steps++;
} while (!quit);
world->DestroyBody(groundBody);
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
sf::RenderWindow renderWindow;
sf::ContextSettings cs;
cs.antialiasingLevel = 8;
renderWindow.create(sf::VideoMode(800, 600), "Reinforcement Learning", sf::Style::Default, cs);
renderWindow.setVerticalSyncEnabled(true);
renderWindow.setFramerateLimit(60);
renderWindow.setMouseCursorVisible(false);
sdr::PredictiveRSDR rsdr;
std::vector<sdr::PredictiveRSDR::LayerDesc> layerDescs(3);
layerDescs[0]._width = 16;
layerDescs[0]._height = 16;
layerDescs[1]._width = 12;
layerDescs[1]._height = 12;
layerDescs[2]._width = 8;
layerDescs[2]._height = 8;
rsdr.createRandom(16, 16, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.1f, generator);
int ticksPerSample = 3;
int predSteps = 4;
int tickCounter = 0;
std::vector<sf::Vector2f> predTrailBuffer;
std::vector<sf::Vector2f> actualTrailBuffer;
int trailBufferLength = 200;
float trailDecay = 0.97f;
predTrailBuffer.resize(trailBufferLength);
actualTrailBuffer.resize(trailBufferLength);
// ------------------------------- Simulation Loop -------------------------------
bool quit = false;
float dt = 0.017f;
sf::Vector2i prevMousePos(-999, -999);
sf::Vector2f predictRenderPos(-999.0f, -999.0f);
do {
sf::Event event;
while (renderWindow.pollEvent(event)) {
switch (event.type) {
case sf::Event::Closed:
quit = true;
break;
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
quit = true;
renderWindow.clear();
sf::Vector2i mousePos = sf::Mouse::getPosition(renderWindow);
if (prevMousePos == sf::Vector2i(-999, -999)) {
prevMousePos = mousePos;
predictRenderPos = sf::Vector2f(mousePos.x, mousePos.y);
for (int t = 0; t < trailBufferLength; t++)
predTrailBuffer[t] = actualTrailBuffer[t] = predictRenderPos;
}
if (tickCounter >= ticksPerSample) {
sf::Vector2i delta = mousePos - prevMousePos;
prevMousePos = mousePos;
int px = std::min(15, std::max(0, static_cast<int>(16.0f * mousePos.x / static_cast<float>(renderWindow.getSize().x))));
int py = std::min(15, std::max(0, static_cast<int>(16.0f * mousePos.y / static_cast<float>(renderWindow.getSize().y))));
for (int i = 0; i < 256; i++)
rsdr.setInput(i, 0.0f);
rsdr.setInput(px, py, 1.0f);
//htsl.setInput(0, mousePos.x / static_cast<float>(renderWindow.getSize().x));
//htsl.setInput(1, mousePos.y / static_cast<float>(renderWindow.getSize().y));
rsdr.simStep();
tickCounter = 0;
}
else
tickCounter++;
sdr::PredictiveRSDR copy = rsdr;
for (int s = 0; s < predSteps; s++) {
copy.setInput(0, copy.getPrediction(0));
copy.setInput(1, copy.getPrediction(1));
copy.setInput(2, copy.getPrediction(2));
copy.setInput(3, copy.getPrediction(3));
copy.simStep(false);
}
sf::Vector2f predPos;
float div = 0.0f;
for (int i = 0; i < 256; i++) {
int x = i % 16;
int y = i / 16;
float v = copy.getPrediction(i);
predPos += v * sf::Vector2f(x / 16.0f * renderWindow.getSize().x, y / 16.0f * renderWindow.getSize().y);
div += v;
}
if (div != 0.0f)
predPos /= div;
predictRenderPos += 0.2f * (predPos - predictRenderPos);
for (int t = trailBufferLength - 1; t > 0; t--) {
predTrailBuffer[t] = predTrailBuffer[t - 1];
actualTrailBuffer[t] = actualTrailBuffer[t - 1];
}
predTrailBuffer[0] = predictRenderPos;
actualTrailBuffer[0] = sf::Vector2f(mousePos.x, mousePos.y);
float intensity = 1.0f;
sf::VertexArray predVertices;
predVertices.setPrimitiveType(sf::LinesStrip);
predVertices.resize(trailBufferLength);
sf::VertexArray actualVertices;
actualVertices.setPrimitiveType(sf::LinesStrip);
actualVertices.resize(trailBufferLength);
for (int t = 0; t < trailBufferLength; t++) {
actualVertices[t].position = actualTrailBuffer[t];
actualVertices[t].color = sf::Color(255, 0, 0, 255 * intensity);
predVertices[t].position = predTrailBuffer[t];
predVertices[t].color = sf::Color(0, 255, 0, 255 * intensity);
intensity *= trailDecay;
}
renderWindow.draw(actualVertices);
renderWindow.draw(predVertices);
float alignment = 0.0f;
float scale = 8.0f;
for (int l = 0; l < rsdr.getLayers().size(); l++) {
sf::Image sdr;
sdr.create(rsdr.getLayerDescs()[l]._width, rsdr.getLayerDescs()[l]._height);
for (int x = 0; x < rsdr.getLayerDescs()[l]._width; x++)
for (int y = 0; y < rsdr.getLayerDescs()[l]._height; y++) {
sf::Color c;
c.r = c.g = c.b = rsdr.getLayers()[l]._sdr.getHiddenState(x, y) * 255.0f;
sdr.setPixel(x, y, c);
}
sf::Texture sdrt;
sdrt.loadFromImage(sdr);
alignment += scale * sdr.getSize().x;
sf::Sprite sdrs;
sdrs.setTexture(sdrt);
sdrs.setPosition(renderWindow.getSize().x - alignment, renderWindow.getSize().y - sdr.getSize().y * scale);
sdrs.setScale(scale, scale);
vis::PrettySDR psdr;
psdr.create(rsdr.getLayerDescs()[l]._width, rsdr.getLayerDescs()[l]._height);
for (int x = 0; x < rsdr.getLayerDescs()[l]._width; x++)
for (int y = 0; y < rsdr.getLayerDescs()[l]._height; y++) {
psdr.at(x, y) = rsdr.getLayers()[l]._predictionNodes[x + y * rsdr.getLayerDescs()[l]._width]._statePrev;
}
psdr._nodeSpaceSize = scale;
psdr.draw(renderWindow, sf::Vector2f(renderWindow.getSize().x - alignment, renderWindow.getSize().y - sdr.getSize().y * scale));
//renderWindow.draw(sdrs);
}
renderWindow.display();
} while (!quit);
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
sys::ComputeSystem cs;
cs.create(sys::ComputeSystem::_gpu);
sys::ComputeProgram prog;
prog.loadFromFile("resources/neoKernels.cl", cs);
std::vector<Level> levels;
std::unordered_set<char> tileDataCharset;
std::unordered_set<char> objectDataCharset;
loadLevels("userlevels.txt", levels, tileDataCharset, objectDataCharset);
// --------------------------- Create the Sparse Coder ---------------------------
const int numTiles = levels.front()._tileData.length();
const int charsetSize = 128 + 2; // + 2 indicating the portion it is on (01 for tiles, 10 for objects)
const cl::size_type visDim = std::ceil(std::sqrt(static_cast<float>(charsetSize)));
const int visArea = visDim * visDim;
const int maxObjects = 1000;
cl::Image2D input = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), visDim, visDim);
std::vector<float> inputData(visArea, 0.0f);
std::vector<float> predData(visArea, 0.0f);
std::vector<neo::PredictiveHierarchy::LayerDesc> layerDescs(3);
layerDescs[0]._size = { 16, 16 };
layerDescs[1]._size = { 16, 16 };
layerDescs[2]._size = { 16, 16 };
neo::PredictiveHierarchy ph;
ph.createRandom(cs, prog, { static_cast<int>(visDim), static_cast<int>(visDim) }, layerDescs, { -0.01f, 0.01f }, generator);
ph._whiteningKernelRadius = 1;
// Learn levels
std::uniform_int_distribution<int> levelDist(0, levels.size() - 1);
for (int iter = 0; iter < 20; iter++) {
// Choose random level
int index = levelDist(generator);
const Level &l = levels[index];
// Set mode for tiles
inputData[charsetSize - 2] = 0.0f;
inputData[charsetSize - 1] = 1.0f;
// Run through once to get PH ready
inputData['#'] = 1.0f;
cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, inputData.data());
ph.simStep(cs, input);
inputData['#'] = 0.0f;
// Run through tile data
for (int i = 0; i < l._tileData.length(); i++) {
// Set character to 1
inputData[l._tileData[i]] = 1.0f;
cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, inputData.data());
ph.simStep(cs, input);
// Unset character
inputData[l._tileData[i]] = 0.0f;
}
// Set mode for objects
inputData[charsetSize - 2] = 1.0f;
inputData[charsetSize - 1] = 0.0f;
// Run through once to get PH ready
inputData['#'] = 1.0f;
cs.getQueue().enqueueWriteImage(input, CL_TRUE, { 0, 0, 0 }, { visDim, visDim, 1 }, 0, 0, inputData.data());
ph.simStep(cs, input);
inputData['#'] = 0.0f;
// Run through object data
for (int i = 0; i < l._objectData.length(); i++) {
// Set character to 1
inputData[l._objectData[i]] = 1.0f;
cs.getQueue().enqueueWriteImage(input, CL_TRUE, { 0, 0, 0 }, { visDim, visDim, 1 }, 0, 0, inputData.data());
ph.simStep(cs, input);
// Unset character
inputData[l._objectData[i]] = 0.0f;
}
std::cout << "Went over level #" << (index + 1) << " \"" << l._name << "\"" << std::endl;
}
// Generate new maps
std::ofstream toFile("generatedNLevels.txt");
std::normal_distribution<float> noiseDist(0.0f, 1.0f);
for (int i = 0; i < 10; i++) {
toFile << "$" << "Generated Level " << (i + 1) << "#NeoRL#Experimental#";
// Generated level data
// Set mode for tiles
inputData[charsetSize - 2] = 0.0f;
inputData[charsetSize - 1] = 1.0f;
// Run through once to get PH ready
//cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, inputData.data());
//ph.simStep(cs, input);
char prevChar = 0;
for (int i = 0; i < numTiles; i++) {
// Set character to 1
inputData[prevChar] = 1.0f;
cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, inputData.data());
ph.simStep(cs, input, false);
// Unset character
inputData[prevChar] = 0.0f;
char newChar = 0;
cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, predData.data());
for (int j = 1; j < charsetSize - 2; j++)
if (predData[j] > predData[newChar])
newChar = j;
// Add new character
toFile << newChar;
prevChar = newChar;
}
toFile << "|";
// Set mode for objects
inputData[charsetSize - 2] = 1.0f;
inputData[charsetSize - 1] = 0.0f;
// Run through once to get PH ready
//cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, inputData.data());
//ph.simStep(cs, input);
prevChar = 0;
for (int i = 0; i < maxObjects; i++) {
// Set character to 1
inputData[prevChar] = 1.0f;
std::vector<float> noisyInputData = inputData;
for (int j = 0; j < noisyInputData.size(); j++) {
noisyInputData[j] += noiseDist(generator) * 0.1f;
}
cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, inputData.data());
ph.simStep(cs, input, false);
// Unset character
inputData[prevChar] = 0.0f;
char newChar = 0;
cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, cl::array<cl::size_type, 3> { 0, 0, 0 }, cl::array<cl::size_type, 3> { visDim, visDim, 1 }, 0, 0, predData.data());
for (int j = 1; j < charsetSize - 2; j++)
if (predData[j] > predData[newChar])
newChar = j;
// If is delimiter, break
if (newChar == '#')
break;
// Add new character
toFile << newChar;
prevChar = newChar;
}
toFile << "#" << std::endl << std::endl;
}
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
dodgeballPositions.resize(1);
dodgeballVelocities.resize(dodgeballPositions.size());
std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
for (int i = 0; i < dodgeballPositions.size(); i++) {
dodgeballPositions[i] = sf::Vector2f(dist01(generator), dist01(generator));
dodgeballVelocities[i].x = dist01(generator) * 2.0f - 1.0f;
dodgeballVelocities[i].y = dist01(generator) * 2.0f - 1.0f;
dodgeballVelocities[i] = dodgeballVelocities[i] * (ballSpeed / std::sqrt(dodgeballVelocities[i].x * dodgeballVelocities[i].x + dodgeballVelocities[i].y * dodgeballVelocities[i].y));
}
agentPosition = sf::Vector2f(0.5f, 0.5f);
sf::RenderWindow window;
window.create(sf::VideoMode(800, 800), "BIDInet", sf::Style::Default);
window.setFramerateLimit(60);
window.setVerticalSyncEnabled(true);
sf::RenderTexture visionRT;
visionRT.create(16, 16);
deep::CSRL swarm;
std::vector<deep::CSRL::LayerDesc> layerDescs(4);
layerDescs[0]._width = 16;
layerDescs[0]._height = 16;
layerDescs[1]._width = 12;
layerDescs[1]._height = 12;
layerDescs[2]._width = 8;
layerDescs[2]._height = 8;
layerDescs[3]._width = 4;
layerDescs[3]._height = 4;
swarm.createRandom(2, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.1f, generator);
// ---------------------------- Game Loop -----------------------------
bool quit = false;
sf::Clock clock;
float dt = 0.017f;
float averageReward = 0.0f;
const float averageRewardDecay = 0.003f;
int steps = 0;
do {
clock.restart();
// ----------------------------- Input -----------------------------
sf::Event windowEvent;
while (window.pollEvent(windowEvent))
{
switch (windowEvent.type)
{
case sf::Event::Closed:
quit = true;
break;
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
quit = true;
visionRT.clear();
renderScene(visionRT);
visionRT.display();
sf::Image img = visionRT.getTexture().copyToImage();
for (int x = 0; x < img.getSize().x; x++)
for (int y = 0; y < img.getSize().y; y++) {
sf::Color c = img.getPixel(x, y);
float valR = 0.0f;
float valG = 0.0f;
if (c.r > 0)
valR = 1.0f;
if (c.g > 0)
valG = 1.0f;
swarm.setState(x, y, 0, valR);
swarm.setState(x, y, 1, valG);
}
float reward = 0.5f;
for (int i = 0; i < dodgeballPositions.size(); i++) {
sf::Vector2f delta = dodgeballPositions[i] - agentPosition;
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
//if (dist < (ballRadius + agentRadius)) {
reward += -dist;
//}
}
reward *= 10.0f;
averageReward = (1.0f - averageRewardDecay) * averageReward + averageRewardDecay * reward;
swarm.simStep(1, reward, generator);
//agent.simStep(reward, 0.1f, 0.99f, 0.01f, 0.2f, 0.01f, 0.01f, 0.01f, 64, 0.05f, 0.98f, 0.04f, 0.01f, 0.01f, 4.0f, generator);
agentPosition.x += agentSpeed * (swarm.getAction(3, 4, 0) * 2.0f - 1.0f);
agentPosition.y += agentSpeed * (swarm.getAction(3, 8, 0) * 2.0f - 1.0f);
//agentPosition.x += agentSpeed * (agent.getAction(0) * 2.0f - 1.0f);
//agentPosition.y += agentSpeed * (agent.getAction(1) * 2.0f - 1.0f);
agentPosition.x = std::min(0.5f + agentFieldRadius - agentRadius, std::max(0.5f - agentFieldRadius + agentRadius, agentPosition.x));
agentPosition.y = std::min(0.5f + agentFieldRadius - agentRadius, std::max(0.5f - agentFieldRadius + agentRadius, agentPosition.y));
for (int i = 0; i < dodgeballPositions.size(); i++) {
sf::Vector2f delta = agentPosition - dodgeballPositions[i];
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
dodgeballVelocities[i] += -dodgeballVelocities[i] * ballVelocityDecay + delta / dist * attraction;
dodgeballPositions[i] += dodgeballVelocities[i];
if (dodgeballPositions[i].x < 0.5f - agentFieldRadius + ballRadius) {
dodgeballPositions[i].x = 0.5f - agentFieldRadius + ballRadius;
dodgeballVelocities[i].x *= -1.0f;
}
else if (dodgeballPositions[i].x > 0.5f + agentFieldRadius - ballRadius) {
dodgeballPositions[i].x = 0.5f + agentFieldRadius - ballRadius;
dodgeballVelocities[i].x *= -1.0f;
}
if (dodgeballPositions[i].y < 0.5f - agentFieldRadius + ballRadius) {
dodgeballPositions[i].y = 0.5f - agentFieldRadius + ballRadius;
dodgeballVelocities[i].y *= -1.0f;
}
else if (dodgeballPositions[i].y > 0.5f + agentFieldRadius - ballRadius) {
dodgeballPositions[i].y = 0.5f + agentFieldRadius - ballRadius;
dodgeballVelocities[i].y *= -1.0f;
}
}
if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
window.clear();
renderScene(window);
sf::Sprite vis;
vis.setTexture(visionRT.getTexture());
vis.setScale(4.0f, 4.0f);
window.draw(vis);
window.display();
}
if (steps % 100 == 0)
std::cout << "Steps: " << steps << " Average Reward: " << averageReward << std::endl;
//dt = clock.getElapsedTime().asSeconds();
steps++;
} while (!quit);
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
std::vector<float> timeSeries;
std::ifstream fromFile("resources/pressures.txt");
if (!fromFile.is_open()) {
std::cerr << "Could not open pressures.txt!" << std::endl;
return 1;
}
float minVal = 999.0f;
float maxVal = -999.0f;
while (fromFile.good() && !fromFile.eof()) {
float value;
fromFile >> value;
timeSeries.push_back(value);
minVal = std::min(minVal, value);
maxVal = std::max(maxVal, value);
}
// Rescale
for (int i = 0; i < timeSeries.size(); i++) {
timeSeries[i] = (timeSeries[i] - minVal) / std::max(0.0001f, (maxVal - minVal));
}
int partition = timeSeries.size() * 0.5f;
/*timeSeries.clear();
timeSeries.resize(10);
timeSeries[0] = { 0.0f, 1.0f, 0.0f };
timeSeries[1] = { 0.0f, 0.0f, 0.0f };
timeSeries[2] = { 1.0f, 1.0f, 0.0f };
timeSeries[3] = { 0.0f, 0.0f, 1.0f };
timeSeries[4] = { 0.0f, 1.0f, 0.0f };
timeSeries[5] = { 0.0f, 0.0f, 1.0f };
timeSeries[6] = { 0.0f, 0.0f, 0.0f };
timeSeries[7] = { 0.0f, 0.0f, 0.0f };
timeSeries[8] = { 0.0f, 1.0f, 0.0f };
timeSeries[9] = { 0.0f, 1.0f, 1.0f };*/
std::vector<sdr::IPredictiveRSDR::LayerDesc> layerDescs(3);
layerDescs[0]._width = 24;
layerDescs[0]._height = 24;
layerDescs[1]._width = 16;
layerDescs[1]._height = 16;
layerDescs[2]._width = 8;
layerDescs[2]._height = 8;
sdr::IPredictiveRSDR prsdr;
prsdr.createRandom(1, 1, 16, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.1f, generator);
float avgError = 1.0f;
float avgErrorDecay = 0.1f;
vis::Plot plot;
plot._curves.push_back(vis::Curve());
plot._curves[0]._name = "Squared Error";
for (int iter = 0; iter < 5; iter++) {
for (int i = 0; i < partition; i++) {
float error = prsdr.getPrediction(0) - timeSeries[i];
float error2 = error * error;
avgError = (1.0f - avgErrorDecay) * avgError + avgErrorDecay * error2;
prsdr.setInput(0, timeSeries[i]);
prsdr.simStep(generator);
if (i % 10 == 0) {
std::cout << "Iteration " << i << ": " << avgError << std::endl;
}
}
}
int step = 0;
for (int i = 0; i < partition; i++) {
float error = prsdr.getPrediction(0) - timeSeries[i];
float error2 = error * error;
avgError = (1.0f - avgErrorDecay) * avgError + avgErrorDecay * error2;
prsdr.setInput(0, timeSeries[i]);
prsdr.simStep(generator, false);
if (step % 10 == 0) {
std::cout << "Iteration " << i << ": " << avgError << std::endl;
}
vis::Point p;
p._position.x = step;
p._position.y = avgError;
p._color = sf::Color::Red;
plot._curves[0]._points.push_back(p);
step++;
}
sf::RenderTexture rt;
rt.create(1024, 1024);
sf::Texture lineGradientTexture;
lineGradientTexture.loadFromFile("resources/lineGradient.png");
sf::Font tickFont;
tickFont.loadFromFile("resources/arial.ttf");
plot.draw(rt, lineGradientTexture, tickFont, 1.0f, sf::Vector2f(0.0f, step), sf::Vector2f(0.0f, 1.0f), sf::Vector2f(128.0f, 128.0f), sf::Vector2f(500.0f, 0.1f), 2.0f, 3.0f, 1.5f, 3.0f, 20.0f, 6);
rt.display();
rt.getTexture().copyToImage().saveToFile("plot.png");
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
std::vector<std::vector<double>> timeSeries;
std::ifstream fromFile("resources/data.txt");
if (!fromFile.is_open()) {
std::cerr << "Could not open data.txt!" << std::endl;
return 1;
}
// Skip first line
std::string line;
std::getline(fromFile, line);
int numEntries = 1;
for (int i = 0; i < line.size(); i++)
if (line[i] == ',')
numEntries++;
int numSkipEntries = 1;
int numEntriesUse = numEntries - numSkipEntries;
std::vector<double> minimums(numEntriesUse, 999999999.0);
std::vector<double> maximums(numEntriesUse, -999999999.0);
while (fromFile.good() && !fromFile.eof()) {
std::vector<double> entries(numEntriesUse);
std::string line;
std::getline(fromFile, line);
std::istringstream fromLine(line);
std::string param;
// Skip entries
for (int i = 0; i < numSkipEntries; i++) {
std::string entry;
std::getline(fromLine, entry, ',');
}
for (int i = 0; i < numEntriesUse; i++) {
std::string entry;
std::getline(fromLine, entry, ',');
if (entry == "")
entries[i] = 0.0;
else {
double value = std::stod(entry);
maximums[i] = std::max(maximums[i], value);
minimums[i] = std::min(minimums[i], value);
entries[i] = value;
}
}
timeSeries.push_back(entries);
}
// Rescale
for (int i = 0; i < timeSeries.size(); i++) {
for (int j = 0; j < timeSeries[i].size(); j++) {
timeSeries[i][j] = (timeSeries[i][j] - minimums[j]) / std::max(0.0001, (maximums[j] - minimums[j]));
}
}
/*timeSeries.clear();
timeSeries.resize(10);
timeSeries[0] = { 0.0f, 1.0f, 0.0f };
timeSeries[1] = { 0.0f, 0.0f, 0.0f };
timeSeries[2] = { 1.0f, 1.0f, 0.0f };
timeSeries[3] = { 0.0f, 0.0f, 1.0f };
timeSeries[4] = { 0.0f, 1.0f, 0.0f };
timeSeries[5] = { 0.0f, 0.0f, 1.0f };
timeSeries[6] = { 0.0f, 0.0f, 0.0f };
timeSeries[7] = { 0.0f, 0.0f, 0.0f };
timeSeries[8] = { 0.0f, 1.0f, 0.0f };
timeSeries[9] = { 0.0f, 1.0f, 1.0f };*/
std::vector<sdr::IPredictiveRSDR::LayerDesc> layerDescs(3);
layerDescs[0]._width = 8;
layerDescs[0]._height = 8;
layerDescs[1]._width = 6;
layerDescs[1]._height = 6;
layerDescs[2]._width = 4;
layerDescs[2]._height = 4;
sdr::IPredictiveRSDR prsdr;
prsdr.createRandom(4, 5, 8, layerDescs, -0.01f, 0.01f, 0.0f, generator);
float avgError = 1.0f;
float avgErrorDecay = 0.01f;
for (int iter = 0; iter < 1000; iter++) {
for (int i = 0; i < timeSeries.size(); i++) {
float error = 0.0f;
for (int j = 0; j < timeSeries[i].size(); j++) {
error += std::pow(prsdr.getPrediction(j) - timeSeries[i][j], 2);
prsdr.setInput(j, timeSeries[i][j]);
}
avgError = (1.0f - avgErrorDecay) * avgError + avgErrorDecay * error;
prsdr.simStep(generator);
if (i % 10 == 0) {
std::cout << "Iteration " << i << ": " << avgError << std::endl;
}
}
}
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
_ballPosition = sf::Vector2f(0.5f, 0.5f);
_ballVelocity = sf::Vector2f(0.44f, 0.55f);
_ballVelocity *= ballSpeed / std::sqrt(_ballVelocity.x * _ballVelocity.x + _ballVelocity.y * _ballVelocity.y);
_paddlePosition = 0.5f;
std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
sf::RenderWindow window;
window.create(sf::VideoMode(800, 800), "BIDInet", sf::Style::Default);
window.setFramerateLimit(60);
window.setVerticalSyncEnabled(true);
sf::RenderTexture visionRT;
visionRT.create(16, 16);
deep::CSRL agent;
std::vector<deep::CSRL::LayerDesc> layerDescs(3);
layerDescs[0]._width = 6;
layerDescs[0]._height = 6;
layerDescs[1]._width = 5;
layerDescs[1]._height = 5;
layerDescs[2]._width = 4;
layerDescs[2]._height = 4;
int inWidth = 16;
int inHeight = 18;
std::vector<deep::CSRL::InputType> inputTypes(inWidth * inHeight, deep::CSRL::_state);
for (int i = 0; i < inWidth; i++) {
inputTypes[i + (inHeight - 2) * inWidth] = deep::CSRL::_action;
inputTypes[i + (inHeight - 1) * inWidth] = deep::CSRL::_q;
}
agent.createRandom(inWidth, inHeight, 8, inputTypes, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.2f, generator);
// ---------------------------- Game Loop -----------------------------
bool quit = false;
sf::Clock clock;
float dt = 0.017f;
float averageReward = 0.0f;
const float averageRewardDecay = 0.003f;
int steps = 0;
do {
clock.restart();
// ----------------------------- Input -----------------------------
sf::Event windowEvent;
while (window.pollEvent(windowEvent))
{
switch (windowEvent.type)
{
case sf::Event::Closed:
quit = true;
break;
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
quit = true;
visionRT.clear();
renderScene(visionRT);
visionRT.display();
sf::Image img = visionRT.getTexture().copyToImage();
for (int x = 0; x < img.getSize().x; x++)
for (int y = 0; y < img.getSize().y; y++) {
sf::Color c = img.getPixel(x, y);
/*float valR = 0.0f;
float valG = 0.0f;
if (c.r > 0)
valR = 1.0f;
if (c.g > 0)
valG = 1.0f;
swarm.setState(x, y, 0, valR);
swarm.setState(x, y, 1, valG);*/
float val = 0.0f;
if (c.r > 0)
val = 0.5f;
if (c.g > 0)
val = 1.0f;
agent.setInput(x, y, val);
}
float reward = 0.0f;
if (_ballPosition.x < 0.0f) {
_ballPosition.x = 0.0f;
_ballVelocity.x *= -1.0f;
}
if (_ballPosition.y < 0.0f) {
_ballPosition.y = 0.0f;
_ballVelocity.y *= -1.0f;
}
if (_ballPosition.x > 1.0f) {
_ballPosition.x = 1.0f;
_ballVelocity.x *= -1.0f;
}
if (_ballPosition.y > 1.0f - bottomRatio) {
_ballPosition.y = 1.0f - bottomRatio;
if (_ballPosition.x > _paddlePosition - paddleWidthRatio && _ballPosition.x < _paddlePosition + paddleWidthRatio) {
reward += 10.0f;
}
else
reward -= 5.0f;
_ballVelocity.y *= -1.0f;
}
_ballPosition += _ballVelocity;
averageReward = (1.0f - averageRewardDecay) * averageReward + averageRewardDecay * reward;
agent.simStep(reward, generator);
float act = 0.0f;
for (int i = 4; i < 5; i++) {
act += agent.getPrediction(i, 16);
}
_paddlePosition = std::min(1.0f, std::max(0.0f, _paddlePosition + 0.1f * std::min(1.0f, std::max(-1.0f, act))));
//std::cout << averageReward << std::endl;
if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
window.clear();
renderScene(window);
sf::Sprite vis;
vis.setTexture(visionRT.getTexture());
vis.setScale(4.0f, 4.0f);
window.draw(vis);
sf::Image predImg;
predImg.create(16, 17);
for (int x = 0; x < 16; x++)
for (int y = 0; y < 17; y++) {
sf::Color c = sf::Color::White;
c.r = c.g = c.b = 255.0f * std::min(1.0f, std::max(0.0f, agent.getPrediction(x, y)));
predImg.setPixel(x, y, c);
}
sf::Texture t;
t.loadFromImage(predImg);
sf::Sprite s;
s.setTexture(t);
s.setScale(4.0f, 4.0f);
s.setPosition(4.0f * 16.0f, 0.0f);
window.draw(s);
window.display();
}
if (steps % 100 == 0)
std::cout << "Steps: " << steps << " Average Reward: " << averageReward << std::endl;
//dt = clock.getElapsedTime().asSeconds();
steps++;
} while (!quit);
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
sys::ComputeSystem cs;
cs.create(sys::ComputeSystem::_gpu);
sys::ComputeProgram prog;
prog.loadFromFile("resources/neoKernels.cl", cs);
// --------------------------- Create the Sparse Coder ---------------------------
cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 64, 64);
std::ifstream fromFile("resources/train-images.idx3-ubyte", std::ios::binary | std::ios::in);
if (!fromFile.is_open()) {
std::cerr << "Could not open train-images.idx3-ubyte!" << std::endl;
return 1;
}
std::vector<neo::PredictiveHierarchy::LayerDesc> layerDescs(4);
layerDescs[0]._size = { 64, 64 };
layerDescs[0]._feedForwardRadius = 8;
layerDescs[1]._size = { 48, 48 };
layerDescs[2]._size = { 32, 32 };
layerDescs[3]._size = { 24, 24 };
neo::PredictiveHierarchy ph;
ph.createRandom(cs, prog, { 64, 64 }, layerDescs, { -0.01f, 0.01f }, { 0.01f, 0.05f }, 0.1f, generator);
float avgError = 1.0f;
float avgErrorDecay = 0.1f;
sf::RenderWindow window;
window.create(sf::VideoMode(1024, 512), "MNIST Video Test");
vis::Plot plot;
plot._curves.push_back(vis::Curve());
plot._curves[0]._name = "Squared Error";
std::uniform_int_distribution<int> digitDist(0, 59999);
std::uniform_real<float> dist01(0.0f, 1.0f);
sf::RenderTexture rt;
rt.create(64, 64);
sf::Image digit0;
sf::Texture digit0Tex;
sf::Image digit1;
sf::Texture digit1Tex;
sf::Image pred;
sf::Texture predTex;
digit0.create(28, 28);
digit1.create(28, 28);
pred.create(rt.getSize().x, rt.getSize().y);
const float boundingSize = (64 - 28) / 2;
const float center = 32;
const float minimum = center - boundingSize;
const float maximum = center + boundingSize;
float avgError2 = 1.0f;
const float avgError2Decay = 0.01f;
std::vector<float> prediction(64 * 64, 0.0f);
for (int iter = 0; iter < 10000; iter++) {
// Select digit indices
int d0 = digitDist(generator);
int d1 = digitDist(generator);
// Load digits
Image img0, img1;
loadMNISTimage(fromFile, d0, img0);
loadMNISTimage(fromFile, d1, img1);
for (int x = 0; x < digit0.getSize().x; x++)
for (int y = 0; y < digit0.getSize().y; y++) {
int index = x + y * digit0.getSize().x;
sf::Color c = sf::Color::White;
c.a = img0._intensities[index];
digit0.setPixel(x, y, c);
}
digit0Tex.loadFromImage(digit0);
for (int x = 0; x < digit1.getSize().x; x++)
for (int y = 0; y < digit1.getSize().y; y++) {
int index = x + y * digit1.getSize().x;
sf::Color c = sf::Color::White;
c.a = img1._intensities[index];
digit1.setPixel(x, y, c);
}
digit1Tex.loadFromImage(digit1);
sf::Vector2f vel0(dist01(generator) * 2.0f - 1.0f, dist01(generator) * 2.0f - 1.0f);
sf::Vector2f vel1(dist01(generator) * 2.0f - 1.0f, dist01(generator) * 2.0f - 1.0f);
sf::Vector2f pos0(dist01(generator) * (maximum - minimum) + minimum, dist01(generator) * (maximum - minimum) + minimum);
sf::Vector2f pos1(dist01(generator) * (maximum - minimum) + minimum, dist01(generator) * (maximum - minimum) + minimum);
float vel0mul = dist01(generator) * 6.0f / std::max(1.0f, std::sqrt(vel0.x * vel0.x + vel0.y + vel0.y));
vel0 *= vel0mul;
float vel1mul = dist01(generator) * 6.0f / std::max(1.0f, std::sqrt(vel1.x * vel1.x + vel1.y + vel1.y));
vel1 *= vel1mul;
// Render video
for (int f = 0; f < 20; f++) {
sf::Event windowEvent;
while (window.pollEvent(windowEvent)) {
switch (windowEvent.type) {
case sf::Event::Closed:
return 0;
}
}
pos0 += vel0;
pos1 += vel1;
if (pos0.x < minimum) {
pos0.x = minimum;
vel0.x *= -1.0f;
}
else if (pos0.x > maximum) {
pos0.x = maximum;
vel0.x *= -1.0f;
}
if (pos0.y < minimum) {
pos0.y = minimum;
vel0.y *= -1.0f;
}
else if (pos0.y > maximum) {
pos0.y = maximum;
vel0.y *= -1.0f;
}
if (pos1.x < minimum) {
pos1.x = minimum;
vel1.x *= -1.0f;
}
else if (pos1.x > maximum) {
pos1.x = maximum;
vel1.x *= -1.0f;
}
if (pos1.y < minimum) {
pos1.y = minimum;
vel1.y *= -1.0f;
}
else if (pos1.y > maximum) {
pos1.y = maximum;
vel1.y *= -1.0f;
}
window.clear();
rt.clear(sf::Color::Black);
sf::Sprite s0;
s0.setTexture(digit0Tex);
s0.setOrigin(28 / 2, 28 / 2);
s0.setPosition(pos0);
rt.draw(s0);
sf::Sprite s1;
s1.setTexture(digit1Tex);
s1.setOrigin(28 / 2, 28 / 2);
s1.setPosition(pos1);
rt.draw(s1);
rt.display();
// Get input image
sf::Image res = rt.getTexture().copyToImage();
// Show RT
const float scale = 4.0f;
sf::Sprite s;
s.setScale(scale, scale);
s.setTexture(rt.getTexture());
window.draw(s);
std::vector<float> input(64 * 64);
// Train
if (sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
for (int x = 0; x < res.getSize().x; x++)
for (int y = 0; y < res.getSize().y; y++) {
input[x + y * 64] = prediction[x + y * 64];
}
}
else {
const float predictionIncorporateRatio = 0.1f;
for (int x = 0; x < res.getSize().x; x++)
for (int y = 0; y < res.getSize().y; y++) {
input[x + y * 64] = (1.0f - predictionIncorporateRatio) * res.getPixel(x, y).r / 255.0f + predictionIncorporateRatio * prediction[x + y * 64];
}
}
// Error
float error = 0.0f;
for (int x = 0; x < res.getSize().x; x++)
for (int y = 0; y < res.getSize().y; y++) {
error += std::pow(res.getPixel(x, y).r / 255.0f - prediction[x + y * 64], 2);
}
error /= res.getSize().x * res.getSize().y;
avgError2 = (1.0f - avgError2Decay) * avgError2 + avgError2Decay * error;
std::cout << "Squared Error: " << avgError2 << std::endl;
cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 64, 64, 1 }, 0, 0, input.data());
ph.simStep(cs, inputImage);
cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, { 0, 0, 0 }, { 64, 64, 1 }, 0, 0, prediction.data());
// Show prediction
for (int x = 0; x < rt.getSize().x; x++)
for (int y = 0; y < rt.getSize().y; y++) {
sf::Color c = sf::Color::White;
c.r = c.b = c.g = std::min(1.0f, std::max(0.0f, prediction[x + y * 64])) * 255.0f;
pred.setPixel(x, y, c);
}
predTex.loadFromImage(pred);
sf::Sprite sp;
sp.setTexture(predTex);
sp.setScale(scale, scale);
sp.setPosition(window.getSize().x - scale * rt.getSize().x, 0);
window.draw(sp);
/*sf::Image sdr;
sdr.create(prsdr.getLayerDescs().front()._width, prsdr.getLayerDescs().front()._height);
for (int x = 0; x < sdr.getSize().x; x++)
for (int y = 0; y < sdr.getSize().y; y++) {
sf::Color c = sf::Color::White;
c.r = c.g = c.b = prsdr.getLayers().front()._sdr.getHiddenState(x, y) * 255.0f;
sdr.setPixel(x, y, c);
}
sf::Texture sdrTex;
sdrTex.loadFromImage(sdr);
sf::Sprite sdrS;
sdrS.setTexture(sdrTex);
sdrS.setPosition(0.0f, window.getSize().y - sdrTex.getSize().y * scale);
sdrS.setScale(scale, scale);
window.draw(sdrS);*/
window.display();
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
return 0;
}
}
/*sf::RenderTexture rt;
rt.create(1024, 1024);
sf::Texture lineGradientTexture;
lineGradientTexture.loadFromFile("resources/lineGradient.png");
sf::Font tickFont;
tickFont.loadFromFile("resources/arial.ttf");
plot.draw(rt, lineGradientTexture, tickFont, 1.0f, sf::Vector2f(0.0f, step), sf::Vector2f(0.0f, 1.0f), sf::Vector2f(128.0f, 128.0f), sf::Vector2f(500.0f, 0.1f), 2.0f, 3.0f, 1.5f, 3.0f, 20.0f, 6);
rt.display();
rt.getTexture().copyToImage().saveToFile("plot.png");*/
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
sys::ComputeSystem cs;
cs.create(sys::ComputeSystem::_gpu);
sys::ComputeProgram program;
program.loadFromFile("resources/htmrl.cl", cs);
htm::HTMRL htmrl;
std::vector<htm::HTMRL::LayerDesc> layerDescs(1);
layerDescs[0]._width = 64;
layerDescs[0]._height = 64;
//layerDescs[1]._width = 64;
//layerDescs[1]._height = 64;
//layerDescs[2]._width = 64;
//layerDescs[2]._height = 64;
htmrl.createRandom(cs, program, 64, 64, layerDescs, -0.1f, 0.1f, generator);
/*htm::HTMFeatureExtractor fExt;
std::vector<htm::HTMFeatureExtractor::LayerDesc> layerDescs(3);
layerDescs[0]._width = 128;
layerDescs[0]._height = 128;
layerDescs[0]._receptiveFieldRadius = 2;
layerDescs[1]._width = 64;
layerDescs[1]._height = 64;
layerDescs[1]._receptiveFieldRadius = 2;
layerDescs[2]._width = 32;
layerDescs[2]._height = 32;
layerDescs[2]._receptiveFieldRadius = 2;*/
float t = 0;
float dotRadius = 16.0f;
bool quit = false;
sf::Clock clock;
sf::RenderWindow window;
window.create(sf::VideoMode(512, 512), "HTM", sf::Style::Default);
//window.setFramerateLimit(60);
do {
clock.restart();
// ----------------------------- Input -----------------------------
sf::Event windowEvent;
while (window.pollEvent(windowEvent)) {
switch (windowEvent.type) {
case sf::Event::Closed:
quit = true;
break;
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
quit = true;
if (sf::Keyboard::isKeyPressed(sf::Keyboard::P)) {
// Save cell data
htmrl.exportCellData(cs, "data");
}
float dotX = 32.0f;// std::cos(t * 4.0f) * 15.0f + 64.0f;
float dotY = 32.0f;
for (int x = 0; x < 64; x++)
for (int y = 0; y < 64; y++) {
//float r = inImage.getPixel(x, y).r / 255.0f;
//float g = inImage.getPixel(x, y).g / 255.0f;
//float b = inImage.getPixel(x, y).b / 255.0f;
float dist = std::sqrt(static_cast<float>(std::pow(dotX - x, 2) + std::pow(dotY - y, 2)));
float greyScale = std::max<float>(0.0f, std::min<float>(1.0f, (dotRadius - dist) / dotRadius)) * (std::cos(t) + 0.5f + 0.5f);
htmrl.setInput(x, y, greyScale);
}
if (!sf::Keyboard::isKeyPressed(sf::Keyboard::L)) {
for (int i = 0; i < 1; i++)
htmrl.step(cs, 0.05f, 0.05f);
t += 0.05f;
if (t > 6.28f)
t = 0.0f;
}
sf::Image outImage;
outImage.create(htmrl.getLayerDescs().back()._width, htmrl.getLayerDescs().back()._height);
if (sf::Keyboard::isKeyPressed(sf::Keyboard::O)) {
for (int x = 0; x < htmrl.getLayerDescs().back()._width; x++)
for (int y = 0; y < htmrl.getLayerDescs().back()._height; y++) {
float outputR = htmrl.getPrediction(x, y);
outImage.setPixel(x, y, sf::Color(outputR * 255.0f, outputR * 255.0f, outputR * 255.0f, 255));
}
}
else {
for (int x = 0; x < htmrl.getLayerDescs().back()._width; x++)
for (int y = 0; y < htmrl.getLayerDescs().back()._height; y++) {
float outputR = htmrl.getSDR(x, y);
outImage.setPixel(x, y, sf::Color(outputR * 255.0f, outputR * 255.0f, outputR * 255.0f, 255));
}
}
//outImage.saveToFile("out.png");
window.clear();
sf::Texture texture;
texture.loadFromImage(outImage);
sf::Sprite sprite;
sprite.setTexture(texture);
sprite.setScale(512.0f / htmrl.getLayerDescs().back()._width, 512.0f / htmrl.getLayerDescs().back()._height);
window.draw(sprite);
window.display();
} while (!quit);
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
sys::ComputeSystem cs;
cs.create(sys::ComputeSystem::_gpu);
sys::ComputeProgram prog;
prog.loadFromFile("resources/neoKernels.cl", cs);
std::vector<neo::AgentSPG::LayerDesc> layerDescs(3);
layerDescs[0]._size = { 32, 32 };
layerDescs[0]._alpha = { 0.1f, 0.001f };
layerDescs[0]._predictiveRadius = 12;
layerDescs[0]._feedBackRadius = 12;
layerDescs[1]._size = { 24, 24 };
layerDescs[2]._size = { 16, 16 };
neo::AgentSPG agent;
int inWidth = 32;
int inHeight = 32;
int aWidth = 3;
int aHeight = 3;
int frameSkip = 3;
agent.createRandom(cs, prog, { inWidth, inHeight }, { aWidth, aHeight }, 12, layerDescs, { -0.05f, 0.05f }, generator);
cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), inWidth, inHeight);
cl::Image2D actionImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), aWidth, aHeight);
sf::VideoMode videoMode(1280, 720);
sf::RenderWindow renderWindow;
renderWindow.create(videoMode, "Reinforcement Learning", sf::Style::Default);
renderWindow.setVerticalSyncEnabled(true);
renderWindow.setFramerateLimit(60);
// --------------------------------- Game Init -----------------------------------
const float slimeRadius = 94.5f;
const float ballRadius = 23.5f;
const float wallRadius = 22.5f;
const float fieldRadius = 640.0f;
const float gravity = 900.0f;
const float slimeBounce = 100.0f;
const float wallBounceDecay = 0.8f;
const float slimeJump = 500.0f;
const float maxSlimeSpeed = 1000.0f;
const float slimeMoveAccel = 5000.0f;
const float slimeMoveDeccel = 8.0f;
std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
sf::Vector2f fieldCenter = sf::Vector2f(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f + 254.0f);
sf::Vector2f wallCenter = fieldCenter + sf::Vector2f(0.0f, -182.0f);
PhyObj blue;
PhyObj red;
PhyObj ball;
blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
blue._velocity = sf::Vector2f(0.0f, 0.0f);
red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
red._velocity = sf::Vector2f(0.0f, 0.0f);
ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
ball._velocity = sf::Vector2f((dist01(generator)) * 600.0f, -(dist01(generator)) * 500.0f);
sf::Texture backgroundTexture;
backgroundTexture.loadFromFile("resources/slimevolleyball/background.png");
sf::Texture blueSlimeTexture;
blueSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyBlue.png");
sf::Texture redSlimeTexture;
redSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyRed.png");
sf::Texture ballTexture;
ballTexture.loadFromFile("resources/slimevolleyball/ball.png");
sf::Texture eyeTexture;
eyeTexture.loadFromFile("resources/slimevolleyball/slimeEye.png");
eyeTexture.setSmooth(true);
sf::Texture arrowTexture;
arrowTexture.loadFromFile("resources/slimevolleyball/arrow.png");
sf::Font scoreFont;
scoreFont.loadFromFile("resources/slimevolleyball/scoreFont.ttf");
int scoreRed = 0;
int scoreBlue = 0;
int prevScoreRed = 0;
int prevScoreBlue = 0;
float prevBallX = fieldCenter.x;
sf::RenderTexture blueRT;
blueRT.create(inWidth, inHeight);
// ------------------------------- Simulation Loop -------------------------------
bool quit = false;
float dt = 0.017f;
int skipFrameCounter = 0;
float blueReward = 0.0f;
float redReward = 0.0f;
std::vector<float> actionTemp(aWidth * aHeight, 0.5f);
std::vector<float> actionTemp2(aWidth * aHeight * 2, 0.5f);
do {
sf::Event event;
while (renderWindow.pollEvent(event)) {
switch (event.type) {
case sf::Event::Closed:
quit = true;
break;
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
quit = true;
// ---------------------------------- Physics ----------------------------------
bool blueBounced = false;
bool redBounced = false;
// Ball
{
ball._velocity.y += gravity * dt;
ball._position += ball._velocity * dt;
// To floor (game restart)
if (ball._position.y + ballRadius > fieldCenter.y) {
if (ball._position.x < fieldCenter.x)
scoreRed++;
else
scoreBlue++;
blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
blue._velocity = sf::Vector2f(0.0f, 0.0f);
red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
red._velocity = sf::Vector2f(0.0f, 0.0f);
ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
ball._velocity = sf::Vector2f(((dist01(generator))) * -600.0f, -(dist01(generator)) * 500.0f);
}
// To wall
if (((ball._position.x + ballRadius) > (wallCenter.x - wallRadius) && ball._position.x < wallCenter.x) || ((ball._position.x - ballRadius) < (wallCenter.x + wallRadius) && ball._position.x > wallCenter.x)) {
// If above rounded part
if (ball._position.y < wallCenter.y) {
sf::Vector2f delta = ball._position - wallCenter;
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
if (dist < wallRadius + ballRadius) {
sf::Vector2f normal = delta / dist;
// Reflect velocity
sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;
ball._velocity = reflectedVelocity * wallBounceDecay;
ball._position = wallCenter + normal * (wallRadius + ballRadius);
}
}
else {
// If on left side
if (ball._position.x < wallCenter.x) {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = wallCenter.x - wallRadius - ballRadius;
}
else {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = wallCenter.x + wallRadius + ballRadius;
}
}
}
// To blue slime
{
sf::Vector2f delta = ball._position - blue._position;
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
if (dist < slimeRadius + ballRadius) {
sf::Vector2f normal = delta / dist;
// Reflect velocity
sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;
float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);
sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;
ball._velocity = blue._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);
ball._position = blue._position + normal * (wallRadius + slimeRadius);
blueBounced = true;
}
}
// To red slime
{
sf::Vector2f delta = ball._position - red._position;
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
if (dist < slimeRadius + ballRadius) {
sf::Vector2f normal = delta / dist;
// Reflect velocity
sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;
float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);
sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;
ball._velocity = red._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);
ball._position = red._position + normal * (wallRadius + slimeRadius);
redBounced = true;
}
}
// Out of field, left and right
{
if (ball._position.x - ballRadius < fieldCenter.x - fieldRadius) {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = fieldCenter.x - fieldRadius + ballRadius;
}
else if (ball._position.x + ballRadius > fieldCenter.x + fieldRadius) {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = fieldCenter.x + fieldRadius - ballRadius;
}
}
}
// Blue slime
{
blue._velocity.y += gravity * dt;
blue._velocity.x += -slimeMoveDeccel * blue._velocity.x * dt;
blue._position += blue._velocity * dt;
bool moveLeft;
bool moveRight;
bool jump;
sf::Image img = blueRT.getTexture().copyToImage();
std::vector<float> greyData(img.getSize().x * img.getSize().y);
for (int x = 0; x < img.getSize().x; x++)
for (int y = 0; y < img.getSize().y; y++) {
sf::Color c = img.getPixel(x, y);
greyData[x + y * img.getSize().x] = 0.333f * (c.r / 255.0f + c.g / 255.0f + c.b / 255.0f);
}
if (skipFrameCounter == 0) {
cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { static_cast<cl::size_type>(inWidth), static_cast<cl::size_type>(inHeight), 1 }, 0, 0, greyData.data());
cs.getQueue().enqueueWriteImage(actionImage, CL_TRUE, { 0, 0, 0 }, { static_cast<cl::size_type>(inWidth), static_cast<cl::size_type>(inHeight), 1 }, 0, 0, actionTemp.data());
agent.simStep(cs, blueReward * 0.01f, inputImage, actionImage, generator);
cs.getQueue().enqueueReadImage(agent.getAction(), CL_TRUE, { 0, 0, 0 }, { static_cast<cl::size_type>(aWidth), static_cast<cl::size_type>(aHeight), 1 }, 0, 0, actionTemp2.data());
for (int i = 0; i < actionTemp.size(); i++) {
if (dist01(generator) < 0.1f)
actionTemp[i] = dist01(generator);
else
actionTemp[i] = actionTemp2[i * 2 + 0];
}
actionTemp[2] = actionTemp[0] * 3.0f + 1.0f;
actionTemp[3] = actionTemp[0] * -2.0f + 0.4f;
actionTemp[4] = actionTemp[0] * 2.0f - 0.4f;
actionTemp[5] = actionTemp[0] * -2.0f - 0.4f;
actionTemp[6] = actionTemp[1] * -1.5f + 0.2f;
actionTemp[7] = actionTemp[1] * 2.1f - 0.4f;
actionTemp[8] = actionTemp[1] * -2.0f + 0.4f;
cs.getQueue().finish();
}
moveLeft = actionTemp[0] > 0.75f;
moveRight = actionTemp[0] < 0.25f;
jump = actionTemp[1] > 0.75f;
std::cout << blueReward << " " << actionTemp[0 * 2 + 0] << " " << actionTemp[1 * 2 + 0] << std::endl;
if (moveLeft) {
blue._velocity.x += -slimeMoveAccel * dt;
if (blue._velocity.x < -maxSlimeSpeed)
blue._velocity.x = -maxSlimeSpeed;
}
else if (moveRight) {
blue._velocity.x += slimeMoveAccel * dt;
if (blue._velocity.x > maxSlimeSpeed)
blue._velocity.x = maxSlimeSpeed;
}
if (blue._position.y > fieldCenter.y) {
blue._velocity.y = 0.0f;
blue._position.y = fieldCenter.y;
if (jump)
blue._velocity.y -= slimeJump;
}
if (blue._position.x - slimeRadius < fieldCenter.x - fieldRadius) {
blue._velocity.x = 0.0f;
blue._position.x = fieldCenter.x - fieldRadius + slimeRadius;
}
if (blue._position.x + slimeRadius > wallCenter.x - wallRadius) {
blue._velocity.x = 0.0f;
blue._position.x = wallCenter.x - wallRadius - slimeRadius;
}
}
// Red slime
{
red._velocity.y += gravity * dt;
red._velocity.x += -slimeMoveDeccel * red._velocity.x * dt;
red._position += red._velocity * dt;
bool moveLeft;
bool moveRight;
bool jump;
moveLeft = sf::Keyboard::isKeyPressed(sf::Keyboard::Left);
moveRight = sf::Keyboard::isKeyPressed(sf::Keyboard::Right);
jump = sf::Keyboard::isKeyPressed(sf::Keyboard::Up);
if (moveLeft) {
red._velocity.x += -slimeMoveAccel * dt;
if (red._velocity.x < -maxSlimeSpeed)
red._velocity.x = -maxSlimeSpeed;
}
else if (moveRight) {
red._velocity.x += slimeMoveAccel * dt;
if (red._velocity.x > maxSlimeSpeed)
red._velocity.x = maxSlimeSpeed;
}
if (red._position.y > fieldCenter.y) {
red._velocity.y = 0.0f;
red._position.y = fieldCenter.y;
if (jump)
red._velocity.y -= slimeJump;
}
if (red._position.x + slimeRadius > fieldCenter.x + fieldRadius) {
red._velocity.x = 0.0f;
red._position.x = fieldCenter.x + fieldRadius - slimeRadius;
}
if (red._position.x - slimeRadius < wallCenter.x + wallRadius) {
red._velocity.x = 0.0f;
red._position.x = wallCenter.x + wallRadius + slimeRadius;
}
}
blueReward = scoreBlue - prevScoreBlue - (scoreRed - prevScoreRed);// -0.00005f * std::abs(ball._position.x - blue._position.x) + (blueBounced ? 0.2f : 0.0f);
redReward = scoreRed - prevScoreRed - (scoreBlue - prevScoreBlue);// -0.00005f * std::abs(ball._position.x - red._position.x) + (redBounced ? 0.2f : 0.0f);
prevScoreRed = scoreRed;
prevScoreBlue = scoreBlue;
prevBallX = ball._position.x;
// --------------------------------- Rendering ---------------------------------
if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
{
renderWindow.clear();
{
sf::Sprite s;
s.setTexture(backgroundTexture);
s.setOrigin(backgroundTexture.getSize().x * 0.5f, backgroundTexture.getSize().y * 0.5f);
s.setPosition(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(blueSlimeTexture);
s.setOrigin(blueSlimeTexture.getSize().x * 0.5f, blueSlimeTexture.getSize().y);
s.setPosition(blue._position);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(eyeTexture);
s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
s.setPosition(blue._position + sf::Vector2f(50.0f, -28.0f));
sf::Vector2f delta = ball._position - s.getPosition();
float angle = std::atan2(delta.y, delta.x);
s.setRotation(angle * 180.0f / 3.141596f);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(redSlimeTexture);
s.setOrigin(redSlimeTexture.getSize().x * 0.5f, redSlimeTexture.getSize().y);
s.setPosition(red._position);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(eyeTexture);
s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
s.setPosition(red._position + sf::Vector2f(-50.0f, -28.0f));
sf::Vector2f delta = ball._position - s.getPosition();
float angle = std::atan2(delta.y, delta.x);
s.setRotation(angle * 180.0f / 3.141596f);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(ballTexture);
s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
s.setPosition(ball._position);
renderWindow.draw(s);
}
if (ball._position.y + ballRadius < 0.0f) {
sf::Sprite s;
s.setTexture(arrowTexture);
s.setOrigin(arrowTexture.getSize().x * 0.5f, 0.0f);
s.setPosition(ball._position.x, 0.0f);
renderWindow.draw(s);
}
{
sf::Text scoreText;
scoreText.setFont(scoreFont);
scoreText.setString(std::to_string(scoreBlue));
scoreText.setCharacterSize(100);
float width = scoreText.getLocalBounds().width;
scoreText.setPosition(fieldCenter.x - width * 0.5f - 100.0f, 10.0f);
scoreText.setColor(sf::Color(100, 133, 255));
renderWindow.draw(scoreText);
}
{
sf::Text scoreText;
scoreText.setFont(scoreFont);
scoreText.setString(std::to_string(scoreRed));
scoreText.setCharacterSize(100);
float width = scoreText.getLocalBounds().width;
scoreText.setPosition(fieldCenter.x - width * 0.5f + 100.0f, 10.0f);
scoreText.setColor(sf::Color(255, 100, 100));
renderWindow.draw(scoreText);
}
{
sf::Sprite visionSprite;
visionSprite.setTexture(blueRT.getTexture());
visionSprite.scale(4.0f, 4.0f);
renderWindow.draw(visionSprite);
}
}
renderWindow.display();
}
blueRT.setView(renderWindow.getView());
{
blueRT.clear();
{
sf::Sprite s;
s.setTexture(backgroundTexture);
s.setOrigin(backgroundTexture.getSize().x * 0.5f, backgroundTexture.getSize().y * 0.5f);
s.setPosition(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f);
blueRT.draw(s);
}
{
sf::Sprite s;
s.setTexture(blueSlimeTexture);
s.setOrigin(blueSlimeTexture.getSize().x * 0.5f, blueSlimeTexture.getSize().y);
s.setPosition(blue._position);
blueRT.draw(s);
}
{
sf::Sprite s;
s.setTexture(eyeTexture);
s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
s.setPosition(blue._position + sf::Vector2f(50.0f, -28.0f));
sf::Vector2f delta = ball._position - s.getPosition();
float angle = std::atan2(delta.y, delta.x);
s.setRotation(angle * 180.0f / 3.141596f);
blueRT.draw(s);
}
{
sf::Sprite s;
s.setTexture(redSlimeTexture);
s.setOrigin(redSlimeTexture.getSize().x * 0.5f, redSlimeTexture.getSize().y);
s.setPosition(red._position);
blueRT.draw(s);
}
{
sf::Sprite s;
s.setTexture(eyeTexture);
s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
s.setPosition(red._position + sf::Vector2f(-50.0f, -28.0f));
sf::Vector2f delta = ball._position - s.getPosition();
float angle = std::atan2(delta.y, delta.x);
s.setRotation(angle * 180.0f / 3.141596f);
blueRT.draw(s);
}
{
sf::Sprite s;
s.setTexture(ballTexture);
s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
s.setPosition(ball._position);
blueRT.draw(s);
}
if (ball._position.y + ballRadius < 0.0f) {
sf::Sprite s;
s.setTexture(arrowTexture);
s.setOrigin(arrowTexture.getSize().x * 0.5f, 0.0f);
s.setPosition(ball._position.x, 0.0f);
blueRT.draw(s);
}
{
sf::Text scoreText;
scoreText.setFont(scoreFont);
scoreText.setString(std::to_string(scoreBlue));
scoreText.setCharacterSize(100);
float width = scoreText.getLocalBounds().width;
scoreText.setPosition(fieldCenter.x - width * 0.5f - 100.0f, 10.0f);
scoreText.setColor(sf::Color(100, 133, 255));
blueRT.draw(scoreText);
}
{
sf::Text scoreText;
scoreText.setFont(scoreFont);
scoreText.setString(std::to_string(scoreRed));
scoreText.setCharacterSize(100);
float width = scoreText.getLocalBounds().width;
scoreText.setPosition(fieldCenter.x - width * 0.5f + 100.0f, 10.0f);
scoreText.setColor(sf::Color(255, 100, 100));
blueRT.draw(scoreText);
}
blueRT.display();
}
skipFrameCounter = (skipFrameCounter + 1) % frameSkip;
} while (!quit);
return 0;
}
int main(int argc, const char** argv) {
// Load the command line config
ArgumentParser parser;
parser.addArgument("-e", "--epochs", 1);
parser.addArgument("-s", "--seed", 1);
parser.addArgument("-l", "--layers", 1);
parser.addArgument("-S", "--samples", 1);
parser.addArgument("-c", "--corpus", 1);
parser.addArgument("--nlayers", 1);
parser.addArgument("--ifbradius", 1);
parser.addArgument("--lw", 1);
parser.addArgument("--lh", 1);
parser.addArgument("--ssize", 1);
parser.addArgument("--sseednoise", 1);
parser.addArgument("--sprednoise", 1);
parser.parse(argc, argv);
// RNG
unsigned int seed = std::atoi(parser.retrieve("seed", std::to_string(time(nullptr))).c_str());
std::mt19937 generator(seed);
std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
// ---------------------------------- Load the Corpus ----------------------------------
std::string corpusPath = parser.retrieve("corpus", "corpus.txt");
std::ifstream fromFile(corpusPath);
fromFile.seekg(0, std::ios::end);
size_t size = fromFile.tellg();
std::string test(size, ' ');
fromFile.seekg(0);
fromFile.read(&test[0], size);
// ---------------------------------- Find Character Set ----------------------------------
VectorCodec textcodec(test);
int numInputs = textcodec.N;
int inputsRoot = std::ceil(std::sqrt(static_cast<float>(numInputs)));
// ---------------------------------- Create Hierarchy ----------------------------------
// Fill out layer descriptions
int nLayers = std::atoi(parser.retrieve("nlayers", "3").c_str());
int layerW = std::atoi(parser.retrieve("lw", "16").c_str());
int layerH = std::atoi(parser.retrieve("lh", "16").c_str());
int inFeedBackRadius = std::atoi(parser.retrieve("ifbradius", "16").c_str());
std::vector<neo::PredictiveHierarchy::LayerDesc> layerDescs(nLayers);
for (int i = 0; i < nLayers; i++) {
layerDescs[i]._width = layerW;
layerDescs[i]._height = layerH;
}
neo::PredictiveHierarchy ph;
ph.createRandom(inputsRoot, inputsRoot, inFeedBackRadius, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.1f, generator);
// ---------------------------------- Iterate Over Corpus ----------------------------------
int numEpochs = std::atoi(parser.retrieve("epochs", "10").c_str());
int numSamples = std::atoi(parser.retrieve("samples", "10").c_str());
int sampleSize = std::atoi(parser.retrieve("ssize", std::to_string(test.length())).c_str());
float sampleSeedNoise = std::atof(parser.retrieve("sseednoise", "0.5").c_str());
float samplePredNoise = std::atof(parser.retrieve("sprednoise", "0.05").c_str());
std::cout << "NeoRL text prediction experiment" << std::endl;
std::cout << "Corpus: " << corpusPath << " size: " << test.length() << " alphabet size: " << textcodec.nSymbols << std::endl;
std::cout << "Model: nLayers: " << nLayers << " layerW: " << layerW << " layerH: " << layerH << " inFeedbackRadius: " << inFeedBackRadius
<< " input: " << inputsRoot << "x" << inputsRoot << std::endl;
std::cout << "Training: epochs: " << numEpochs << std::endl;
std::cout << "Sampling: samples: " << numSamples << " size: " << sampleSize << " seed noise: " << sampleSeedNoise << " pred noise " << samplePredNoise << std::endl;
std::cout << "--[ Start training ]--" << std::endl;
train(ph, generator, test, numEpochs, textcodec);
std::cout << "--[ Start sampling ]--" << std::endl;
for (int i = 0; i < numSamples; i++) {
sample(ph, generator, textcodec.getRandomSymbol(generator), sampleSize, textcodec, sampleSeedNoise, samplePredNoise);
}
return 0;
}
int main() {
std::mt19937 generator(std::time(nullptr));
sys::ComputeSystem cs;
cs.create(sys::ComputeSystem::_gpu);
sys::ComputeProgram prog;
prog.loadFromFile("resources/neoKernels.cl", cs);
// --------------------------- Create the Sparse Coder ---------------------------
cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 4, 4);
std::vector<neo::PredictiveHierarchy::LayerDesc> layerDescs(3);
layerDescs[0]._size = { 16, 16 };
layerDescs[1]._size = { 12, 12 };
layerDescs[2]._size = { 8, 8 };
neo::PredictiveHierarchy ph;
ph.createRandom(cs, prog, { 4, 4 }, 12, layerDescs, { -0.01f, 0.01f }, generator);
std::uniform_int_distribution<int> item_dist(0, 9);
std::vector<float> inputVec(16, 0.0f);
float avg_error = 0.0f;
for (int train_iter = 0; train_iter < 1000; train_iter++) {
std::vector<int> items(10);
for (int show_iter = 0; show_iter < 10; show_iter++) {
items[show_iter] = item_dist(generator);
for (int i = 0; i < 16; i++)
inputVec[i] = 0.0f;
inputVec[items[show_iter]] = 1.0f;
cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data());
ph.simStep(cs, inputImage);
}
for (int i = 0; i < 16; i++)
inputVec[i] = 0.0f;
cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data());
for (int wait_iter = 0; wait_iter < 10; wait_iter++) {
ph.simStep(cs, inputImage);
}
// Show delimiter (item = 10)
for (int i = 0; i < 16; i++)
inputVec[i] = 0.0f;
inputVec[10] = 1.0f;
cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data());
ph.simStep(cs, inputImage);
float error = 0.0f;
std::vector<float> pred(16, 0.0f);
for (int recall_iter = 0; recall_iter < 10; recall_iter++) {
cs.getQueue().enqueueReadImage(ph.getFirstLayerPred().getHiddenStates()[neo::_back], CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, pred.data());
for (int i = 0; i < 16; i++) {
if (i == items[recall_iter])
error += std::pow(1.0f - pred[i], 2);
else
error += std::pow(0.0f - pred[i], 2);
}
for (int i = 0; i < 16; i++)
inputVec[i] = 0.0f;
inputVec[items[recall_iter]] = 1.0f;
cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data());
ph.simStep(cs, inputImage);
}
std::cout << error << std::endl;
}
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
/*std::vector<sparse::SDRSPS> sdrsps(4);
std::cout << "Initializing..." << std::endl;
for (int i = 0; i < sdrsps.size(); i++)
sdrsps[i].create(1000, 1000, 1000, 1000, 3, -0.01f, 0.01f, generator);
std::cout << "Initialized." << std::endl;
std::uniform_int_distribution<int> bitDist(0, 1000 * 1000 - 1);
for (int iter = 0; iter < 10000; iter++) {
std::vector<int> bits(1000);
for (int i = 0; i < bits.size(); i++) {
bits[i] = bitDist(generator);
}
sdrsps.front().activate(bits, 100, 100, 3, 0.001f);
for (int i = 1; i < sdrsps.size(); i++) {
sdrsps[i].activate(sdrsps[i - 1].getBitIndices(), 100, 100, 3, 0.001f);
}
std::cout << "Iteration " << iter << std::endl;
}*/
std::vector<sparse::PredictiveHierarchy::LayerDesc> layerDescs(1);
sparse::PredictiveHierarchy ph;
ph.create(4, 4, layerDescs, -0.01f, 0.01f, generator);
std::vector<std::vector<sparse::BitIndexType>> bits = {
{ 0, 4, 9 },
{ 1, 4, 5 },
{ 4, 7, 9 },
{ 3, 6, 9 }
};
int iter = 0;
sf::RenderWindow rw;
rw.create(sf::VideoMode(800, 600), "Test");
sf::Clock clock;
const float frameTime = 1.0f / 60.0f;
const float renderMouseTime = 0.001f;
bool quit = false;
vis::PrettySDR psdr;
psdr.create(16, 16);
while (!quit) {
clock.restart();
sf::Event e;
while (rw.pollEvent(e)) {
switch (e.type) {
case sf::Event::Closed:
quit = true;
break;
}
}
rw.clear();
ph.simStep(bits[iter % bits.size()]);
for (int i = 0; i < 256; i++)
psdr[i] = 0.0f;
for (int i = 0; i < ph.getLayers()[0]._sdr.getBitIndices().size(); i++)
psdr[ph.getLayers()[0]._sdr.getBitIndices()[i]] = 1.0f;
psdr.draw(rw, sf::Vector2f(0.0f, 0.0f));
for (int i = 0; i < ph.getPredBitIndices().size(); i++)
std::cout << ph.getPredBitIndices()[i] << " ";
std::cout << std::endl;
rw.display();
iter++;
}
/*for (int iter = 0; iter < 10000; iter++) {
ph.simStep(bits[iter % bits.size()]);
//for (int i = 0; i < ph.getPredBitIndices().size(); i++)
// std::cout << ph.getPredBitIndices()[i] << " ";
//for (int i = 0; i < ph.getLayers()[2]._pred.getBitIndices().size(); i++)
// std::cout << ph.getLayers()[2]._pred.getBitIndices()[i] << " ";
//std::cout << std::endl;
}
for (int iter = 0; iter < 100; iter++) {
ph.simStep(bits[iter % bits.size()]);
for (int i = 0; i < ph.getPredBitIndices().size(); i++)
std::cout << ph.getPredBitIndices()[i] << " ";
//for (int i = 0; i < ph.getLayers()[2]._pred.getBitIndices().size(); i++)
// std::cout << ph.getLayers()[2]._pred.getBitIndices()[i] << " ";
std::cout << std::endl;
}*/
//system("pause");
return 0;
}
int main() {
std::mt19937 generator(time(nullptr));
sf::RenderWindow renderWindow;
renderWindow.create(sf::VideoMode(1280, 720), "Reinforcement Learning", sf::Style::Default);
renderWindow.setVerticalSyncEnabled(true);
renderWindow.setFramerateLimit(60);
// ---------------------------------- RL Init ------------------------------------
/*sc::HTSLPVLV agentBlue;
sc::HTSLPVLV agentRed;
std::vector<sc::HTSLPVLV::InputType> inputTypes(20);
for (int i = 0; i < 12; i++)
inputTypes[i] = sc::HTSLPVLV::_state;
for (int i = 12; i < 14; i++)
inputTypes[i] = sc::HTSLPVLV::_action;
for (int i = 14; i < 16; i++)
inputTypes[i] = sc::HTSLPVLV::_pv;
for (int i = 16; i < 18; i++)
inputTypes[i] = sc::HTSLPVLV::_lve;
for (int i = 18; i < 20; i++)
inputTypes[i] = sc::HTSLPVLV::_lvi;
std::vector<sc::HTSL::LayerDesc> layerDescs(2);
layerDescs[0]._width = 20;
layerDescs[0]._height = 20;
layerDescs[1]._width = 12;
layerDescs[1]._height = 12;
//layerDescs[2]._width = 8;
//layerDescs[2]._height = 8;
agentBlue.createRandom(5, 4, 8, inputTypes, layerDescs, generator);
agentRed.createRandom(5, 4, 8, inputTypes, layerDescs, generator);*/
float rewardTimer = 0.0f;
float rewardTime = 0.25f;
float lastReward = 0.5f;
//sc::HTSLSARSA agentBlue;
sc::HTSLSARSA agentRed;
std::vector<sc::HTSLSARSA::InputType> inputTypes(16);
for (int i = 0; i < 12; i++)
inputTypes[i] = sc::HTSLSARSA::_state;
for (int i = 12; i < 14; i++)
inputTypes[i] = sc::HTSLSARSA::_action;
for (int i = 14; i < 16; i++)
inputTypes[i] = sc::HTSLSARSA::_q;
std::vector<sc::HTSL::LayerDesc> layerDescs(3);
layerDescs[0]._width = 28;
layerDescs[0]._height = 28;
layerDescs[1]._width = 18;
layerDescs[1]._height = 18;
layerDescs[2]._width = 12;
layerDescs[2]._height = 12;
//agentBlue.createRandom(4, 4, 8, inputTypes, layerDescs, generator);
agentRed.createRandom(4, 4, 8, inputTypes, layerDescs, generator);
//deep::FERL agentBlue;
//agentBlue.createRandom(12, 2, 32, 0.01f, generator);
//deep::FERL agentRed;
//agentRed.createRandom(12, 2, 32, 0.01f, generator);
// --------------------------------- Game Init -----------------------------------
const float slimeRadius = 94.5f;
const float ballRadius = 23.5f;
const float wallRadius = 22.5f;
const float fieldRadius = 640.0f;
const float gravity = 900.0f;
const float slimeBounce = 100.0f;
const float wallBounceDecay = 0.8f;
const float slimeJump = 500.0f;
const float maxSlimeSpeed = 1000.0f;
const float slimeMoveAccel = 5000.0f;
const float slimeMoveDeccel = 8.0f;
std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
sf::Vector2f fieldCenter = sf::Vector2f(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f + 254.0f);
sf::Vector2f wallCenter = fieldCenter + sf::Vector2f(0.0f, -182.0f);
PhyObj blue;
PhyObj red;
PhyObj ball;
blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
blue._velocity = sf::Vector2f(0.0f, 0.0f);
red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
red._velocity = sf::Vector2f(0.0f, 0.0f);
ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
ball._velocity = sf::Vector2f((dist01(generator)) * 600.0f, -(dist01(generator)) * 500.0f);
sf::Texture backgroundTexture;
backgroundTexture.loadFromFile("resources/slimevolleyball/background.png");
sf::Texture blueSlimeTexture;
blueSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyBlue.png");
sf::Texture redSlimeTexture;
redSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyRed.png");
sf::Texture ballTexture;
ballTexture.loadFromFile("resources/slimevolleyball/ball.png");
sf::Texture eyeTexture;
eyeTexture.loadFromFile("resources/slimevolleyball/slimeEye.png");
eyeTexture.setSmooth(true);
sf::Texture arrowTexture;
arrowTexture.loadFromFile("resources/slimevolleyball/arrow.png");
sf::Font scoreFont;
scoreFont.loadFromFile("resources/slimevolleyball/scoreFont.ttf");
int scoreRed = 0;
int scoreBlue = 0;
int prevScoreRed = 0;
int prevScoreBlue = 0;
float prevBallX = fieldCenter.x;
// ------------------------------- Simulation Loop -------------------------------
bool noRender = false;
bool prevPressK = false;
bool quit = false;
float dt = 0.017f;
do {
sf::Event event;
while (renderWindow.pollEvent(event)) {
switch (event.type) {
case sf::Event::Closed:
quit = true;
break;
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
quit = true;
// ---------------------------------- Physics ----------------------------------
bool blueBounced = false;
bool redBounced = false;
// Ball
{
ball._velocity.y += gravity * dt;
ball._position += ball._velocity * dt;
// To floor (game restart)
if (ball._position.y + ballRadius > fieldCenter.y) {
if (ball._position.x < fieldCenter.x)
scoreRed++;
else
scoreBlue++;
blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
blue._velocity = sf::Vector2f(0.0f, 0.0f);
red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
red._velocity = sf::Vector2f(0.0f, 0.0f);
ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
ball._velocity = sf::Vector2f((dist01(generator)) * 600.0f, -(dist01(generator)) * 500.0f);
}
// To wall
if (((ball._position.x + ballRadius) > (wallCenter.x - wallRadius) && ball._position.x < wallCenter.x) || ((ball._position.x - ballRadius) < (wallCenter.x + wallRadius) && ball._position.x > wallCenter.x)) {
// If above rounded part
if (ball._position.y < wallCenter.y) {
sf::Vector2f delta = ball._position - wallCenter;
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
if (dist < wallRadius + ballRadius) {
sf::Vector2f normal = delta / dist;
// Reflect velocity
sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;
ball._velocity = reflectedVelocity * wallBounceDecay;
ball._position = wallCenter + normal * (wallRadius + ballRadius);
}
}
else {
// If on left side
if (ball._position.x < wallCenter.x) {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = wallCenter.x - wallRadius - ballRadius;
}
else {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = wallCenter.x + wallRadius + ballRadius;
}
}
}
// To blue slime
{
sf::Vector2f delta = ball._position - blue._position;
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
if (dist < slimeRadius + ballRadius) {
sf::Vector2f normal = delta / dist;
// Reflect velocity
sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;
float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);
sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;
ball._velocity = blue._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);
ball._position = blue._position + normal * (wallRadius + slimeRadius);
blueBounced = true;
}
}
// To red slime
{
sf::Vector2f delta = ball._position - red._position;
float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);
if (dist < slimeRadius + ballRadius) {
sf::Vector2f normal = delta / dist;
// Reflect velocity
sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;
float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);
sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;
ball._velocity = red._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);
ball._position = red._position + normal * (wallRadius + slimeRadius);
redBounced = true;
}
}
// Out of field, left and right
{
if (ball._position.x - ballRadius < fieldCenter.x - fieldRadius) {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = fieldCenter.x - fieldRadius + ballRadius;
}
else if (ball._position.x + ballRadius > fieldCenter.x + fieldRadius) {
ball._velocity.x = wallBounceDecay * -ball._velocity.x;
ball._position.x = fieldCenter.x + fieldRadius - ballRadius;
}
}
}
// Blue slime
{
blue._velocity.y += gravity * dt;
blue._velocity.x += -slimeMoveDeccel * blue._velocity.x * dt;
blue._position += blue._velocity * dt;
if (sf::Keyboard::isKeyPressed(sf::Keyboard::A)) {
blue._velocity.x += -slimeMoveAccel * dt;
if (blue._velocity.x < -maxSlimeSpeed)
blue._velocity.x = -maxSlimeSpeed;
}
else if (sf::Keyboard::isKeyPressed(sf::Keyboard::D)) {
blue._velocity.x += slimeMoveAccel * dt;
if (blue._velocity.x > maxSlimeSpeed)
blue._velocity.x = maxSlimeSpeed;
}
if (blue._position.y > fieldCenter.y) {
blue._velocity.y = 0.0f;
blue._position.y = fieldCenter.y;
if (sf::Keyboard::isKeyPressed(sf::Keyboard::W))
blue._velocity.y -= slimeJump;
}
if (blue._position.x - slimeRadius < fieldCenter.x - fieldRadius) {
blue._velocity.x = 0.0f;
blue._position.x = fieldCenter.x - fieldRadius + slimeRadius;
}
if (blue._position.x + slimeRadius > wallCenter.x - wallRadius) {
blue._velocity.x = 0.0f;
blue._position.x = wallCenter.x - wallRadius - slimeRadius;
}
}
// Blue slime
/*{
const float scalar = 0.001f;
// Percepts
std::vector<float> inputs(12);
int index = 0;
inputs[index++] = (ball._position.x - blue._position.x) * scalar;
inputs[index++] = (ball._position.y - blue._position.y) * scalar;
inputs[index++] = ball._velocity.x * scalar;
inputs[index++] = ball._velocity.y * scalar;
inputs[index++] = (red._position.x - blue._position.x) * scalar;
inputs[index++] = (red._position.y - blue._position.y) * scalar;
inputs[index++] = red._velocity.x * scalar;
inputs[index++] = red._velocity.y * scalar;
inputs[index++] = blue._position.x * scalar;
inputs[index++] = blue._position.y * scalar;
inputs[index++] = blue._velocity.x * scalar;
inputs[index++] = blue._velocity.y * scalar;
std::vector<float> outputs(2);
// Actions
for (int i = 0; i < 12; i++)
agentBlue.setState(i, inputs[i]);
float reward = (ball._position.x > fieldCenter.x && prevBallX < fieldCenter.x ? 1.0f : 0.5f) * 0.5f + (scoreRed > prevScoreRed ? 0.0f : 0.5f) * 0.5f;
if (reward != 0.5f) {
lastReward = reward = reward > 0.5f ? 1.0f : 0.0f;
rewardTimer = 0.0f;
}
else if (rewardTimer < rewardTime) {
reward = lastReward;
rewardTimer += dt;
}
if (blueBounced)
reward = std::max(reward, 0.55f);
reward = (reward * 2.0f - 1.0f) * 0.01f - std::abs(ball._position.x - blue._position.x) * 0.001f;
//reward = (scoreBlue - prevScoreBlue) - (scoreRed - prevScoreRed) - std::abs(ball._position.x - blue._position.x) * 0.001f;
agentBlue.update(reward, generator);
float move = agentBlue.getActionFromNodeIndex(0) * 2.0f - 1.0f;
bool jump = agentBlue.getActionFromNodeIndex(1) > 0.5f;
//std::vector<float> action(2);
//agentBlue.step(inputs, action, reward, 0.01f, 0.995f, 0.99f, 10.0f, 64, 3, 0.02f, 0.04f, 0.04f, 800, 200, 0.003f, 0.0f, generator);
//float move = action[0];
//bool jump = action[1] > 0.0f;
blue._velocity.y += gravity * dt;
blue._velocity.x += -slimeMoveDeccel * blue._velocity.x * dt;
blue._position += blue._velocity * dt;
{
blue._velocity.x += move * slimeMoveAccel * dt;
if (blue._velocity.x < -maxSlimeSpeed)
blue._velocity.x = -maxSlimeSpeed;
else if (blue._velocity.x > maxSlimeSpeed)
blue._velocity.x = maxSlimeSpeed;
}
if (blue._position.y > fieldCenter.y) {
blue._velocity.y = 0.0f;
blue._position.y = fieldCenter.y;
if (jump)
blue._velocity.y -= slimeJump;
}
if (blue._position.x - slimeRadius < fieldCenter.x - fieldRadius) {
blue._velocity.x = 0.0f;
blue._position.x = fieldCenter.x - fieldRadius + slimeRadius;
}
if (blue._position.x + slimeRadius > wallCenter.x - wallRadius) {
blue._velocity.x = 0.0f;
blue._position.x = wallCenter.x - wallRadius - slimeRadius;
}
}*/
// Red slime
{
const float scalar = 0.001f;
// Percepts
std::vector<float> inputs(12);
int index = 0;
inputs[index++] = (ball._position.x - red._position.x) * scalar;
inputs[index++] = (ball._position.y - red._position.y) * scalar;
inputs[index++] = ball._velocity.x * scalar;
inputs[index++] = ball._velocity.y * scalar;
inputs[index++] = (blue._position.x - red._position.x) * scalar;
inputs[index++] = (blue._position.y - red._position.y) * scalar;
inputs[index++] = red._velocity.x * scalar;
inputs[index++] = red._velocity.y * scalar;
inputs[index++] = blue._position.x * scalar;
inputs[index++] = blue._position.y * scalar;
inputs[index++] = blue._velocity.x * scalar;
inputs[index++] = blue._velocity.y * scalar;
//std::vector<float> outputs(2);
// Actions
for (int i = 0; i < 12; i++)
agentRed.setState(i, inputs[i]);
float reward = (ball._position.x < fieldCenter.x && prevBallX >= fieldCenter.x ? 1.0f : 0.5f);
if (reward != 0.5f) {
lastReward = reward = reward > 0.5f ? 1.0f : 0.0f;
rewardTimer = 0.0f;
}
else if (rewardTimer < rewardTime) {
reward = lastReward;
rewardTimer += dt;
}
if (redBounced)
reward = std::max(reward, 0.55f);
reward = (reward * 2.0f - 1.0f) * 0.1f - std::abs(ball._position.x - red._position.x) * 0.008f;
//reward *= 0.05f;
//reward = (scoreRed - prevScoreRed) - (scoreBlue - prevScoreBlue) - std::abs(ball._position.x - red._position.x) * 0.001f;
//std::cout << "Reward: " << reward << std::endl;
//agentRed.update(reward, generator);
agentRed.update(reward, generator);
//std::vector<float> action(2);
//agentRed.step(inputs, action, reward, 0.01f, 0.995f, 0.99f, 10.0f, 32, 6, 0.05f, 0.04f, 0.04f, 800, 200, 0.003f, 0.0f, generator);
//float move = action[0];
//bool jump = action[1] > 0.0f;
float move = agentRed.getActionFromNodeIndex(0) * 2.0f - 1.0f;
bool jump = agentRed.getActionFromNodeIndex(1) > 0.5f;
red._velocity.y += gravity * dt;
red._velocity.x += -slimeMoveDeccel * red._velocity.x * dt;
red._position += red._velocity * dt;
{
red._velocity.x += move * slimeMoveAccel * dt;
if (red._velocity.x < -maxSlimeSpeed)
red._velocity.x = -maxSlimeSpeed;
else if (red._velocity.x > maxSlimeSpeed)
red._velocity.x = maxSlimeSpeed;
}
if (red._position.y > fieldCenter.y) {
red._velocity.y = 0.0f;
red._position.y = fieldCenter.y;
if (jump)
red._velocity.y -= slimeJump;
}
if (red._position.x + slimeRadius > fieldCenter.x + fieldRadius) {
red._velocity.x = 0.0f;
red._position.x = fieldCenter.x + fieldRadius - slimeRadius;
}
if (red._position.x - slimeRadius < wallCenter.x + wallRadius) {
red._velocity.x = 0.0f;
red._position.x = wallCenter.x + wallRadius + slimeRadius;
}
}
prevScoreRed = scoreRed;
prevScoreBlue = scoreBlue;
prevBallX = ball._position.x;
if (sf::Keyboard::isKeyPressed(sf::Keyboard::K) && !prevPressK) {
noRender = !noRender;
}
prevPressK = sf::Keyboard::isKeyPressed(sf::Keyboard::K);
if (noRender)
continue;
// --------------------------------- Rendering ---------------------------------
renderWindow.clear();
{
sf::Sprite s;
s.setTexture(backgroundTexture);
s.setOrigin(backgroundTexture.getSize().x * 0.5f, backgroundTexture.getSize().y * 0.5f);
s.setPosition(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(blueSlimeTexture);
s.setOrigin(blueSlimeTexture.getSize().x * 0.5f, blueSlimeTexture.getSize().y);
s.setPosition(blue._position);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(eyeTexture);
s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
s.setPosition(blue._position + sf::Vector2f(50.0f, -28.0f));
sf::Vector2f delta = ball._position - s.getPosition();
float angle = std::atan2(delta.y, delta.x);
s.setRotation(angle * 180.0f / 3.141596f);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(redSlimeTexture);
s.setOrigin(redSlimeTexture.getSize().x * 0.5f, redSlimeTexture.getSize().y);
s.setPosition(red._position);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(eyeTexture);
s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
s.setPosition(red._position + sf::Vector2f(-50.0f, -28.0f));
sf::Vector2f delta = ball._position - s.getPosition();
float angle = std::atan2(delta.y, delta.x);
s.setRotation(angle * 180.0f / 3.141596f);
renderWindow.draw(s);
}
{
sf::Sprite s;
s.setTexture(ballTexture);
s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
s.setPosition(ball._position);
renderWindow.draw(s);
}
if (ball._position.y + ballRadius < 0.0f) {
sf::Sprite s;
s.setTexture(arrowTexture);
s.setOrigin(arrowTexture.getSize().x * 0.5f, 0.0f);
s.setPosition(ball._position.x, 0.0f);
renderWindow.draw(s);
}
{
sf::Text scoreText;
scoreText.setFont(scoreFont);
scoreText.setString(std::to_string(scoreBlue));
scoreText.setCharacterSize(100);
float width = scoreText.getLocalBounds().width;
scoreText.setPosition(fieldCenter.x - width * 0.5f - 100.0f, 10.0f);
scoreText.setColor(sf::Color(100, 133, 255));
renderWindow.draw(scoreText);
}
{
sf::Text scoreText;
scoreText.setFont(scoreFont);
scoreText.setString(std::to_string(scoreRed));
scoreText.setCharacterSize(100);
float width = scoreText.getLocalBounds().width;
scoreText.setPosition(fieldCenter.x - width * 0.5f + 100.0f, 10.0f);
scoreText.setColor(sf::Color(255, 100, 100));
renderWindow.draw(scoreText);
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::P)) {
float scale = 4.0f;
float alignment = 0.0f;
/*for (int l = 0; l < agentBlue.getHTSL().getLayers().size(); l++) {
sf::Image sdr;
sdr.create(agentBlue.getHTSL().getLayerDescs()[l]._width, agentBlue.getHTSL().getLayerDescs()[l]._height);
for (int x = 0; x < agentBlue.getHTSL().getLayerDescs()[l]._width; x++)
for (int y = 0; y < agentBlue.getHTSL().getLayerDescs()[l]._height; y++) {
sf::Color c;
c.r = c.g = c.b = agentBlue.getHTSL().getLayers()[l]._rsc.getHiddenState(x, y) * 255.0f;
sdr.setPixel(x, y, c);
}
sf::Texture sdrt;
sdrt.loadFromImage(sdr);
sf::Sprite sdrs;
sdrs.setTexture(sdrt);
sdrs.setPosition(alignment, renderWindow.getSize().y - sdr.getSize().y * scale);
sdrs.setScale(scale, scale);
renderWindow.draw(sdrs);
alignment += scale * sdr.getSize().x;
}*/
alignment = 0.0f;
for (int l = 0; l < agentRed.getHTSL().getLayers().size(); l++) {
sf::Image sdr;
sdr.create(agentRed.getHTSL().getLayerDescs()[l]._width, agentRed.getHTSL().getLayerDescs()[l]._height);
for (int x = 0; x < agentRed.getHTSL().getLayerDescs()[l]._width; x++)
for (int y = 0; y < agentRed.getHTSL().getLayerDescs()[l]._height; y++) {
sf::Color c;
c.r = c.g = c.b = agentRed.getHTSL().getLayers()[l]._rsc.getHiddenState(x, y) * 255.0f;
sdr.setPixel(x, y, c);
}
sf::Texture sdrt;
sdrt.loadFromImage(sdr);
alignment += scale * sdr.getSize().x;
sf::Sprite sdrs;
sdrs.setTexture(sdrt);
sdrs.setPosition(renderWindow.getSize().x - alignment, renderWindow.getSize().y - sdr.getSize().y * scale);
sdrs.setScale(scale, scale);
renderWindow.draw(sdrs);
}
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
sf::Vector2f position;
const float scalar = 1.0f / 0.001f;
position.x = red._position.x + scalar * agentRed.getHTSL().getPrediction(0);
position.y = red._position.y + scalar * agentRed.getHTSL().getPrediction(1);
sf::Sprite s;
s.setTexture(ballTexture);
s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
s.setPosition(position);
renderWindow.draw(s);
}
if (sf::Keyboard::isKeyPressed(sf::Keyboard::F)) {
for (int i = 0; i < 16; i++)
std::cout << agentRed.getHTSL().getPrediction(i) << std::endl;
std::cout << std::endl;
}
renderWindow.display();
} while (!quit);
return 0;
}
