Ejemplo n.º 1
0
string Network::initState()
{
    //http://stackoverflow.com/questions/13445688/how-to-generate-a-random-number-in-c
    std::mt19937 rng;
    rng.seed(std::random_device()());
    std::uniform_int_distribution<std::mt19937::result_type> dist01(0,1);

    for( map<string,Node*>::iterator it = nodeList.begin(); it != nodeList.end(); ++it )
    {
        it->second->setVal(dist01(rng));
    }

    return(stringState(0));
}
Ejemplo n.º 2
0
Archivo: ga.cpp Proyecto: srgkoval/GA
template <int N, int N_obj> void GA<N, N_obj>::selection_tournament(int n)
{
    for(int i = 0; i < n; i++)
    {
        int best = floor(options.population_size * dist01(rnd_generator));

        for(int j = 1; j < options.tournament_size; j++)
        {
            int candidate = floor(options.population_size * dist01(rnd_generator));
            
            if(fitness[candidate] > fitness[best])
                best = candidate;
        }
        parents[i] = best;
    }
}
Ejemplo n.º 3
0
Archivo: ga.cpp Proyecto: srgkoval/GA
template <int N, int N_obj> void GA<N, N_obj>::selection_stochastic_uniform(int n)
{
    double *wheel = new double[options.population_size];
    
    wheel[0] = fitness[0];
    for(int i = 1; i < options.population_size; ++i)
        wheel[i] = wheel[i - 1] +  fitness[i];

    double step = 1. / (double)n,
           position = dist01(rnd_generator) * step;
    int j_low = 0;
    
    for(int i = 0; i < n; ++i)
    {
        for(int j = j_low; j < options.population_size; ++j)
        {
            if(position < wheel[j])
            {
                parents[i] = j;
                j_low = j;
                break;
            }
        }
        position += step;
    }
}
Ejemplo n.º 4
0
Archivo: ga.cpp Proyecto: srgkoval/GA
template <int N, int N_obj> void GA<N, N_obj>::selection_tournament_multiobjective(int n)
{
    for(int i = 0; i < n; i++)
    {
        int best = floor(options.population_size * dist01(rnd_generator));
        best = archive[best];

        for(int j = 1; j < options.tournament_size_multiobjective; j++)
        {
            int candidate = floor(options.population_size * dist01(rnd_generator));
            candidate = archive[candidate];
            
            if(rank[candidate] < rank[best])
                best = candidate;
            else
            {
                if(rank[candidate] == rank[best] && distance[candidate] > distance[best])
                    best = candidate;
            }
        }
        parents[i] = best;
    }
}
Ejemplo n.º 5
0
void sample(neo::PredictiveHierarchy& ph, std::mt19937& generator,
           char seed, int nSamples, VectorCodec& textcodec, float seedNoise = 0.5, float predNoise = 0.05) {
    
	std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
	
	textcodec.symbol = seed;
	textcodec.encode();
	
	for (int j = 0; j < textcodec.N; j++) {
        ph.setInput(j, textcodec.vector[j] + dist01(generator)*seedNoise);
    }
    ph.simStep(generator, false);
    
    std::cout << "seed: " << seed << " i: " << textcodec.symIndex << " sample: \"";
    
    for (size_t i = 1; i < nSamples; i++) {

		for (int j = 0; j < textcodec.N; j++) {
			textcodec.vector[j] = ph.getPrediction(j);
		}
        
		textcodec.decode();
			
		char predChar = textcodec.symbol;
        
		std::cout << predChar;
		
        for (int j = 0; j < textcodec.N; j++) {
            ph.setInput(j, ph.getPrediction(j) + dist01(generator)*predNoise);
        }
        
        ph.simStep(generator, false);
    }
    
    std::cout << "\"" << std::endl;
}
Ejemplo n.º 6
0
Archivo: ga.cpp Proyecto: srgkoval/GA
template <int N, int N_obj> void GA<N, N_obj>::crossover_scattered
	(const Individual &parent1, const Individual &parent2, Individual &child)
{
    for(int i = 0; i < N; ++i)
    {
        if(dist01(rnd_generator) >= 0.5)
        {
            child[i] = parent1[i];
        }
        else
        {
            child[i] = parent2[i];
        }
    }
    
    if( !feasible(child) )
        crossover_arithmetic(parent1, parent2, child);
}
Ejemplo n.º 7
0
Archivo: ga.cpp Proyecto: srgkoval/GA
template <int N, int N_obj> void GA<N, N_obj>::crossover_BLX
	(const Individual &parent1, const Individual &parent2, Individual &child)
{
    for(int i = 0; i < N; i++)
    {
        double R = dist01(rnd_generator);

        double I = parent2[i] - parent1[i],
               low = parent1[i] - I * options.crossover_BLX_alpha,
               high = parent2[i] + I * options.crossover_BLX_alpha;

        child[i] = R * low + (1. - R) * high;

        // modify child to satisfy constraints
        //child[i] = std::max(child[i], lower_boundary[i]);
        //child[i] = std::min(child[i], upper_boundary[i]);
        if(child[i] < lower_boundary[i] || child[i] > upper_boundary[i])
            child[i] = (R * parent1[i] + (1. - R) * parent2[i]);
    }
}
Ejemplo n.º 8
0
int main() {
	std::mt19937 generator(time(nullptr));

	dodgeballPositions.resize(1);
	dodgeballVelocities.resize(dodgeballPositions.size());

	std::uniform_real_distribution<float> dist01(0.0f, 1.0f);

	for (int i = 0; i < dodgeballPositions.size(); i++) {
		dodgeballPositions[i] = sf::Vector2f(dist01(generator), dist01(generator));

		dodgeballVelocities[i].x = dist01(generator) * 2.0f - 1.0f;
		dodgeballVelocities[i].y = dist01(generator) * 2.0f - 1.0f;

		dodgeballVelocities[i] = dodgeballVelocities[i] * (ballSpeed / std::sqrt(dodgeballVelocities[i].x * dodgeballVelocities[i].x + dodgeballVelocities[i].y * dodgeballVelocities[i].y));
	}

	agentPosition = sf::Vector2f(0.5f, 0.5f);

	sf::RenderWindow window;

	window.create(sf::VideoMode(800, 800), "BIDInet", sf::Style::Default);

	window.setFramerateLimit(60);
	window.setVerticalSyncEnabled(true);

	sf::RenderTexture visionRT;

	visionRT.create(16, 16);

	deep::CSRL swarm;

	std::vector<deep::CSRL::LayerDesc> layerDescs(4);

	layerDescs[0]._width = 16;
	layerDescs[0]._height = 16;

	layerDescs[1]._width = 12;
	layerDescs[1]._height = 12;

	layerDescs[2]._width = 8;
	layerDescs[2]._height = 8;

	layerDescs[3]._width = 4;
	layerDescs[3]._height = 4;

	swarm.createRandom(2, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.1f, generator);

	// ---------------------------- Game Loop -----------------------------

	bool quit = false;

	sf::Clock clock;

	float dt = 0.017f;

	float averageReward = 0.0f;
	const float averageRewardDecay = 0.003f;

	int steps = 0;

	do {
		clock.restart();

		// ----------------------------- Input -----------------------------

		sf::Event windowEvent;

		while (window.pollEvent(windowEvent))
		{
			switch (windowEvent.type)
			{
			case sf::Event::Closed:
				quit = true;
				break;
			}
		}

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
			quit = true;

		visionRT.clear();

		renderScene(visionRT);

		visionRT.display();

		sf::Image img = visionRT.getTexture().copyToImage();

		for (int x = 0; x < img.getSize().x; x++)
			for (int y = 0; y < img.getSize().y; y++) {
				sf::Color c = img.getPixel(x, y);

				float valR = 0.0f;
				float valG = 0.0f;

				if (c.r > 0)
				valR = 1.0f;

				if (c.g > 0)
				valG = 1.0f;

				swarm.setState(x, y, 0, valR);
				swarm.setState(x, y, 1, valG);
			}

		float reward = 0.5f;

		for (int i = 0; i < dodgeballPositions.size(); i++) {
			sf::Vector2f delta = dodgeballPositions[i] - agentPosition;

			float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

			//if (dist < (ballRadius + agentRadius)) {
			reward += -dist;
			//}
		}

		reward *= 10.0f;

		averageReward = (1.0f - averageRewardDecay) * averageReward + averageRewardDecay * reward;

		swarm.simStep(1, reward, generator);

		//agent.simStep(reward, 0.1f, 0.99f, 0.01f, 0.2f, 0.01f, 0.01f, 0.01f, 64, 0.05f, 0.98f, 0.04f, 0.01f, 0.01f, 4.0f, generator);

		agentPosition.x += agentSpeed * (swarm.getAction(3, 4, 0) * 2.0f - 1.0f);
		agentPosition.y += agentSpeed * (swarm.getAction(3, 8, 0) * 2.0f - 1.0f);
		//agentPosition.x += agentSpeed * (agent.getAction(0) * 2.0f - 1.0f);
		//agentPosition.y += agentSpeed * (agent.getAction(1) * 2.0f - 1.0f);

		agentPosition.x = std::min(0.5f + agentFieldRadius - agentRadius, std::max(0.5f - agentFieldRadius + agentRadius, agentPosition.x));
		agentPosition.y = std::min(0.5f + agentFieldRadius - agentRadius, std::max(0.5f - agentFieldRadius + agentRadius, agentPosition.y));

		for (int i = 0; i < dodgeballPositions.size(); i++) {
			sf::Vector2f delta = agentPosition - dodgeballPositions[i];

			float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

			dodgeballVelocities[i] += -dodgeballVelocities[i] * ballVelocityDecay + delta / dist * attraction;

			dodgeballPositions[i] += dodgeballVelocities[i];

			if (dodgeballPositions[i].x < 0.5f - agentFieldRadius + ballRadius) {
				dodgeballPositions[i].x = 0.5f - agentFieldRadius + ballRadius;
				dodgeballVelocities[i].x *= -1.0f;
			}
			else if (dodgeballPositions[i].x > 0.5f + agentFieldRadius - ballRadius) {
				dodgeballPositions[i].x = 0.5f + agentFieldRadius - ballRadius;
				dodgeballVelocities[i].x *= -1.0f;
			}

			if (dodgeballPositions[i].y < 0.5f - agentFieldRadius + ballRadius) {
				dodgeballPositions[i].y = 0.5f - agentFieldRadius + ballRadius;
				dodgeballVelocities[i].y *= -1.0f;
			}
			else if (dodgeballPositions[i].y > 0.5f + agentFieldRadius - ballRadius) {
				dodgeballPositions[i].y = 0.5f + agentFieldRadius - ballRadius;
				dodgeballVelocities[i].y *= -1.0f;
			}
		}

		if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
			window.clear();

			renderScene(window);

			sf::Sprite vis;

			vis.setTexture(visionRT.getTexture());

			vis.setScale(4.0f, 4.0f);

			window.draw(vis);

			window.display();
		}

		if (steps % 100 == 0)
			std::cout << "Steps: " << steps << " Average Reward: " << averageReward << std::endl;

		//dt = clock.getElapsedTime().asSeconds();

		steps++;

	} while (!quit);

	return 0;
}
Ejemplo n.º 9
0
int main() {
	std::mt19937 generator(time(nullptr));

	_ballPosition = sf::Vector2f(0.5f, 0.5f);
	_ballVelocity = sf::Vector2f(0.44f, 0.55f);

	_ballVelocity *= ballSpeed / std::sqrt(_ballVelocity.x * _ballVelocity.x + _ballVelocity.y * _ballVelocity.y);

	_paddlePosition = 0.5f;

	std::uniform_real_distribution<float> dist01(0.0f, 1.0f);

	sf::RenderWindow window;

	window.create(sf::VideoMode(800, 800), "BIDInet", sf::Style::Default);

	window.setFramerateLimit(60);
	window.setVerticalSyncEnabled(true);

	sf::RenderTexture visionRT;

	visionRT.create(16, 16);

	deep::CSRL agent;

	std::vector<deep::CSRL::LayerDesc> layerDescs(3);

	layerDescs[0]._width = 6;
	layerDescs[0]._height = 6;

	layerDescs[1]._width = 5;
	layerDescs[1]._height = 5;

	layerDescs[2]._width = 4;
	layerDescs[2]._height = 4;

	int inWidth = 16;
	int inHeight = 18;

	std::vector<deep::CSRL::InputType> inputTypes(inWidth * inHeight, deep::CSRL::_state);

	for (int i = 0; i < inWidth; i++) {
		inputTypes[i + (inHeight - 2) * inWidth] = deep::CSRL::_action;
		inputTypes[i + (inHeight - 1) * inWidth] = deep::CSRL::_q;
	}

	agent.createRandom(inWidth, inHeight, 8, inputTypes, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.2f, generator);

	// ---------------------------- Game Loop -----------------------------

	bool quit = false;

	sf::Clock clock;

	float dt = 0.017f;

	float averageReward = 0.0f;
	const float averageRewardDecay = 0.003f;

	int steps = 0;

	do {
		clock.restart();

		// ----------------------------- Input -----------------------------

		sf::Event windowEvent;

		while (window.pollEvent(windowEvent))
		{
			switch (windowEvent.type)
			{
			case sf::Event::Closed:
				quit = true;
				break;
			}
		}

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
			quit = true;

		visionRT.clear();

		renderScene(visionRT);

		visionRT.display();

		sf::Image img = visionRT.getTexture().copyToImage();

		for (int x = 0; x < img.getSize().x; x++)
			for (int y = 0; y < img.getSize().y; y++) {
				sf::Color c = img.getPixel(x, y);

				/*float valR = 0.0f;
				float valG = 0.0f;

				if (c.r > 0)
				valR = 1.0f;

				if (c.g > 0)
				valG = 1.0f;

				swarm.setState(x, y, 0, valR);
				swarm.setState(x, y, 1, valG);*/

				float val = 0.0f;

				if (c.r > 0)
					val = 0.5f;

				if (c.g > 0)
					val = 1.0f;

				agent.setInput(x, y, val);
			}

		float reward = 0.0f;

		if (_ballPosition.x < 0.0f) {
			_ballPosition.x = 0.0f;

			_ballVelocity.x *= -1.0f;
		}

		if (_ballPosition.y < 0.0f) {
			_ballPosition.y = 0.0f;

			_ballVelocity.y *= -1.0f;
		}

		if (_ballPosition.x > 1.0f) {
			_ballPosition.x = 1.0f;

			_ballVelocity.x *= -1.0f;
		}

		if (_ballPosition.y > 1.0f - bottomRatio) {
			_ballPosition.y = 1.0f - bottomRatio;

			if (_ballPosition.x > _paddlePosition - paddleWidthRatio && _ballPosition.x < _paddlePosition + paddleWidthRatio) {
				reward += 10.0f;
			}
			else
				reward -= 5.0f;

			_ballVelocity.y *= -1.0f;
		}

		_ballPosition += _ballVelocity;

		averageReward = (1.0f - averageRewardDecay) * averageReward + averageRewardDecay * reward;

		agent.simStep(reward, generator);

		float act = 0.0f;

		for (int i = 4; i < 5; i++) {
			act += agent.getPrediction(i, 16);
		}

		_paddlePosition = std::min(1.0f, std::max(0.0f, _paddlePosition + 0.1f * std::min(1.0f, std::max(-1.0f, act))));

		//std::cout << averageReward << std::endl;

		if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
			window.clear();

			renderScene(window);

			sf::Sprite vis;

			vis.setTexture(visionRT.getTexture());

			vis.setScale(4.0f, 4.0f);

			window.draw(vis);

			sf::Image predImg;

			predImg.create(16, 17);

			for (int x = 0; x < 16; x++)
				for (int y = 0; y < 17; y++) {
					sf::Color c = sf::Color::White;

					c.r = c.g = c.b = 255.0f * std::min(1.0f, std::max(0.0f, agent.getPrediction(x, y)));

					predImg.setPixel(x, y, c);
				}

			sf::Texture t;
			t.loadFromImage(predImg);

			sf::Sprite s;

			s.setTexture(t);

			s.setScale(4.0f, 4.0f);

			s.setPosition(4.0f * 16.0f, 0.0f);

			window.draw(s);

			window.display();
		}

		if (steps % 100 == 0)
			std::cout << "Steps: " << steps << " Average Reward: " << averageReward << std::endl;

		//dt = clock.getElapsedTime().asSeconds();

		steps++;

	} while (!quit);

	return 0;
}
Ejemplo n.º 10
0
void Level::create(Game* pGame, int width, int height, int drunkSteps, int maxStepDist,
	const std::vector<std::shared_ptr<sf::Texture>> &backgroundTextures,
	const std::vector<std::shared_ptr<sf::Texture>> &doorTextures,
	const std::vector<std::shared_ptr<sf::Texture>> &roofTextures,
	std::mt19937 &generator)
{
	_width = width;
	_height = height;

	_pGame = pGame;

	_backgroundTextures = backgroundTextures;
	_doorTextures = doorTextures;
	_roofTextures = roofTextures;

	_cells.resize(_width * _height);

	int drunkX = _width / 2;
	int drunkY = _height / 2;

	_currentCellX = drunkX;
	_currentCellY = drunkY;

	std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
	std::uniform_int_distribution<int> backgroundDist(0, backgroundTextures.size() - 1);

	// First room
	{
		int cIndex = drunkX + drunkY * _width;

		Cell &c = _cells[cIndex];

		if (c._room == nullptr) {
			c._room = std::make_unique<Room>();

			c._room->create(this, drunkX, drunkY, backgroundTextures.front()->getSize().x, backgroundTextures.front()->getSize().y, std::vector<int>(1, cIndex), backgroundDist(generator));
		}
	}

	for (int i = 0; i < drunkSteps; i++) {
		int dir = static_cast<int>(dist01(generator) * 4.0f) % 4;

		for (int l = 0; l < maxStepDist; l++) {
			int prevDrunkX = drunkX;
			int prevDrunkY = drunkY;

			switch (dir) {
			case 0:
				drunkX++;

				break;

			case 1:
				drunkY++;

				break;

			case 2:
				drunkX--;

				break;

			case 3:
				drunkY--;

				break;
			}
			
			if (drunkX < 0 || drunkY < 0 || drunkX >= _width || drunkY >= _height) {
				prevDrunkX = drunkX;
				prevDrunkY = drunkY;

				break;
			}

			int cIndex = drunkX + drunkY * _width;

			Cell &c = _cells[cIndex];

			if (c._room == nullptr) {
				c._room = std::make_unique<Room>();

				c._room->create(this, drunkX, drunkY, backgroundTextures.front()->getSize().x, backgroundTextures.front()->getSize().y, std::vector<int>(1, cIndex), backgroundDist(generator));
			}
		}
	}
}
Ejemplo n.º 11
0
int main() {
	std::mt19937 generator(time(nullptr));

	sys::ComputeSystem cs;

	cs.create(sys::ComputeSystem::_gpu);

	sys::ComputeProgram prog;

	prog.loadFromFile("resources/neoKernels.cl", cs);

	// --------------------------- Create the Sparse Coder ---------------------------

	cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 64, 64);

	std::ifstream fromFile("resources/train-images.idx3-ubyte", std::ios::binary | std::ios::in);

	if (!fromFile.is_open()) {
		std::cerr << "Could not open train-images.idx3-ubyte!" << std::endl;

		return 1;
	}

	std::vector<neo::PredictiveHierarchy::LayerDesc> layerDescs(4);

	layerDescs[0]._size = { 64, 64 };
	layerDescs[0]._feedForwardRadius = 8;

	layerDescs[1]._size = { 48, 48 };

	layerDescs[2]._size = { 32, 32 };

	layerDescs[3]._size = { 24, 24 };

	neo::PredictiveHierarchy ph;

	ph.createRandom(cs, prog, { 64, 64 }, layerDescs, { -0.01f, 0.01f }, { 0.01f, 0.05f }, 0.1f, generator);

	float avgError = 1.0f;

	float avgErrorDecay = 0.1f;

	sf::RenderWindow window;

	window.create(sf::VideoMode(1024, 512), "MNIST Video Test");

	vis::Plot plot;

	plot._curves.push_back(vis::Curve());

	plot._curves[0]._name = "Squared Error";

	std::uniform_int_distribution<int> digitDist(0, 59999);
	std::uniform_real<float> dist01(0.0f, 1.0f);

	sf::RenderTexture rt;

	rt.create(64, 64);

	sf::Image digit0;
	sf::Texture digit0Tex;
	sf::Image digit1;
	sf::Texture digit1Tex;

	sf::Image pred;
	sf::Texture predTex;

	digit0.create(28, 28);
	digit1.create(28, 28);

	pred.create(rt.getSize().x, rt.getSize().y);

	const float boundingSize = (64 - 28) / 2;
	const float center = 32;
	const float minimum = center - boundingSize;
	const float maximum = center + boundingSize;

	float avgError2 = 1.0f;

	const float avgError2Decay = 0.01f;

	std::vector<float> prediction(64 * 64, 0.0f);

	for (int iter = 0; iter < 10000; iter++) {
		// Select digit indices
		int d0 = digitDist(generator);
		int d1 = digitDist(generator);

		// Load digits
		Image img0, img1;

		loadMNISTimage(fromFile, d0, img0);
		loadMNISTimage(fromFile, d1, img1);

		for (int x = 0; x < digit0.getSize().x; x++)
			for (int y = 0; y < digit0.getSize().y; y++) {
				int index = x + y * digit0.getSize().x;

				sf::Color c = sf::Color::White;

				c.a = img0._intensities[index];

				digit0.setPixel(x, y, c);
			}

		digit0Tex.loadFromImage(digit0);

		for (int x = 0; x < digit1.getSize().x; x++)
			for (int y = 0; y < digit1.getSize().y; y++) {
				int index = x + y * digit1.getSize().x;

				sf::Color c = sf::Color::White;

				c.a = img1._intensities[index];

				digit1.setPixel(x, y, c);
			}

		digit1Tex.loadFromImage(digit1);

		sf::Vector2f vel0(dist01(generator) * 2.0f - 1.0f, dist01(generator) * 2.0f - 1.0f);
		sf::Vector2f vel1(dist01(generator) * 2.0f - 1.0f, dist01(generator) * 2.0f - 1.0f);

		sf::Vector2f pos0(dist01(generator) * (maximum - minimum) + minimum, dist01(generator) * (maximum - minimum) + minimum);
		sf::Vector2f pos1(dist01(generator) * (maximum - minimum) + minimum, dist01(generator) * (maximum - minimum) + minimum);

		float vel0mul = dist01(generator) * 6.0f / std::max(1.0f, std::sqrt(vel0.x * vel0.x + vel0.y + vel0.y));

		vel0 *= vel0mul;

		float vel1mul = dist01(generator) * 6.0f / std::max(1.0f, std::sqrt(vel1.x * vel1.x + vel1.y + vel1.y));

		vel1 *= vel1mul;

		// Render video
		for (int f = 0; f < 20; f++) {
			sf::Event windowEvent;

			while (window.pollEvent(windowEvent)) {
				switch (windowEvent.type) {
				case sf::Event::Closed:
					return 0;
				}
			}

			pos0 += vel0;

			pos1 += vel1;

			if (pos0.x < minimum) {
				pos0.x = minimum;

				vel0.x *= -1.0f;
			}
			else if (pos0.x > maximum) {
				pos0.x = maximum;

				vel0.x *= -1.0f;
			}

			if (pos0.y < minimum) {
				pos0.y = minimum;

				vel0.y *= -1.0f;
			}
			else if (pos0.y > maximum) {
				pos0.y = maximum;

				vel0.y *= -1.0f;
			}

			if (pos1.x < minimum) {
				pos1.x = minimum;

				vel1.x *= -1.0f;
			}
			else if (pos1.x > maximum) {
				pos1.x = maximum;

				vel1.x *= -1.0f;
			}

			if (pos1.y < minimum) {
				pos1.y = minimum;

				vel1.y *= -1.0f;
			}
			else if (pos1.y > maximum) {
				pos1.y = maximum;

				vel1.y *= -1.0f;
			}

			window.clear();
			rt.clear(sf::Color::Black);

			sf::Sprite s0;

			s0.setTexture(digit0Tex);

			s0.setOrigin(28 / 2, 28 / 2);

			s0.setPosition(pos0);

			rt.draw(s0);

			sf::Sprite s1;

			s1.setTexture(digit1Tex);

			s1.setOrigin(28 / 2, 28 / 2);

			s1.setPosition(pos1);

			rt.draw(s1);

			rt.display();

			// Get input image
			sf::Image res = rt.getTexture().copyToImage();

			// Show RT
			const float scale = 4.0f;

			sf::Sprite s;

			s.setScale(scale, scale);

			s.setTexture(rt.getTexture());

			window.draw(s);

			std::vector<float> input(64 * 64);

			// Train
			if (sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
				for (int x = 0; x < res.getSize().x; x++)
					for (int y = 0; y < res.getSize().y; y++) {
						input[x + y * 64] = prediction[x + y * 64];
					}
			}
			else {
				const float predictionIncorporateRatio = 0.1f;

				for (int x = 0; x < res.getSize().x; x++)
					for (int y = 0; y < res.getSize().y; y++) {
						input[x + y * 64] = (1.0f - predictionIncorporateRatio) * res.getPixel(x, y).r / 255.0f + predictionIncorporateRatio * prediction[x + y * 64];
					}
			}

			// Error
			float error = 0.0f;

			for (int x = 0; x < res.getSize().x; x++)
				for (int y = 0; y < res.getSize().y; y++) {
					error += std::pow(res.getPixel(x, y).r / 255.0f - prediction[x + y * 64], 2);
				}

			error /= res.getSize().x * res.getSize().y;

			avgError2 = (1.0f - avgError2Decay) * avgError2 + avgError2Decay * error;

			std::cout << "Squared Error: " << avgError2 << std::endl;

			cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 64, 64, 1 }, 0, 0, input.data());

			ph.simStep(cs, inputImage);

			cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, { 0, 0, 0 }, { 64, 64, 1 }, 0, 0, prediction.data());

			// Show prediction
			for (int x = 0; x < rt.getSize().x; x++)
				for (int y = 0; y < rt.getSize().y; y++) {
					sf::Color c = sf::Color::White;

					c.r = c.b = c.g = std::min(1.0f, std::max(0.0f, prediction[x + y * 64])) * 255.0f;

					pred.setPixel(x, y, c);
				}

			predTex.loadFromImage(pred);

			sf::Sprite sp;

			sp.setTexture(predTex);

			sp.setScale(scale, scale);

			sp.setPosition(window.getSize().x - scale * rt.getSize().x, 0);

			window.draw(sp);

			/*sf::Image sdr;

			sdr.create(prsdr.getLayerDescs().front()._width, prsdr.getLayerDescs().front()._height);

			for (int x = 0; x < sdr.getSize().x; x++)
				for (int y = 0; y < sdr.getSize().y; y++) {
					sf::Color c = sf::Color::White;

					c.r = c.g = c.b = prsdr.getLayers().front()._sdr.getHiddenState(x, y) * 255.0f;

					sdr.setPixel(x, y, c);
				}

			sf::Texture sdrTex;

			sdrTex.loadFromImage(sdr);

			sf::Sprite sdrS;

			sdrS.setTexture(sdrTex);

			sdrS.setPosition(0.0f, window.getSize().y - sdrTex.getSize().y * scale);

			sdrS.setScale(scale, scale);

			window.draw(sdrS);*/

			window.display();
	
			if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
				return 0;
		}
	}

	/*sf::RenderTexture rt;

	rt.create(1024, 1024);

	sf::Texture lineGradientTexture;

	lineGradientTexture.loadFromFile("resources/lineGradient.png");

	sf::Font tickFont;

	tickFont.loadFromFile("resources/arial.ttf");

	plot.draw(rt, lineGradientTexture, tickFont, 1.0f, sf::Vector2f(0.0f, step), sf::Vector2f(0.0f, 1.0f), sf::Vector2f(128.0f, 128.0f), sf::Vector2f(500.0f, 0.1f), 2.0f, 3.0f, 1.5f, 3.0f, 20.0f, 6);

	rt.display();

	rt.getTexture().copyToImage().saveToFile("plot.png");*/

	return 0;
}
Ejemplo n.º 12
0
int main() {
	std::mt19937 generator(time(nullptr));

	sys::ComputeSystem cs;

	cs.create(sys::ComputeSystem::_gpu);

	sys::ComputeProgram prog;

	prog.loadFromFile("resources/neoKernels.cl", cs);

	std::vector<neo::AgentSPG::LayerDesc> layerDescs(3);

	layerDescs[0]._size = { 32, 32 };
	layerDescs[0]._alpha = { 0.1f, 0.001f };
	layerDescs[0]._predictiveRadius = 12;
	layerDescs[0]._feedBackRadius = 12;
	layerDescs[1]._size = { 24, 24 };
	layerDescs[2]._size = { 16, 16 };

	neo::AgentSPG agent;

	int inWidth = 32;
	int inHeight = 32;
	int aWidth = 3;
	int aHeight = 3;
	int frameSkip = 3;

	agent.createRandom(cs, prog, { inWidth, inHeight }, { aWidth, aHeight }, 12, layerDescs, { -0.05f, 0.05f }, generator);

	cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), inWidth, inHeight);
	cl::Image2D actionImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), aWidth, aHeight);

	sf::VideoMode videoMode(1280, 720);

	sf::RenderWindow renderWindow;

	renderWindow.create(videoMode, "Reinforcement Learning", sf::Style::Default);

	renderWindow.setVerticalSyncEnabled(true);

	renderWindow.setFramerateLimit(60);

	// --------------------------------- Game Init -----------------------------------

	const float slimeRadius = 94.5f;
	const float ballRadius = 23.5f;
	const float wallRadius = 22.5f;
	const float fieldRadius = 640.0f;

	const float gravity = 900.0f;
	const float slimeBounce = 100.0f;
	const float wallBounceDecay = 0.8f;
	const float slimeJump = 500.0f;
	const float maxSlimeSpeed = 1000.0f;
	const float slimeMoveAccel = 5000.0f;
	const float slimeMoveDeccel = 8.0f;

	std::uniform_real_distribution<float> dist01(0.0f, 1.0f);

	sf::Vector2f fieldCenter = sf::Vector2f(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f + 254.0f);
	sf::Vector2f wallCenter = fieldCenter + sf::Vector2f(0.0f, -182.0f);

	PhyObj blue;
	PhyObj red;
	PhyObj ball;

	blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
	blue._velocity = sf::Vector2f(0.0f, 0.0f);
	red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
	red._velocity = sf::Vector2f(0.0f, 0.0f);
	ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
	ball._velocity = sf::Vector2f((dist01(generator)) * 600.0f, -(dist01(generator)) * 500.0f);

	sf::Texture backgroundTexture;
	backgroundTexture.loadFromFile("resources/slimevolleyball/background.png");

	sf::Texture blueSlimeTexture;
	blueSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyBlue.png");

	sf::Texture redSlimeTexture;
	redSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyRed.png");

	sf::Texture ballTexture;
	ballTexture.loadFromFile("resources/slimevolleyball/ball.png");

	sf::Texture eyeTexture;
	eyeTexture.loadFromFile("resources/slimevolleyball/slimeEye.png");
	eyeTexture.setSmooth(true);

	sf::Texture arrowTexture;
	arrowTexture.loadFromFile("resources/slimevolleyball/arrow.png");

	sf::Font scoreFont;
	scoreFont.loadFromFile("resources/slimevolleyball/scoreFont.ttf");

	int scoreRed = 0;
	int scoreBlue = 0;

	int prevScoreRed = 0;
	int prevScoreBlue = 0;

	float prevBallX = fieldCenter.x;

	sf::RenderTexture blueRT;

	blueRT.create(inWidth, inHeight);

	// ------------------------------- Simulation Loop -------------------------------

	bool quit = false;

	float dt = 0.017f;

	int skipFrameCounter = 0;

	float blueReward = 0.0f;
	float redReward = 0.0f;

	std::vector<float> actionTemp(aWidth * aHeight, 0.5f);
	std::vector<float> actionTemp2(aWidth * aHeight * 2, 0.5f);

	do {
		sf::Event event;

		while (renderWindow.pollEvent(event)) {
			switch (event.type) {
			case sf::Event::Closed:
				quit = true;
				break;
			}
		}

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
			quit = true;

		// ---------------------------------- Physics ----------------------------------

		bool blueBounced = false;
		bool redBounced = false;

		// Ball
		{
			ball._velocity.y += gravity * dt;
			ball._position += ball._velocity * dt;

			// To floor (game restart)
			if (ball._position.y + ballRadius > fieldCenter.y) {
				if (ball._position.x < fieldCenter.x)
					scoreRed++;
				else
					scoreBlue++;

				blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
				blue._velocity = sf::Vector2f(0.0f, 0.0f);
				red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
				red._velocity = sf::Vector2f(0.0f, 0.0f);
				ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
				ball._velocity = sf::Vector2f(((dist01(generator))) * -600.0f, -(dist01(generator)) * 500.0f);
			}

			// To wall
			if (((ball._position.x + ballRadius) > (wallCenter.x - wallRadius) && ball._position.x < wallCenter.x) || ((ball._position.x - ballRadius) < (wallCenter.x + wallRadius) && ball._position.x > wallCenter.x)) {
				// If above rounded part
				if (ball._position.y < wallCenter.y) {
					sf::Vector2f delta = ball._position - wallCenter;

					float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

					if (dist < wallRadius + ballRadius) {
						sf::Vector2f normal = delta / dist;

						// Reflect velocity
						sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;

						ball._velocity = reflectedVelocity * wallBounceDecay;

						ball._position = wallCenter + normal * (wallRadius + ballRadius);
					}
				}
				else {
					// If on left side
					if (ball._position.x < wallCenter.x) {
						ball._velocity.x = wallBounceDecay * -ball._velocity.x;
						ball._position.x = wallCenter.x - wallRadius - ballRadius;
					}
					else {
						ball._velocity.x = wallBounceDecay * -ball._velocity.x;
						ball._position.x = wallCenter.x + wallRadius + ballRadius;
					}
				}
			}

			// To blue slime			
			{
				sf::Vector2f delta = ball._position - blue._position;

				float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

				if (dist < slimeRadius + ballRadius) {
					sf::Vector2f normal = delta / dist;

					// Reflect velocity
					sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;

					float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);

					sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;

					ball._velocity = blue._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);

					ball._position = blue._position + normal * (wallRadius + slimeRadius);

					blueBounced = true;
				}
			}

			// To red slime			
			{
				sf::Vector2f delta = ball._position - red._position;

				float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

				if (dist < slimeRadius + ballRadius) {
					sf::Vector2f normal = delta / dist;

					// Reflect velocity
					sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;

					float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);

					sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;

					ball._velocity = red._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);

					ball._position = red._position + normal * (wallRadius + slimeRadius);

					redBounced = true;
				}
			}

			// Out of field, left and right
			{
				if (ball._position.x - ballRadius < fieldCenter.x - fieldRadius) {
					ball._velocity.x = wallBounceDecay * -ball._velocity.x;
					ball._position.x = fieldCenter.x - fieldRadius + ballRadius;
				}
				else if (ball._position.x + ballRadius > fieldCenter.x + fieldRadius) {
					ball._velocity.x = wallBounceDecay * -ball._velocity.x;
					ball._position.x = fieldCenter.x + fieldRadius - ballRadius;
				}
			}
		}

		// Blue slime
		{
			blue._velocity.y += gravity * dt;
			blue._velocity.x += -slimeMoveDeccel * blue._velocity.x * dt;
			blue._position += blue._velocity * dt;

			bool moveLeft;
			bool moveRight;
			bool jump;

			sf::Image img = blueRT.getTexture().copyToImage();

			std::vector<float> greyData(img.getSize().x * img.getSize().y);

			for (int x = 0; x < img.getSize().x; x++)
				for (int y = 0; y < img.getSize().y; y++) {
					sf::Color c = img.getPixel(x, y);
					greyData[x + y * img.getSize().x] = 0.333f * (c.r / 255.0f + c.g / 255.0f + c.b / 255.0f);
				}
	
			if (skipFrameCounter == 0) {
				cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { static_cast<cl::size_type>(inWidth), static_cast<cl::size_type>(inHeight), 1 }, 0, 0, greyData.data());

				cs.getQueue().enqueueWriteImage(actionImage, CL_TRUE, { 0, 0, 0 }, { static_cast<cl::size_type>(inWidth), static_cast<cl::size_type>(inHeight), 1 }, 0, 0, actionTemp.data());

				agent.simStep(cs, blueReward * 0.01f, inputImage, actionImage, generator);
				
				cs.getQueue().enqueueReadImage(agent.getAction(), CL_TRUE, { 0, 0, 0 }, { static_cast<cl::size_type>(aWidth), static_cast<cl::size_type>(aHeight), 1 }, 0, 0, actionTemp2.data());
			
				for (int i = 0; i < actionTemp.size(); i++) {
					if (dist01(generator) < 0.1f)
						actionTemp[i] = dist01(generator);
					else
						actionTemp[i] = actionTemp2[i * 2 + 0];
				}

				actionTemp[2] = actionTemp[0] * 3.0f + 1.0f;
				actionTemp[3] = actionTemp[0] * -2.0f + 0.4f;
				actionTemp[4] = actionTemp[0] * 2.0f - 0.4f;
				actionTemp[5] = actionTemp[0] * -2.0f - 0.4f;
				actionTemp[6] = actionTemp[1] * -1.5f + 0.2f;
				actionTemp[7] = actionTemp[1] * 2.1f - 0.4f;
				actionTemp[8] = actionTemp[1] * -2.0f + 0.4f;

				cs.getQueue().finish();
			}

			moveLeft = actionTemp[0] > 0.75f;
			moveRight = actionTemp[0] < 0.25f;
			jump = actionTemp[1] > 0.75f;

			std::cout << blueReward << " " << actionTemp[0 * 2 + 0] << " " << actionTemp[1 * 2 + 0] << std::endl;

			if (moveLeft) {
				blue._velocity.x += -slimeMoveAccel * dt;

				if (blue._velocity.x < -maxSlimeSpeed)
					blue._velocity.x = -maxSlimeSpeed;
			}
			else if (moveRight) {
				blue._velocity.x += slimeMoveAccel * dt;

				if (blue._velocity.x > maxSlimeSpeed)
					blue._velocity.x = maxSlimeSpeed;
			}

			if (blue._position.y > fieldCenter.y) {
				blue._velocity.y = 0.0f;
				blue._position.y = fieldCenter.y;

				if (jump)
					blue._velocity.y -= slimeJump;
			}

			if (blue._position.x - slimeRadius < fieldCenter.x - fieldRadius) {
				blue._velocity.x = 0.0f;
				blue._position.x = fieldCenter.x - fieldRadius + slimeRadius;
			}

			if (blue._position.x + slimeRadius > wallCenter.x - wallRadius) {
				blue._velocity.x = 0.0f;
				blue._position.x = wallCenter.x - wallRadius - slimeRadius;
			}
		}

		// Red slime
		{
			red._velocity.y += gravity * dt;
			red._velocity.x += -slimeMoveDeccel * red._velocity.x * dt;
			red._position += red._velocity * dt;

			bool moveLeft;
			bool moveRight;
			bool jump;

			moveLeft = sf::Keyboard::isKeyPressed(sf::Keyboard::Left);
			moveRight = sf::Keyboard::isKeyPressed(sf::Keyboard::Right);
			jump = sf::Keyboard::isKeyPressed(sf::Keyboard::Up);

			if (moveLeft) {
				red._velocity.x += -slimeMoveAccel * dt;

				if (red._velocity.x < -maxSlimeSpeed)
					red._velocity.x = -maxSlimeSpeed;
			}
			else if (moveRight) {
				red._velocity.x += slimeMoveAccel * dt;

				if (red._velocity.x > maxSlimeSpeed)
					red._velocity.x = maxSlimeSpeed;
			}

			if (red._position.y > fieldCenter.y) {
				red._velocity.y = 0.0f;
				red._position.y = fieldCenter.y;

				if (jump)
					red._velocity.y -= slimeJump;
			}

			if (red._position.x + slimeRadius > fieldCenter.x + fieldRadius) {
				red._velocity.x = 0.0f;
				red._position.x = fieldCenter.x + fieldRadius - slimeRadius;
			}

			if (red._position.x - slimeRadius < wallCenter.x + wallRadius) {
				red._velocity.x = 0.0f;
				red._position.x = wallCenter.x + wallRadius + slimeRadius;
			}
		}

		blueReward = scoreBlue - prevScoreBlue - (scoreRed - prevScoreRed);// -0.00005f * std::abs(ball._position.x - blue._position.x) + (blueBounced ? 0.2f : 0.0f);
		redReward = scoreRed - prevScoreRed - (scoreBlue - prevScoreBlue);// -0.00005f * std::abs(ball._position.x - red._position.x) + (redBounced ? 0.2f : 0.0f);

		prevScoreRed = scoreRed;
		prevScoreBlue = scoreBlue;
		prevBallX = ball._position.x;

		// --------------------------------- Rendering ---------------------------------

		if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
			{
				renderWindow.clear();

				{
					sf::Sprite s;
					s.setTexture(backgroundTexture);
					s.setOrigin(backgroundTexture.getSize().x * 0.5f, backgroundTexture.getSize().y * 0.5f);
					s.setPosition(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f);

					renderWindow.draw(s);
				}

				{
					sf::Sprite s;
					s.setTexture(blueSlimeTexture);
					s.setOrigin(blueSlimeTexture.getSize().x * 0.5f, blueSlimeTexture.getSize().y);
					s.setPosition(blue._position);

					renderWindow.draw(s);
				}

				{
					sf::Sprite s;
					s.setTexture(eyeTexture);
					s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
					s.setPosition(blue._position + sf::Vector2f(50.0f, -28.0f));

					sf::Vector2f delta = ball._position - s.getPosition();

					float angle = std::atan2(delta.y, delta.x);

					s.setRotation(angle * 180.0f / 3.141596f);

					renderWindow.draw(s);
				}

				{
					sf::Sprite s;
					s.setTexture(redSlimeTexture);
					s.setOrigin(redSlimeTexture.getSize().x * 0.5f, redSlimeTexture.getSize().y);
					s.setPosition(red._position);

					renderWindow.draw(s);
				}

				{
					sf::Sprite s;
					s.setTexture(eyeTexture);
					s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
					s.setPosition(red._position + sf::Vector2f(-50.0f, -28.0f));

					sf::Vector2f delta = ball._position - s.getPosition();

					float angle = std::atan2(delta.y, delta.x);

					s.setRotation(angle * 180.0f / 3.141596f);

					renderWindow.draw(s);
				}

				{
					sf::Sprite s;
					s.setTexture(ballTexture);
					s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
					s.setPosition(ball._position);

					renderWindow.draw(s);
				}

				if (ball._position.y + ballRadius < 0.0f) {
					sf::Sprite s;
					s.setTexture(arrowTexture);
					s.setOrigin(arrowTexture.getSize().x * 0.5f, 0.0f);
					s.setPosition(ball._position.x, 0.0f);

					renderWindow.draw(s);
				}

				{
					sf::Text scoreText;
					scoreText.setFont(scoreFont);
					scoreText.setString(std::to_string(scoreBlue));
					scoreText.setCharacterSize(100);

					float width = scoreText.getLocalBounds().width;

					scoreText.setPosition(fieldCenter.x - width * 0.5f - 100.0f, 10.0f);

					scoreText.setColor(sf::Color(100, 133, 255));

					renderWindow.draw(scoreText);
				}

				{
					sf::Text scoreText;
					scoreText.setFont(scoreFont);
					scoreText.setString(std::to_string(scoreRed));
					scoreText.setCharacterSize(100);

					float width = scoreText.getLocalBounds().width;

					scoreText.setPosition(fieldCenter.x - width * 0.5f + 100.0f, 10.0f);

					scoreText.setColor(sf::Color(255, 100, 100));

					renderWindow.draw(scoreText);
				}

				{
					sf::Sprite visionSprite;
					visionSprite.setTexture(blueRT.getTexture());
					visionSprite.scale(4.0f, 4.0f);

					renderWindow.draw(visionSprite);
				}
			}

			renderWindow.display();
		}

		blueRT.setView(renderWindow.getView());

		{
			blueRT.clear();

			{
				sf::Sprite s;
				s.setTexture(backgroundTexture);
				s.setOrigin(backgroundTexture.getSize().x * 0.5f, backgroundTexture.getSize().y * 0.5f);
				s.setPosition(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f);

				blueRT.draw(s);
			}

			{
				sf::Sprite s;
				s.setTexture(blueSlimeTexture);
				s.setOrigin(blueSlimeTexture.getSize().x * 0.5f, blueSlimeTexture.getSize().y);
				s.setPosition(blue._position);

				blueRT.draw(s);
			}

			{
				sf::Sprite s;
				s.setTexture(eyeTexture);
				s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
				s.setPosition(blue._position + sf::Vector2f(50.0f, -28.0f));

				sf::Vector2f delta = ball._position - s.getPosition();

				float angle = std::atan2(delta.y, delta.x);

				s.setRotation(angle * 180.0f / 3.141596f);

				blueRT.draw(s);
			}

			{
				sf::Sprite s;
				s.setTexture(redSlimeTexture);
				s.setOrigin(redSlimeTexture.getSize().x * 0.5f, redSlimeTexture.getSize().y);
				s.setPosition(red._position);

				blueRT.draw(s);
			}

			{
				sf::Sprite s;
				s.setTexture(eyeTexture);
				s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
				s.setPosition(red._position + sf::Vector2f(-50.0f, -28.0f));

				sf::Vector2f delta = ball._position - s.getPosition();

				float angle = std::atan2(delta.y, delta.x);

				s.setRotation(angle * 180.0f / 3.141596f);

				blueRT.draw(s);
			}

			{
				sf::Sprite s;
				s.setTexture(ballTexture);
				s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
				s.setPosition(ball._position);

				blueRT.draw(s);
			}

			if (ball._position.y + ballRadius < 0.0f) {
				sf::Sprite s;
				s.setTexture(arrowTexture);
				s.setOrigin(arrowTexture.getSize().x * 0.5f, 0.0f);
				s.setPosition(ball._position.x, 0.0f);

				blueRT.draw(s);
			}

			{
				sf::Text scoreText;
				scoreText.setFont(scoreFont);
				scoreText.setString(std::to_string(scoreBlue));
				scoreText.setCharacterSize(100);

				float width = scoreText.getLocalBounds().width;

				scoreText.setPosition(fieldCenter.x - width * 0.5f - 100.0f, 10.0f);

				scoreText.setColor(sf::Color(100, 133, 255));

				blueRT.draw(scoreText);
			}

			{
				sf::Text scoreText;
				scoreText.setFont(scoreFont);
				scoreText.setString(std::to_string(scoreRed));
				scoreText.setCharacterSize(100);

				float width = scoreText.getLocalBounds().width;

				scoreText.setPosition(fieldCenter.x - width * 0.5f + 100.0f, 10.0f);

				scoreText.setColor(sf::Color(255, 100, 100));

				blueRT.draw(scoreText);
			}

			blueRT.display();
		}

		skipFrameCounter = (skipFrameCounter + 1) % frameSkip;
	} while (!quit);

	return 0;
}
Ejemplo n.º 13
0
int main(int argc, const char** argv) {
	
    // Load the command line config
    
    ArgumentParser parser;
        
    parser.addArgument("-e", "--epochs", 1);
    parser.addArgument("-s", "--seed", 1);
    parser.addArgument("-l", "--layers", 1);
    parser.addArgument("-S", "--samples", 1);
    parser.addArgument("-c", "--corpus", 1);
    parser.addArgument("--nlayers", 1);
    parser.addArgument("--ifbradius", 1);
    parser.addArgument("--lw", 1);
    parser.addArgument("--lh", 1);
    parser.addArgument("--ssize", 1);
    parser.addArgument("--sseednoise", 1);
    parser.addArgument("--sprednoise", 1);
    
    parser.parse(argc, argv);
    
	// RNG
    unsigned int seed = std::atoi(parser.retrieve("seed", std::to_string(time(nullptr))).c_str());
	std::mt19937 generator(seed);

	std::uniform_real_distribution<float> dist01(0.0f, 1.0f);
    
	// ---------------------------------- Load the Corpus ----------------------------------
	std::string corpusPath = parser.retrieve("corpus", "corpus.txt");
	std::ifstream fromFile(corpusPath);

	fromFile.seekg(0, std::ios::end);
	size_t size = fromFile.tellg();
	std::string test(size, ' ');
	fromFile.seekg(0);
	fromFile.read(&test[0], size);
	
	// ---------------------------------- Find Character Set ----------------------------------
	
	VectorCodec textcodec(test);
	int numInputs = textcodec.N;
	int inputsRoot = std::ceil(std::sqrt(static_cast<float>(numInputs)));
	
	// ---------------------------------- Create Hierarchy ----------------------------------
	
	// Fill out layer descriptions
	int nLayers = std::atoi(parser.retrieve("nlayers", "3").c_str());
	int layerW = std::atoi(parser.retrieve("lw", "16").c_str());
	int layerH = std::atoi(parser.retrieve("lh", "16").c_str());
	int inFeedBackRadius = std::atoi(parser.retrieve("ifbradius", "16").c_str());
	
	std::vector<neo::PredictiveHierarchy::LayerDesc> layerDescs(nLayers);
	
	for (int i = 0; i < nLayers; i++) {
		layerDescs[i]._width = layerW;
		layerDescs[i]._height = layerH;
	}
	
	neo::PredictiveHierarchy ph;
	
	ph.createRandom(inputsRoot, inputsRoot, inFeedBackRadius, layerDescs, -0.01f, 0.01f, 0.01f, 0.05f, 0.1f, generator);
	
	// ---------------------------------- Iterate Over Corpus ----------------------------------
    int numEpochs = std::atoi(parser.retrieve("epochs", "10").c_str());
    int numSamples = std::atoi(parser.retrieve("samples", "10").c_str());
    int sampleSize = std::atoi(parser.retrieve("ssize", std::to_string(test.length())).c_str());
    float sampleSeedNoise = std::atof(parser.retrieve("sseednoise", "0.5").c_str());
    float samplePredNoise = std::atof(parser.retrieve("sprednoise", "0.05").c_str());
    
    std::cout << "NeoRL text prediction experiment" << std::endl;
    std::cout << "Corpus: " << corpusPath << " size: " << test.length() << " alphabet size: " << textcodec.nSymbols << std::endl;
    std::cout << "Model: nLayers: " << nLayers << " layerW: " << layerW << " layerH: " << layerH << " inFeedbackRadius: " << inFeedBackRadius 
			  << " input: " << inputsRoot << "x" << inputsRoot << std::endl;
	std::cout << "Training: epochs: " << numEpochs << std::endl;
	std::cout << "Sampling: samples: " << numSamples << " size: " << sampleSize << " seed noise: " << sampleSeedNoise << " pred noise " << samplePredNoise << std::endl;    
    std::cout << "--[ Start training ]--" << std::endl;
    
    train(ph, generator, test, numEpochs, textcodec);
    
    std::cout << "--[ Start sampling ]--" << std::endl;
    for (int i = 0; i < numSamples; i++) {
        sample(ph, generator, textcodec.getRandomSymbol(generator), sampleSize, textcodec, sampleSeedNoise, samplePredNoise);
    }
    
	return 0;
}
Ejemplo n.º 14
0
Archivo: ga.cpp Proyecto: srgkoval/GA
template <int N, int N_obj> void GA<N, N_obj>::mutation_adaptive
	(const Individual &parent, Individual &child)
{
    double tol = 1.e-8;             // tolerance, pure magic

    // set step size
    if(generation <= 1)
    {
        ma_step_size = 1.;
    }
    else
    {
        if(!ma_step_changed)
        {
            ma_step_changed = true;

            if(N_obj == 1)
            {
                if(best_score[generation] < best_score[generation - 1])
                    ma_step_size = std::min(1., ma_step_size * 4.);
                else
                    ma_step_size = std::max(tol, ma_step_size / 4.);
            }
            else
            {
                if(spread[generation] > spread[generation - 1])
                    ma_step_size = std::min(1., ma_step_size * 4.);
                else
                    ma_step_size = std::max(tol, ma_step_size / 4.);
            }
            
        }
    }

    // set logarithmic scale
    Individual scale;
    for(int i = 0; i < N; i++)
    {
        double exponent = 0.;
        if( fabs(lower_boundary[i]) > tol )
            exponent += 0.5 * log(fabs(lower_boundary[i])) / log(2.);
        if( fabs(upper_boundary[i]) > tol )
            exponent += 0.5 * log(fabs(upper_boundary[i])) / log(2.);

        scale[i] = pow(2., exponent);
    }

    Individual raw_basis[N],
               basis[N],
               tangent_cone[N],
               dir[2 * N];
    double dir_sign[4 * N];

    int n_tangent = 0,
        n_basis = N,
        index_vector[4 * N],
        order_vector[4 * N];

    // calculate mutation direction set
    // tangent components
    for(int i = 0; i < N; i++)
    {
        if( fabs(parent[i] - lower_boundary[i]) < tol || fabs(parent[i] - upper_boundary[i]) < tol )
        {
            tangent_cone[n_tangent] = 0.;
            tangent_cone[n_tangent][i] = 1.;
            n_tangent++;
        }
    }
    
    // raw basis vectors
    double poll_param = 1. / sqrt(ma_step_size);

    for(int i = 0; i < n_basis; i++)
    {
        raw_basis[i] = 0.;
        raw_basis[i][i] = poll_param * (dist01(rnd_generator) >= 0.5 ? 1. : -1.);
        for(int j = i + 1; j < n_basis; j++)
        {
            raw_basis[i][j] = round( (poll_param + 1.) * dist01(rnd_generator) - 0.5);
        }
    }

    // basis as random permutation of raw basis
    for(int j = 0; j < n_basis; j++)
        order_vector[j] = j;
    std::random_shuffle(order_vector, order_vector + n_basis);

    for(int i = 0; i < n_basis; i++)
        for(int j = 0; j < n_basis; j++)
            basis[i][j] = raw_basis[order_vector[i]][order_vector[j]];

    // prerare random direction mutation
    int n_dir = n_tangent + n_basis;

    for(int i = 0; i < n_basis; i++)
        dir[i] = basis[i];

    for(int i = 0; i < n_tangent; i++)
        dir[n_basis + i] = tangent_cone[i];

    for(int i = 0; i < n_basis; i++)
    {
        index_vector[i] = i;
        dir_sign[i] = 1.;
    }
    
    int i_base = n_basis;
    for(int i = i_base; i < i_base + n_basis; i++)
    {
        index_vector[i] = i - i_base;
        dir_sign[i] = -1.;
    }
    
    i_base += n_basis;
    for(int i = i_base; i < i_base + n_tangent; i++)
    {
        index_vector[i] = i - i_base;
        dir_sign[i] = 1.;
    }
    
    i_base += n_tangent;
    for(int i = i_base; i < i_base + n_tangent; i++)
    {
        index_vector[i] = i - i_base;
        dir_sign[i] = -1.;
    }
    
    int n_dir_total = 2 * n_dir;
    for(int i = 0; i < n_dir_total; i++)
        order_vector[i] = i;
    std::random_shuffle(order_vector, order_vector + n_dir_total);

    // finally, mutate
    double success = false;
    for(int i = 0; i < n_dir_total; i++)
    {
        int k = index_vector[order_vector[i]];
        Individual direction = dir_sign[k] * dir[k];
        
        child = parent + ma_step_size * scale * direction;

        if(feasible(child))
        {
            success = true;
            break;
        }
    }

    if( !success )
    {
        child = parent;
        if(options.verbose && N < 5)
            std::cout << "mutation failed at x = " << parent << "\n";
    }
}
Ejemplo n.º 15
0
Archivo: ga.cpp Proyecto: srgkoval/GA
template <int N, int N_obj> void GA<N, N_obj>::crossover_arithmetic
	(const Individual &parent1, const Individual &parent2, Individual &child)
{
    double R = dist01(rnd_generator);
    child = R * parent1 + (1. - R) * parent2;
}
Ejemplo n.º 16
0
int main() {
	std::mt19937 generator(time(nullptr));

	sf::RenderWindow renderWindow;

	renderWindow.create(sf::VideoMode(1280, 720), "Reinforcement Learning", sf::Style::Default);

	renderWindow.setVerticalSyncEnabled(true);
	renderWindow.setFramerateLimit(60);

	// ---------------------------------- RL Init ------------------------------------

	/*sc::HTSLPVLV agentBlue;
	sc::HTSLPVLV agentRed;

	std::vector<sc::HTSLPVLV::InputType> inputTypes(20);

	for (int i = 0; i < 12; i++)
		inputTypes[i] = sc::HTSLPVLV::_state;
	
	for (int i = 12; i < 14; i++)
		inputTypes[i] = sc::HTSLPVLV::_action;

	for (int i = 14; i < 16; i++)
		inputTypes[i] = sc::HTSLPVLV::_pv;

	for (int i = 16; i < 18; i++)
		inputTypes[i] = sc::HTSLPVLV::_lve;

	for (int i = 18; i < 20; i++)
		inputTypes[i] = sc::HTSLPVLV::_lvi;

	std::vector<sc::HTSL::LayerDesc> layerDescs(2);

	layerDescs[0]._width = 20;
	layerDescs[0]._height = 20;

	layerDescs[1]._width = 12;
	layerDescs[1]._height = 12;

	//layerDescs[2]._width = 8;
	//layerDescs[2]._height = 8;

	agentBlue.createRandom(5, 4, 8, inputTypes, layerDescs, generator);
	agentRed.createRandom(5, 4, 8, inputTypes, layerDescs, generator);*/

	float rewardTimer = 0.0f;
	float rewardTime = 0.25f;
	float lastReward = 0.5f;

	//sc::HTSLSARSA agentBlue;
	sc::HTSLSARSA agentRed;

	std::vector<sc::HTSLSARSA::InputType> inputTypes(16);

	for (int i = 0; i < 12; i++)
		inputTypes[i] = sc::HTSLSARSA::_state;

	for (int i = 12; i < 14; i++)
		inputTypes[i] = sc::HTSLSARSA::_action;

	for (int i = 14; i < 16; i++)
		inputTypes[i] = sc::HTSLSARSA::_q;

	std::vector<sc::HTSL::LayerDesc> layerDescs(3);

	layerDescs[0]._width = 28;
	layerDescs[0]._height = 28;

	layerDescs[1]._width = 18;
	layerDescs[1]._height = 18;

	layerDescs[2]._width = 12;
	layerDescs[2]._height = 12;

	//agentBlue.createRandom(4, 4, 8, inputTypes, layerDescs, generator);
	agentRed.createRandom(4, 4, 8, inputTypes, layerDescs, generator);

	//deep::FERL agentBlue;
	//agentBlue.createRandom(12, 2, 32, 0.01f, generator);

	//deep::FERL agentRed;
	//agentRed.createRandom(12, 2, 32, 0.01f, generator);

	// --------------------------------- Game Init -----------------------------------

	const float slimeRadius = 94.5f;
	const float ballRadius = 23.5f;
	const float wallRadius = 22.5f;
	const float fieldRadius = 640.0f;

	const float gravity = 900.0f;
	const float slimeBounce = 100.0f;
	const float wallBounceDecay = 0.8f;
	const float slimeJump = 500.0f;
	const float maxSlimeSpeed = 1000.0f;
	const float slimeMoveAccel = 5000.0f;
	const float slimeMoveDeccel = 8.0f;

	std::uniform_real_distribution<float> dist01(0.0f, 1.0f);

	sf::Vector2f fieldCenter = sf::Vector2f(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f + 254.0f);
	sf::Vector2f wallCenter = fieldCenter + sf::Vector2f(0.0f, -182.0f);

	PhyObj blue;
	PhyObj red;
	PhyObj ball;

	blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
	blue._velocity = sf::Vector2f(0.0f, 0.0f);
	red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
	red._velocity = sf::Vector2f(0.0f, 0.0f);
	ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
	ball._velocity = sf::Vector2f((dist01(generator)) * 600.0f, -(dist01(generator)) * 500.0f);

	sf::Texture backgroundTexture;
	backgroundTexture.loadFromFile("resources/slimevolleyball/background.png");

	sf::Texture blueSlimeTexture;
	blueSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyBlue.png");

	sf::Texture redSlimeTexture;
	redSlimeTexture.loadFromFile("resources/slimevolleyball/slimeBodyRed.png");

	sf::Texture ballTexture;
	ballTexture.loadFromFile("resources/slimevolleyball/ball.png");

	sf::Texture eyeTexture;
	eyeTexture.loadFromFile("resources/slimevolleyball/slimeEye.png");
	eyeTexture.setSmooth(true);

	sf::Texture arrowTexture;
	arrowTexture.loadFromFile("resources/slimevolleyball/arrow.png");

	sf::Font scoreFont;
	scoreFont.loadFromFile("resources/slimevolleyball/scoreFont.ttf");

	int scoreRed = 0;
	int scoreBlue = 0;

	int prevScoreRed = 0;
	int prevScoreBlue = 0;

	float prevBallX = fieldCenter.x;

	// ------------------------------- Simulation Loop -------------------------------

	bool noRender = false;
	bool prevPressK = false;

	bool quit = false;

	float dt = 0.017f;

	do {
		sf::Event event;

		while (renderWindow.pollEvent(event)) {
			switch (event.type) {
			case sf::Event::Closed:
				quit = true;
				break;
			}
		}

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape))
			quit = true;

		// ---------------------------------- Physics ----------------------------------

		bool blueBounced = false;
		bool redBounced = false;

		// Ball
		{
			ball._velocity.y += gravity * dt;
			ball._position += ball._velocity * dt;

			// To floor (game restart)
			if (ball._position.y + ballRadius > fieldCenter.y) {
				if (ball._position.x < fieldCenter.x)
					scoreRed++;
				else
					scoreBlue++;

				blue._position = fieldCenter + sf::Vector2f(-200.0f, 0.0f);
				blue._velocity = sf::Vector2f(0.0f, 0.0f);
				red._position = fieldCenter + sf::Vector2f(200.0f, 0.0f);
				red._velocity = sf::Vector2f(0.0f, 0.0f);
				ball._position = fieldCenter + sf::Vector2f(2.0f, -300.0f);
				ball._velocity = sf::Vector2f((dist01(generator)) * 600.0f, -(dist01(generator)) * 500.0f);
			}

			// To wall
			if (((ball._position.x + ballRadius) > (wallCenter.x - wallRadius) && ball._position.x < wallCenter.x) || ((ball._position.x - ballRadius) < (wallCenter.x + wallRadius) && ball._position.x > wallCenter.x)) {
				// If above rounded part
				if (ball._position.y < wallCenter.y) {
					sf::Vector2f delta = ball._position - wallCenter;

					float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

					if (dist < wallRadius + ballRadius) {
						sf::Vector2f normal = delta / dist;

						// Reflect velocity
						sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;

						ball._velocity = reflectedVelocity * wallBounceDecay;

						ball._position = wallCenter + normal * (wallRadius + ballRadius);
					}
				}
				else {
					// If on left side
					if (ball._position.x < wallCenter.x) {
						ball._velocity.x = wallBounceDecay * -ball._velocity.x;
						ball._position.x = wallCenter.x - wallRadius - ballRadius;
					}
					else {
						ball._velocity.x = wallBounceDecay * -ball._velocity.x;
						ball._position.x = wallCenter.x + wallRadius + ballRadius;
					}
				}
			}

			// To blue slime			
			{
				sf::Vector2f delta = ball._position - blue._position;

				float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

				if (dist < slimeRadius + ballRadius) {
					sf::Vector2f normal = delta / dist;

					// Reflect velocity
					sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;

					float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);

					sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;

					ball._velocity = blue._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);

					ball._position = blue._position + normal * (wallRadius + slimeRadius);

					blueBounced = true;
				}
			}

			// To red slime			
			{
				sf::Vector2f delta = ball._position - red._position;

				float dist = std::sqrt(delta.x * delta.x + delta.y * delta.y);

				if (dist < slimeRadius + ballRadius) {
					sf::Vector2f normal = delta / dist;

					// Reflect velocity
					sf::Vector2f reflectedVelocity = ball._velocity - 2.0f * (ball._velocity.x * normal.x + ball._velocity.y * normal.y) * normal;

					float magnitude = std::sqrt(reflectedVelocity.x * reflectedVelocity.x + reflectedVelocity.y * reflectedVelocity.y);

					sf::Vector2f normalizedReflected = reflectedVelocity / magnitude;

					ball._velocity = red._velocity + (magnitude > slimeBounce ? reflectedVelocity : normalizedReflected * slimeBounce);

					ball._position = red._position + normal * (wallRadius + slimeRadius);

					redBounced = true;
				}
			}

			// Out of field, left and right
			{
				if (ball._position.x - ballRadius < fieldCenter.x - fieldRadius) {
					ball._velocity.x = wallBounceDecay * -ball._velocity.x;
					ball._position.x = fieldCenter.x - fieldRadius + ballRadius;
				}
				else if (ball._position.x + ballRadius > fieldCenter.x + fieldRadius) {
					ball._velocity.x = wallBounceDecay * -ball._velocity.x;
					ball._position.x = fieldCenter.x + fieldRadius - ballRadius;
				}
			}
		}

		// Blue slime
		{		
			blue._velocity.y += gravity * dt;
			blue._velocity.x += -slimeMoveDeccel * blue._velocity.x * dt;
			blue._position += blue._velocity * dt;

			if (sf::Keyboard::isKeyPressed(sf::Keyboard::A)) {
				blue._velocity.x += -slimeMoveAccel * dt;

				if (blue._velocity.x < -maxSlimeSpeed)
					blue._velocity.x = -maxSlimeSpeed;
			}
			else if (sf::Keyboard::isKeyPressed(sf::Keyboard::D)) {
				blue._velocity.x += slimeMoveAccel * dt;

				if (blue._velocity.x > maxSlimeSpeed)
					blue._velocity.x = maxSlimeSpeed;
			}

			if (blue._position.y > fieldCenter.y) {
				blue._velocity.y = 0.0f;
				blue._position.y = fieldCenter.y;
				
				if (sf::Keyboard::isKeyPressed(sf::Keyboard::W))
					blue._velocity.y -= slimeJump;
			}

			if (blue._position.x - slimeRadius < fieldCenter.x - fieldRadius) {
				blue._velocity.x = 0.0f;
				blue._position.x = fieldCenter.x - fieldRadius + slimeRadius;
			}

			if (blue._position.x + slimeRadius > wallCenter.x - wallRadius) {
				blue._velocity.x = 0.0f;
				blue._position.x = wallCenter.x - wallRadius - slimeRadius;
			}
		}

		// Blue slime
		/*{
			const float scalar = 0.001f;
			// Percepts
			std::vector<float> inputs(12);

			int index = 0;

			inputs[index++] = (ball._position.x - blue._position.x) * scalar;
			inputs[index++] = (ball._position.y - blue._position.y) * scalar;
			inputs[index++] = ball._velocity.x * scalar;
			inputs[index++] = ball._velocity.y * scalar;
			inputs[index++] = (red._position.x - blue._position.x) * scalar;
			inputs[index++] = (red._position.y - blue._position.y) * scalar;
			inputs[index++] = red._velocity.x * scalar;
			inputs[index++] = red._velocity.y * scalar;
			inputs[index++] = blue._position.x * scalar;
			inputs[index++] = blue._position.y * scalar;
			inputs[index++] = blue._velocity.x * scalar;
			inputs[index++] = blue._velocity.y * scalar;

			std::vector<float> outputs(2);

			// Actions
			for (int i = 0; i < 12; i++)
				agentBlue.setState(i, inputs[i]);

			float reward = (ball._position.x > fieldCenter.x && prevBallX < fieldCenter.x ? 1.0f : 0.5f) * 0.5f + (scoreRed > prevScoreRed ? 0.0f : 0.5f) * 0.5f;

			if (reward != 0.5f) {
				lastReward = reward = reward > 0.5f ? 1.0f : 0.0f;
				rewardTimer = 0.0f;
			}
			else if (rewardTimer < rewardTime) {
				reward = lastReward;

				rewardTimer += dt;
			}

			if (blueBounced)
				reward = std::max(reward, 0.55f);

			reward = (reward * 2.0f - 1.0f) * 0.01f - std::abs(ball._position.x - blue._position.x) * 0.001f;

			//reward = (scoreBlue - prevScoreBlue) - (scoreRed - prevScoreRed) - std::abs(ball._position.x - blue._position.x) * 0.001f;

			agentBlue.update(reward, generator);

			float move = agentBlue.getActionFromNodeIndex(0) * 2.0f - 1.0f;
			bool jump = agentBlue.getActionFromNodeIndex(1) > 0.5f;

			//std::vector<float> action(2);
			//agentBlue.step(inputs, action, reward, 0.01f, 0.995f, 0.99f, 10.0f, 64, 3, 0.02f, 0.04f, 0.04f, 800, 200, 0.003f, 0.0f, generator);

			//float move = action[0];
			//bool jump = action[1] > 0.0f;

			blue._velocity.y += gravity * dt;
			blue._velocity.x += -slimeMoveDeccel * blue._velocity.x * dt;
			blue._position += blue._velocity * dt;

			{
				blue._velocity.x += move * slimeMoveAccel * dt;

				if (blue._velocity.x < -maxSlimeSpeed)
					blue._velocity.x = -maxSlimeSpeed;
				else if (blue._velocity.x > maxSlimeSpeed)
					blue._velocity.x = maxSlimeSpeed;
			}

			if (blue._position.y > fieldCenter.y) {
				blue._velocity.y = 0.0f;
				blue._position.y = fieldCenter.y;

				if (jump)
					blue._velocity.y -= slimeJump;
			}

			if (blue._position.x - slimeRadius < fieldCenter.x - fieldRadius) {
				blue._velocity.x = 0.0f;
				blue._position.x = fieldCenter.x - fieldRadius + slimeRadius;
			}

			if (blue._position.x + slimeRadius > wallCenter.x - wallRadius) {
				blue._velocity.x = 0.0f;
				blue._position.x = wallCenter.x - wallRadius - slimeRadius;
			}
		}*/

		// Red slime
		{
			const float scalar = 0.001f;
			// Percepts
			std::vector<float> inputs(12);

			int index = 0;

			inputs[index++] = (ball._position.x - red._position.x) * scalar;
			inputs[index++] = (ball._position.y - red._position.y) * scalar;
			inputs[index++] = ball._velocity.x * scalar;
			inputs[index++] = ball._velocity.y * scalar;
			inputs[index++] = (blue._position.x - red._position.x) * scalar;
			inputs[index++] = (blue._position.y - red._position.y) * scalar;
			inputs[index++] = red._velocity.x * scalar;
			inputs[index++] = red._velocity.y * scalar;
			inputs[index++] = blue._position.x * scalar;
			inputs[index++] = blue._position.y * scalar;
			inputs[index++] = blue._velocity.x * scalar;
			inputs[index++] = blue._velocity.y * scalar;

			//std::vector<float> outputs(2);

			// Actions
			for (int i = 0; i < 12; i++)
				agentRed.setState(i, inputs[i]);

			float reward = (ball._position.x < fieldCenter.x && prevBallX >= fieldCenter.x ? 1.0f : 0.5f);
			
			if (reward != 0.5f) {
				lastReward = reward = reward > 0.5f ? 1.0f : 0.0f;
				rewardTimer = 0.0f;
			}
			else if (rewardTimer < rewardTime) {
				reward = lastReward;

				rewardTimer += dt;
			}

			if (redBounced)
				reward = std::max(reward, 0.55f);

			reward = (reward * 2.0f - 1.0f) * 0.1f - std::abs(ball._position.x - red._position.x) * 0.008f;

			//reward *= 0.05f;

			//reward = (scoreRed - prevScoreRed) - (scoreBlue - prevScoreBlue) - std::abs(ball._position.x - red._position.x) * 0.001f;

			//std::cout << "Reward: " << reward << std::endl;

			//agentRed.update(reward, generator);
			agentRed.update(reward, generator);
			//std::vector<float> action(2);
			//agentRed.step(inputs, action, reward, 0.01f, 0.995f, 0.99f, 10.0f, 32, 6, 0.05f, 0.04f, 0.04f, 800, 200, 0.003f, 0.0f, generator);

			//float move = action[0];
			//bool jump = action[1] > 0.0f;

			float move = agentRed.getActionFromNodeIndex(0) * 2.0f - 1.0f;
			bool jump = agentRed.getActionFromNodeIndex(1) > 0.5f;

			red._velocity.y += gravity * dt;
			red._velocity.x += -slimeMoveDeccel * red._velocity.x * dt;
			red._position += red._velocity * dt;

			{
				red._velocity.x += move * slimeMoveAccel * dt;

				if (red._velocity.x < -maxSlimeSpeed)
					red._velocity.x = -maxSlimeSpeed;
				else if (red._velocity.x > maxSlimeSpeed)
					red._velocity.x = maxSlimeSpeed;
			}
			
			if (red._position.y > fieldCenter.y) {
				red._velocity.y = 0.0f;
				red._position.y = fieldCenter.y;

				if (jump)
					red._velocity.y -= slimeJump;
			}

			if (red._position.x + slimeRadius > fieldCenter.x + fieldRadius) {
				red._velocity.x = 0.0f;
				red._position.x = fieldCenter.x + fieldRadius - slimeRadius;
			}

			if (red._position.x - slimeRadius < wallCenter.x + wallRadius) {
				red._velocity.x = 0.0f;
				red._position.x = wallCenter.x + wallRadius + slimeRadius;
			}
		}

		prevScoreRed = scoreRed;
		prevScoreBlue = scoreBlue;
		prevBallX = ball._position.x;

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::K) && !prevPressK) {
			noRender = !noRender;
		}

		prevPressK = sf::Keyboard::isKeyPressed(sf::Keyboard::K);

		if (noRender)
			continue;

		// --------------------------------- Rendering ---------------------------------

		renderWindow.clear();

		{
			sf::Sprite s;
			s.setTexture(backgroundTexture);
			s.setOrigin(backgroundTexture.getSize().x * 0.5f, backgroundTexture.getSize().y * 0.5f);
			s.setPosition(renderWindow.getSize().x * 0.5f, renderWindow.getSize().y * 0.5f);

			renderWindow.draw(s);
		}

		{
			sf::Sprite s;
			s.setTexture(blueSlimeTexture);
			s.setOrigin(blueSlimeTexture.getSize().x * 0.5f, blueSlimeTexture.getSize().y);
			s.setPosition(blue._position);

			renderWindow.draw(s);
		}

		{
			sf::Sprite s;
			s.setTexture(eyeTexture);
			s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
			s.setPosition(blue._position + sf::Vector2f(50.0f, -28.0f));

			sf::Vector2f delta = ball._position - s.getPosition();

			float angle = std::atan2(delta.y, delta.x);

			s.setRotation(angle * 180.0f / 3.141596f);

			renderWindow.draw(s);
		}

		{
			sf::Sprite s;
			s.setTexture(redSlimeTexture);
			s.setOrigin(redSlimeTexture.getSize().x * 0.5f, redSlimeTexture.getSize().y);
			s.setPosition(red._position);

			renderWindow.draw(s);
		}

		{
			sf::Sprite s;
			s.setTexture(eyeTexture);
			s.setOrigin(eyeTexture.getSize().x * 0.5f, eyeTexture.getSize().y * 0.5f);
			s.setPosition(red._position + sf::Vector2f(-50.0f, -28.0f));

			sf::Vector2f delta = ball._position - s.getPosition();

			float angle = std::atan2(delta.y, delta.x);

			s.setRotation(angle * 180.0f / 3.141596f);

			renderWindow.draw(s);
		}

		{
			sf::Sprite s;
			s.setTexture(ballTexture);
			s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
			s.setPosition(ball._position);

			renderWindow.draw(s);
		}

		if (ball._position.y + ballRadius < 0.0f) {
			sf::Sprite s;
			s.setTexture(arrowTexture);
			s.setOrigin(arrowTexture.getSize().x * 0.5f, 0.0f);
			s.setPosition(ball._position.x, 0.0f);

			renderWindow.draw(s);
		}

		{
			sf::Text scoreText;
			scoreText.setFont(scoreFont);
			scoreText.setString(std::to_string(scoreBlue));
			scoreText.setCharacterSize(100);

			float width = scoreText.getLocalBounds().width;

			scoreText.setPosition(fieldCenter.x - width * 0.5f - 100.0f, 10.0f);
			
			scoreText.setColor(sf::Color(100, 133, 255));

			renderWindow.draw(scoreText);
		}

		{
			sf::Text scoreText;
			scoreText.setFont(scoreFont);
			scoreText.setString(std::to_string(scoreRed));
			scoreText.setCharacterSize(100);

			float width = scoreText.getLocalBounds().width;

			scoreText.setPosition(fieldCenter.x - width * 0.5f + 100.0f, 10.0f);

			scoreText.setColor(sf::Color(255, 100, 100));
			
			renderWindow.draw(scoreText);
		}

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::P)) {
			float scale = 4.0f;

			float alignment = 0.0f;

			/*for (int l = 0; l < agentBlue.getHTSL().getLayers().size(); l++) {
				sf::Image sdr;
				sdr.create(agentBlue.getHTSL().getLayerDescs()[l]._width, agentBlue.getHTSL().getLayerDescs()[l]._height);

				for (int x = 0; x < agentBlue.getHTSL().getLayerDescs()[l]._width; x++)
					for (int y = 0; y < agentBlue.getHTSL().getLayerDescs()[l]._height; y++) {
						sf::Color c;
						c.r = c.g = c.b = agentBlue.getHTSL().getLayers()[l]._rsc.getHiddenState(x, y) * 255.0f;

						sdr.setPixel(x, y, c);
					}

				sf::Texture sdrt;
				sdrt.loadFromImage(sdr);

				sf::Sprite sdrs;
				sdrs.setTexture(sdrt);
				sdrs.setPosition(alignment, renderWindow.getSize().y - sdr.getSize().y * scale);
				sdrs.setScale(scale, scale);

				renderWindow.draw(sdrs);

				alignment += scale * sdr.getSize().x;
			}*/

			alignment = 0.0f;

			for (int l = 0; l < agentRed.getHTSL().getLayers().size(); l++) {
				sf::Image sdr;
				sdr.create(agentRed.getHTSL().getLayerDescs()[l]._width, agentRed.getHTSL().getLayerDescs()[l]._height);

				for (int x = 0; x < agentRed.getHTSL().getLayerDescs()[l]._width; x++)
					for (int y = 0; y < agentRed.getHTSL().getLayerDescs()[l]._height; y++) {
						sf::Color c;
						c.r = c.g = c.b = agentRed.getHTSL().getLayers()[l]._rsc.getHiddenState(x, y) * 255.0f;

						sdr.setPixel(x, y, c);
					}

				sf::Texture sdrt;
				sdrt.loadFromImage(sdr);

				alignment += scale * sdr.getSize().x;

				sf::Sprite sdrs;
				sdrs.setTexture(sdrt);
				sdrs.setPosition(renderWindow.getSize().x - alignment, renderWindow.getSize().y - sdr.getSize().y * scale);
				sdrs.setScale(scale, scale);

				renderWindow.draw(sdrs);	
			}
		}

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::T)) {
			sf::Vector2f position;

			const float scalar = 1.0f / 0.001f;

			position.x = red._position.x + scalar * agentRed.getHTSL().getPrediction(0);
			position.y = red._position.y + scalar * agentRed.getHTSL().getPrediction(1);

			sf::Sprite s;
			s.setTexture(ballTexture);
			s.setOrigin(ballTexture.getSize().x * 0.5f, ballTexture.getSize().y * 0.5f);
			s.setPosition(position);

			renderWindow.draw(s);
		}

		if (sf::Keyboard::isKeyPressed(sf::Keyboard::F)) {
			for (int i = 0; i < 16; i++)
				std::cout << agentRed.getHTSL().getPrediction(i) << std::endl;

			std::cout << std::endl;
		}
		
		renderWindow.display();
	} while (!quit);

	return 0;
}