/* Free memory when neuron is destroyed */
FANN_EXTERNAL void FANN_API fann_neuron_destructor_fully_recurrent(struct fann_neuron* neuron)
{
	assert(neuron != NULL);

	fann_safe_free(neuron->weights);
	fann_safe_free(neuron->weights_deltas);
	fann_safe_free(neuron->sums);
}
Exemple #2
0
/*
 * deallocate the train data structure. 
 */
FANN_EXTERNAL void FANN_API fann_destroy_train(struct fann_train_data *data)
{
	if(data == NULL)
		return;
	if(data->input != NULL)
		fann_safe_free(data->input[0]);
	if(data->output != NULL)
		fann_safe_free(data->output[0]);
	fann_safe_free(data->input);
	fann_safe_free(data->output);
	fann_safe_free(data);
}
Exemple #3
0
FANN_EXTERNAL void FANN_API fann_sparse_neuron_destructor(struct fann_neuron* neuron)
{
	struct fann_neuron_private_data_connected_any_any *priv = (struct fann_neuron_private_data_connected_any_any *) neuron->private_data;
	fann_safe_free(neuron->weights);
	fann_safe_free(neuron->weights_deltas);
	fann_safe_free(neuron->sums);
	fann_safe_free(neuron->weights_deltas);
	fann_safe_free(priv->prev_weights_deltas);
	fann_safe_free(priv->prev_steps);
	fann_safe_free( ((struct fann_sparse_neuron_private_data*) neuron->private_data)->mask );
	fann_safe_free( ((struct fann_sparse_neuron_private_data*) neuron->private_data)->generic );
	fann_safe_free(neuron->private_data);

}
/* Deallocate a layer */
FANN_EXTERNAL void FANN_API fann_layer_destructor_fully_recurrent(struct fann_layer* layer)
{
	struct fann_neuron *neuron_it;

	assert(layer != NULL);
	assert(layer->first_neuron != NULL);
	assert(layer->last_neuron  != NULL);

	/* Deallocate memory for each neuron in this layer */
	for (neuron_it=layer->first_neuron; neuron_it!=layer->last_neuron; neuron_it++)
	{
		if (neuron_it->destructor != NULL)
		{
			neuron_it->destructor(neuron_it);
		}
	}

	fann_safe_free(layer->first_neuron);
	fann_safe_free(layer->train_errors);
	fann_safe_free(layer->outputs);
}
/* Runs a single layer for one iteration */
FANN_EXTERNAL void FANN_API fann_layer_run_fully_recurrent(
	struct fann *ann, struct fann_layer* layer)
{
	struct fann_neuron * last_neuron = layer->last_neuron;
	struct fann_neuron * neuron_it;
	unsigned int neuron_num = 0;
	unsigned int i = 0;
	
	fann_type sum      = (fann_type)0.0;
	fann_type *outputs = NULL;

	if ((outputs = calloc(ann->num_output, sizeof(fann_type))) == NULL)
	{
		printf("Run Layer: 'outputs' allocation failed!\n");
		return;
	}


	/* Run each neuron in the layer a single iteration */
	for(neuron_it = layer->first_neuron; neuron_it != last_neuron; neuron_it++)
	{
		/* Find the sum of weights*inputs*/
		sum = (fann_type)0.0;
		for (i=0; i<neuron_it->num_weights; i++)
		{
			if (i < ann->num_input + 1)
			{
				neuron_it->sums[i] = neuron_it->weights[i] * layer->inputs[i];
			}
			else
			{
				neuron_it->sums[i] = neuron_it->weights[i] * layer->outputs[i - ann->num_input - 1];
			}

			sum += neuron_it->sums[i];
		}

		/* FIXME: Support alternate functions other than sigmoid!*/
		outputs[neuron_num] = 1.0f / (1.0f + exp(-sum));
		neuron_num++;
	}

	/* Copy the new values over to the ANN*/
	for (i=0; i<ann->num_output; i++)
	{
		ann->output[i] = outputs[i];
	}
	
	fann_safe_free(outputs);
}
Exemple #6
0
/**************************************************
 REAL-TIME RECURRENT LEARNING

 Williams and Zipser, "A Learning Algorithm for
   Continually Running Fully Recurrent Neural
   Networks," Neural Computation, 1. (1989)

 NOTE: This function is still being debugged.
       MSE does not decrease properly.
 *************************************************/
FANN_EXTERNAL void FANN_API fann_train_rtrl(struct fann *ann, struct fann_train_data *pattern, 
											float max_MSE, unsigned int max_iters, float rate)
{
	struct fann_neuron *neuron = NULL;
	struct fann_layer *layer = NULL;
	fann_type *curr_outputs = NULL;
	fann_type *curr_weight = NULL;

	unsigned int num_neurons = 0;
	unsigned int curr_neuron = 0;
	unsigned int num_iters = 0;
	unsigned int i = 0, j = 0, l = 0;

	float *dodw = NULL;				/* deriv of output wrt weight*/
	float *curr_dodw = NULL;
	float *next_dodw = NULL;		/* dodw for time 'n+1'*/
	float *curr_next_dodw = NULL;
	float *start_dodw = NULL;
	float *temp_swap = NULL;		/* for swapping dodw pointers*/
	float dw = 0.0;					/* change in weight*/

	assert(ann != NULL);
	assert(pattern != NULL);

	/* Only one MIMO neuron and layer in recurrent nets*/
	layer  = ann->first_layer;
	neuron = layer->first_neuron;

	memset(layer->outputs, 0, num_neurons * sizeof(fann_type));

	/* Allocate memory for new outputs*/
	/* TODO: Return an error*/
	num_neurons = layer->num_outputs;
	if ((curr_outputs = calloc(num_neurons, sizeof(fann_type))) == NULL)
	{
		/*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/
		printf("RTRL: Could not allocate 'curr_outputs'\n");
		return;
	}

	/* Allocate memory for derivatives do_k(t)/dw_i,j*/
	/* TODO: Return an error*/
	if ((dodw = calloc(ann->num_output * neuron->num_weights * neuron->num_weights, sizeof(float))) == NULL)
	{
		/*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/
		printf("RTRL: Could not allocate 'dodw'\n");
		return;
	}

	/* Allocate memory for derivatives do_k(t)/dw_i,j*/
	/* TODO: Return an error*/
	if ((next_dodw = calloc(neuron->num_weights * num_neurons, sizeof(float))) == NULL)
	{
		/*fann_error((struct fann_error *) orig, FANN_E_CANT_ALLOCATE_MEM);*/
		printf("RTRL: Could not allocate 'next_dodw'\n");
		return;
	}

	/* Randomize weights, initialize for training*/
	fann_randomize_weights(ann, -0.5, 0.5);

	if (layer->train_errors==NULL)
	{
		layer->initialize_train_errors(ann, ann->first_layer);
	}

	/* RTRL: Continue learning until MSE low enough or reach*/
	/*       max iterations*/
	num_iters = 0;
	ann->training_params->MSE_value = 100;
	while (ann->training_params->MSE_value > max_MSE && num_iters <= max_iters)
	{
		/* Set the input lines for this time step*/
		/*printf("%d inputs: ", ann->num_input);*/
		for (i=0; i<ann->num_input; i++)
		{
			ann->inputs[i] = pattern->input[num_iters][i];
			printf("%f ", (double) ann->inputs[i]);
		}
		/*printf("(output: %f) (bias: %f) \n", pattern->output[num_iters][0], ann->inputs[ann->num_input]);*/

		/* Copy the outputs of each neuron before they're updated*/
		memcpy(curr_outputs, layer->outputs, num_neurons * sizeof(fann_type));


		/* Update the output of all nodes*/
		layer->run(ann, layer);
		/*printf("NEW OUTPUTS: %f %f %f\n", layer->outputs[0], layer->outputs[1], layer->outputs[2]);*/
		/*printf("ANN OUTPUTS: %f\n", ann->output[0]);*/

		/*curr_weight = neuron->weights;
		for (i=0; i<num_neurons; i++)
		{
			for (j=0; j<layer->num_inputs + num_neurons; j++)
			{
				printf("weight_prev (%d,%d): %f ", i, j, *curr_weight);
				curr_weight++;
			}
		}
		printf("\n");*/

		/* Compute new MSE*/
		fann_reset_MSE(ann);
		fann_compute_MSE(ann, pattern->output[num_iters]);
		printf("%d MSE: %f\n", num_iters, fann_get_MSE(ann));

		/* Modify the weights*/
		start_dodw  = dodw + (num_neurons - ann->num_output) * neuron->num_weights;
		for (i=0; i<num_neurons; i++)
		{
			curr_weight = neuron[i].weights;
			for (j=0; j<layer->num_inputs + num_neurons; j++)
			{
				dw = 0.0;
				curr_dodw = start_dodw;
				/* For each neuron in which is not an input node*/
				for (curr_neuron=num_neurons - ann->num_output; curr_neuron<num_neurons; curr_neuron++)
				{
					dw += (pattern->output[num_iters][curr_neuron - (num_neurons - ann->num_output)] -
						curr_outputs[curr_neuron]) * *curr_dodw;

					curr_dodw += neuron->num_weights;
				}

				*curr_weight += dw * rate;
				/*printf("weight (%d,%d): %f\n", i, j, *curr_weight);*/

				curr_weight++;
				start_dodw++;
			}
		}

		/* Compute next dodw derivatives*/
		curr_next_dodw = next_dodw;
		for (curr_neuron=0; curr_neuron<num_neurons; curr_neuron++)
		{
			start_dodw = dodw;
			curr_weight = neuron->weights;
			for (i=0; i<num_neurons; i++)
			{
				for (j=0; j<layer->num_inputs + num_neurons; j++)
				{
					curr_dodw = start_dodw;

					*curr_next_dodw = 0.0;
					for (l=0; l<num_neurons; l++)
					{
						*curr_next_dodw += *curr_dodw *
							neuron->weights[curr_neuron * (layer->num_inputs + num_neurons) + l + layer->num_inputs];
						curr_dodw += neuron->num_weights;
					}

					/* kronecker_{i,k} * z_j(t)*/
					*curr_next_dodw += (i != curr_neuron) ? 0 :
						((j < layer->num_inputs) ? ann->inputs[j] : curr_outputs[j - layer->num_inputs]);

					*curr_next_dodw *= layer->outputs[curr_neuron]*(1 - layer->outputs[curr_neuron]);
					/*printf("(%d,%d): %f\n", i, j, *curr_next_dodw);*/

					curr_next_dodw++;
					curr_weight++;
					start_dodw++;
				}
			}
		}

		/* Swap the next and the current dodw*/
		/*  (to avoid a costly memory transfer)*/
		temp_swap = dodw;
		dodw = next_dodw;
		next_dodw = temp_swap;

		num_iters++;
	}

	fann_safe_free(dodw);
	fann_safe_free(curr_outputs);
}
Exemple #7
0
FANN_EXTERNAL fann_type *FANN_API fann_run_hopfield(struct fann *ann, fann_type *input)
{
	struct fann_neuron *neuron = NULL;
	unsigned int num_neurons = 0;
	unsigned int rand_neuron = 0;
	unsigned int i = 0;
	unsigned int neuron_array_size = 0;
	unsigned int iters = 0;
	int statediff = 0;
	
	fann_type sum = 0;
	fann_type *old_output = NULL;
	fann_type *weights = NULL;

	assert(ann != NULL);
	assert(input != NULL);

	neuron = ann->first_layer->first_neuron;
	num_neurons = ann->first_layer->num_outputs;

	/* Initialization*/
	for (i=0; i<num_neurons; i++)
	{
		ann->output[i] = input[i];
	}

	neuron_array_size = num_neurons * sizeof(fann_type);
	old_output = (fann_type *)malloc(neuron_array_size);

	/* Iterate until states unchanged*/
	/* FIXME: The number of iterations is currently */
	/*   somewhat arbitrary (10*num_neurons once appears*/
	/*   stable). Having a better measure of whether*/
	/*   the network is stable would be nice.*/
	do
	{

		/* Asynchronously update the neurons*/
		rand_neuron = floor(rand()%num_neurons);
		/*printf("Iters: %d (rand = %d)\n", iters, rand_neuron);*/
		weights = neuron[rand_neuron].weights;
		memcpy(old_output, ann->output, neuron_array_size);

		/* Compute the new output vector*/
		sum = 0;
		for (i=0; i<num_neurons; i++)
		{
			sum += weights[i] * ann->output[i];	
		}

		ann->output[rand_neuron] = (sum >= 0) ? 1 : -1;


		/* Compare the old output vector to the new output vector*/
		if ((statediff = memcmp(old_output, ann->output, neuron_array_size)) != 0)
		{
			iters = 0;
		}

		iters += (statediff == 0) ? 1 : 0;

	} while (iters < 10*num_neurons);

	fann_safe_free(old_output);

	/* FIXME: Is it OK to return an internal fp? */
	return ann->output;
}
Exemple #8
0
/* Creates a feedforward, layered net which is an "unrolled" recurrent network.
 For example, the recurrent net:
  A <-> B <-> C<- (where C autosynapses)
 Becomes (unrolled two time steps):
  A  B  C    input layer
   \/ \/|
  A  B  C    hidden layer I
   \/ \/|
  A  B  C    output layer
*/
FANN_EXTERNAL struct fann *FANN_API fann_create_unrolled_recurrent(
	unsigned int num_neurons, fann_type *weights, unsigned int time_steps)
{
	struct fann *ann        = NULL;
	unsigned int *layers    = NULL;
	unsigned int num_layers = time_steps + 1;
	unsigned int layern     = 0;

	struct fann_layer *curr_layer   = NULL;
	struct fann_neuron *curr_neuron = NULL;
	fann_type *curr_weights         = weights;

	
	/*************************************
	  CREATE THE FEEDFORWARD STRUCTURE 
	 *************************************/

	/* Allocate number of neurons per layer array */
	layers = (unsigned int *)calloc(num_layers, sizeof(unsigned int));
	if (layers == NULL)
	{
		return NULL;
	}

	/* Populate each layer with the number of neurons */
	for (layern=0; layern < num_layers; layern++)
	{
		layers[layern] = num_neurons;
	}

	/* Create the feedforward network */
	ann = fann_create_standard_array(num_layers, layers);
	fann_safe_free(layers);

	/*printf("REQUESTED: LAYERS=%d, NEURONS/LAYER=%d\n", num_layers, num_neurons);
	printf("NUM LAYERS: %d\n", ann->last_layer - ann->first_layer);
	printf("IN: %d, NEURONS: %d, OUTPUT: %d\n",
		ann->num_input, ann->num_neurons, ann->num_output);*/


	/*************************************
	  SET THE FEEDFORWARD WEIGHTS
	 *************************************/

	/* Visit each layer */
    for (curr_layer = ann->first_layer; 
		curr_layer != ann->last_layer; 
		curr_layer++)
	{
		/* The weights are the same for each feedforward layer! */
		curr_weights = weights;

		/* Copy the weight matrix into the neurons, 
		   one row per neuron */
		for (curr_neuron = curr_layer->first_neuron; 
			curr_neuron != curr_layer->last_neuron; 
			curr_neuron++)
		{
            memcpy(curr_neuron->weights, curr_weights, num_neurons * num_neurons * sizeof(fann_type));

			curr_weights += num_neurons;
		}
	}

	return ann;
}
Exemple #9
0
FANN_EXTERNAL void FANN_API fann_destroy(struct fann *ann)
{
	if(ann == NULL)
		return;
	fann_safe_free(ann->weights);
	fann_safe_free(ann->connections);
	fann_safe_free(ann->first_layer->first_neuron);
	//AK
	fann_safe_free(ann->first_layer->sum);
	fann_safe_free(ann->first_layer->value);
	//
	fann_safe_free(ann->first_layer);
	fann_safe_free(ann->output);
	fann_safe_free(ann->train_errors);
	fann_safe_free(ann->train_slopes);
	fann_safe_free(ann->prev_train_slopes);
	fann_safe_free(ann->prev_steps);
	fann_safe_free(ann->prev_weights_deltas);
	fann_safe_free(ann->errstr);
	
	fann_safe_free( ann->scale_mean_in );
	fann_safe_free( ann->scale_deviation_in );
	fann_safe_free( ann->scale_new_min_in );
	fann_safe_free( ann->scale_factor_in );

	fann_safe_free( ann->scale_mean_out );
	fann_safe_free( ann->scale_deviation_out );
	fann_safe_free( ann->scale_new_min_out );
	fann_safe_free( ann->scale_factor_out );
	
	fann_safe_free(ann);
}
FANN_EXTERNAL void FANN_API fann_destroy(struct fann *ann)
{
	if(ann == NULL)
		return;
	fann_safe_free(ann->weights);
	fann_safe_free(ann->connections);
	fann_safe_free(ann->first_layer->first_neuron);
	fann_safe_free(ann->first_layer);
	fann_safe_free(ann->output);
	fann_safe_free(ann->train_errors);
	fann_safe_free(ann->train_slopes);
	fann_safe_free(ann->prev_train_slopes);
	fann_safe_free(ann->prev_steps);
	fann_safe_free(ann->prev_weights_deltas);
	fann_safe_free(ann->errstr);
	fann_safe_free(ann->cascade_activation_functions);
	fann_safe_free(ann->cascade_activation_steepnesses);
	
#ifndef FIXEDFANN
	fann_safe_free( ann->scale_mean_in );
	fann_safe_free( ann->scale_deviation_in );
	fann_safe_free( ann->scale_new_min_in );
	fann_safe_free( ann->scale_factor_in );

	fann_safe_free( ann->scale_mean_out );
	fann_safe_free( ann->scale_deviation_out );
	fann_safe_free( ann->scale_new_min_out );
	fann_safe_free( ann->scale_factor_out );
#endif
	
	fann_safe_free(ann);
}
/* INTERNAL FUNCTION
   Allocates the main structure and sets some default values.
 */
struct fann *fann_allocate_structure(unsigned int num_layers)
{
	struct fann *ann;

	if(num_layers < 2)
	{
#ifdef DEBUG
		printf("less than 2 layers - ABORTING.\n");
#endif
		return NULL;
	}

	/* allocate and initialize the main network structure */
	ann = (struct fann *) malloc(sizeof(struct fann));
	if(ann == NULL)
	{
		fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
		return NULL;
	}

	ann->errno_f = FANN_E_NO_ERROR;
	ann->error_log = fann_default_error_log;
	ann->errstr = NULL;
	ann->learning_rate = 0.7f;
	ann->learning_momentum = 0.0;
	ann->total_neurons = 0;
	ann->total_connections = 0;
	ann->num_input = 0;
	ann->num_output = 0;
	ann->train_errors = NULL;
	ann->train_slopes = NULL;
	ann->prev_steps = NULL;
	ann->prev_train_slopes = NULL;
	ann->prev_weights_deltas = NULL;
	ann->training_algorithm = FANN_TRAIN_RPROP;
	ann->num_MSE = 0;
	ann->MSE_value = 0;
	ann->num_bit_fail = 0;
	ann->bit_fail_limit = (fann_type)0.35;
	ann->network_type = FANN_NETTYPE_LAYER;
	ann->train_error_function = FANN_ERRORFUNC_TANH;
	ann->train_stop_function = FANN_STOPFUNC_MSE;
	ann->callback = NULL;
    ann->user_data = NULL; /* User is responsible for deallocation */
	ann->weights = NULL;
	ann->connections = NULL;
	ann->output = NULL;
#ifndef FIXEDFANN
	ann->scale_mean_in = NULL;
	ann->scale_deviation_in = NULL;
	ann->scale_new_min_in = NULL;
	ann->scale_factor_in = NULL;
	ann->scale_mean_out = NULL;
	ann->scale_deviation_out = NULL;
	ann->scale_new_min_out = NULL;
	ann->scale_factor_out = NULL;
#endif	
	
	/* variables used for cascade correlation (reasonable defaults) */
	ann->cascade_output_change_fraction = 0.01f;
	ann->cascade_candidate_change_fraction = 0.01f;
	ann->cascade_output_stagnation_epochs = 12;
	ann->cascade_candidate_stagnation_epochs = 12;
	ann->cascade_num_candidate_groups = 2;
	ann->cascade_weight_multiplier = (fann_type)0.4;
	ann->cascade_candidate_limit = (fann_type)1000.0;
	ann->cascade_max_out_epochs = 150;
	ann->cascade_max_cand_epochs = 150;
	ann->cascade_candidate_scores = NULL;
	ann->cascade_activation_functions_count = 10;
	ann->cascade_activation_functions = 
		(enum fann_activationfunc_enum *)calloc(ann->cascade_activation_functions_count, 
							   sizeof(enum fann_activationfunc_enum));
	if(ann->cascade_activation_functions == NULL)
	{
		fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
		free(ann);
		return NULL;
	}
							   
	ann->cascade_activation_functions[0] = FANN_SIGMOID;
	ann->cascade_activation_functions[1] = FANN_SIGMOID_SYMMETRIC;
	ann->cascade_activation_functions[2] = FANN_GAUSSIAN;
	ann->cascade_activation_functions[3] = FANN_GAUSSIAN_SYMMETRIC;
	ann->cascade_activation_functions[4] = FANN_ELLIOT;
	ann->cascade_activation_functions[5] = FANN_ELLIOT_SYMMETRIC;
	ann->cascade_activation_functions[6] = FANN_SIN_SYMMETRIC;
	ann->cascade_activation_functions[7] = FANN_COS_SYMMETRIC;
	ann->cascade_activation_functions[8] = FANN_SIN;
	ann->cascade_activation_functions[9] = FANN_COS;

	ann->cascade_activation_steepnesses_count = 4;
	ann->cascade_activation_steepnesses = 
		(fann_type *)calloc(ann->cascade_activation_steepnesses_count, 
							   sizeof(fann_type));
	if(ann->cascade_activation_steepnesses == NULL)
	{
		fann_safe_free(ann->cascade_activation_functions);
		fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
		free(ann);
		return NULL;
	}
	
	ann->cascade_activation_steepnesses[0] = (fann_type)0.25;
	ann->cascade_activation_steepnesses[1] = (fann_type)0.5;
	ann->cascade_activation_steepnesses[2] = (fann_type)0.75;
	ann->cascade_activation_steepnesses[3] = (fann_type)1.0;

	/* Variables for use with with Quickprop training (reasonable defaults) */
	ann->quickprop_decay = (float) -0.0001;
	ann->quickprop_mu = 1.75;

	/* Variables for use with with RPROP training (reasonable defaults) */
	ann->rprop_increase_factor = (float) 1.2;
	ann->rprop_decrease_factor = 0.5;
	ann->rprop_delta_min = 0.0;
	ann->rprop_delta_max = 50.0;
	ann->rprop_delta_zero = 0.1f;
	
	fann_init_error_data((struct fann_error *) ann);

#ifdef FIXEDFANN
	/* these values are only boring defaults, and should really
	 * never be used, since the real values are always loaded from a file. */
	ann->decimal_point = 8;
	ann->multiplier = 256;
#endif

	/* allocate room for the layers */
	ann->first_layer = (struct fann_layer *) calloc(num_layers, sizeof(struct fann_layer));
	if(ann->first_layer == NULL)
	{
		fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
		free(ann);
		return NULL;
	}

	ann->last_layer = ann->first_layer + num_layers;

	return ann;
}