Esempio n. 1
0
void initialize_candidate_weights(struct fann *ann, unsigned int first_con, unsigned int last_con, float scale_factor)
{
	fann_type prev_step;
	unsigned int i = 0;
	unsigned int bias_weight = (unsigned int)(first_con + (ann->first_layer->last_neuron - ann->first_layer->first_neuron) - 1);

	if(ann->training_algorithm == FANN_TRAIN_RPROP)
		prev_step = ann->rprop_delta_zero;
	else
		prev_step = 0;

	for(i = first_con; i < last_con; i++)
	{
		if(i == bias_weight) 
			ann->weights[i] = fann_rand(-scale_factor, scale_factor);
		else
			ann->weights[i] = fann_rand(0,scale_factor);
					
		ann->train_slopes[i] = 0;
		ann->prev_steps[i] = prev_step;
		ann->prev_train_slopes[i] = 0;
	}
}
Esempio n. 2
0
FANN_EXTERNAL void FANN_API fann_randomize_weights(struct fann *ann, fann_type min_weight,
												   fann_type max_weight)
{
	fann_type *last_weight;
	fann_type *weights = ann->weights;

	last_weight = weights + ann->total_connections;
	for(; weights != last_weight; weights++)
	{
		*weights = (fann_type) (fann_rand(min_weight, max_weight));
	}

#ifndef FIXEDFANN
	if(ann->prev_train_slopes != NULL)
	{
		fann_clear_train_arrays(ann);
	}
#endif
}
Esempio n. 3
0
/* Allocates room inside the neuron for the connections.
 * Creates a fully connected neuron
 */
FANN_EXTERNAL int FANN_API fann_sparse_neuron_constructor(struct fann *ann, struct fann_layer *layer, 
		struct fann_neuron *neuron, struct fann_neuron_descr * descr)
{
	unsigned int i, j;
	unsigned int min_connections, max_connections, num_connections;
	unsigned int connections_per_output;
	float connection_rate = * ((float* )descr->private_data);
	struct fann_sparse_neuron_private_data* private_data;
	struct fann_neuron_private_data_connected_any_any* generic_private_data;
	fann_type *mask, *weights;
	struct dice *dices;

#ifdef FIXEDFANN
	fann_type multiplier = ann->fixed_params->multiplier;
	neuron->activation_steepness = ann->fixed_params->multiplier / 2;
#else
	neuron->activation_steepness = 0.5;
#endif


	connection_rate = connection_rate > 1.0f ? 1.0f : connection_rate;
	
	neuron->activation_function = FANN_SIGMOID_STEPWISE;


	neuron->num_outputs=descr->num_outputs;
	neuron->inputs=layer->inputs;
	neuron->num_inputs=layer->num_inputs;

	/* set the error array to null (lazy allocation) */
	neuron->train_errors=NULL;
	
	/* this is the number of actually allocated weights (some are unused) */
	neuron->num_weights=neuron->num_outputs*neuron->num_inputs;
	
	/* allocate the weights even for unused connections */
	if ( (weights = neuron->weights = (fann_type*) calloc(neuron->num_weights, sizeof(fann_type))) == NULL)
		return 1;
	
	/* allocate space for the dot products results */
	if ( (neuron->sums = (fann_type*) malloc(neuron->num_outputs*sizeof(fann_type))) == NULL)
		return 1;

	/* allocate private data */
	if ( (private_data = neuron->private_data = (struct fann_sparse_neuron_private_data*) malloc(sizeof(struct fann_sparse_neuron_private_data))) == NULL)
		return 1;
	/* private data stores the connection mask, allocate it */
	if ( (mask = private_data->mask = (fann_type*) calloc(neuron->num_weights, sizeof(fann_type))) == NULL)
		return 1;
	if ( (generic_private_data = private_data->generic = (struct fann_neuron_private_data_connected_any_any*) malloc (sizeof(struct fann_neuron_private_data_connected_any_any))) == NULL)
		return 1;
	generic_private_data->prev_steps=NULL;
	generic_private_data->prev_weights_deltas=NULL;

	/* alocate a set of dices to select rows */
	if ( (dices = (struct dice*) malloc(neuron->num_inputs*sizeof(struct dice))) == NULL)
		return 1;
	
	for (i=0; i<neuron->num_inputs; i++)
	{
		dices[i].idx=i;
		dices[i].value=0;
	}
	
	min_connections = fann_max(neuron->num_inputs, neuron->num_outputs);
	max_connections = neuron->num_inputs * neuron->num_outputs;
	num_connections = fann_max(min_connections,
			(unsigned int) (0.5 + (connection_rate * max_connections)));

	connections_per_output = num_connections / neuron->num_outputs;

	/* Dice throw simulation: a float value is assigned to each input.  
	 * The value decimal component is chosen randomly between 0 and 0.4 ("dice throw").
	 * The integer components is equal to the number of output neurons already
	 * connected to this input.
	 * For each output neuron ecah input gets a new "dice throw". Then the inputs and are 
	 * sorted in ascending order according to the value.
	 * The first ones in the array had less output neurons attached to the 
	 * and better luck in "dice thow". This ones are selected and theyr value is incremented.
	 */
	for (i=0; i<neuron->num_outputs; i++)
	{
		/* throw one dice per input */
		for (j=0; j<neuron->num_inputs; j++)
			dices[j].value= ((int)dices[j].value) + fann_rand(0, 0.4);

		/* sort: smaller (dice value + num_connections) wins) */
		qsort((void*) dices, neuron->num_inputs, sizeof(struct dice), dice_sorter);

		/* assign connections to the output to the winner inputs */
		for (j=0; j<connections_per_output; j++)
		{
			dices[j].value+=1;
			mask[dices[j].idx] = (fann_type) 1.0f;
			weights[dices[j].idx] = (fann_type) fann_random_weight();
		}
		weights += neuron->num_inputs;
	}
	free(dices);

	/* set the function pointers */
	neuron->destructor = fann_sparse_neuron_destructor;
	neuron->run = fann_sparse_neuron_run;
	neuron->backpropagate = fann_sparse_neuron_backprop;
	neuron->update_weights = fann_sparse_neuron_update;
	neuron->compute_error = fann_sparse_neuron_compute_MSE;
	
	return 0;
}
Esempio n. 4
0
/* Initialize the weights using Widrow + Nguyen's algorithm.
*/
FANN_EXTERNAL void FANN_API fann_init_weights(struct fann *ann, struct fann_train_data *train_data)
{
	fann_type smallest_inp, largest_inp;
	unsigned int dat = 0, elem, num_connect, num_hidden_neurons;
	struct fann_layer *layer_it;
	struct fann_neuron *neuron_it, *last_neuron, *bias_neuron;

#ifdef FIXEDFANN
	unsigned int multiplier = ann->multiplier;
#endif
	float scale_factor;

	for(smallest_inp = largest_inp = train_data->input[0][0]; dat < train_data->num_data; dat++)
	{
		for(elem = 0; elem < train_data->num_input; elem++)
		{
			if(train_data->input[dat][elem] < smallest_inp)
				smallest_inp = train_data->input[dat][elem];
			if(train_data->input[dat][elem] > largest_inp)
				largest_inp = train_data->input[dat][elem];
		}
	}

	num_hidden_neurons =
		ann->total_neurons - (ann->num_input + ann->num_output +
							  (unsigned int)(ann->last_layer - ann->first_layer));
	scale_factor =
		(float) (pow
				 ((double) (0.7f * (double) num_hidden_neurons),
				  (double) (1.0f / (double) ann->num_input)) / (double) (largest_inp -
																		 smallest_inp));

#ifdef DEBUG
	printf("Initializing weights with scale factor %f\n", scale_factor);
#endif
	bias_neuron = ann->first_layer->last_neuron - 1;
	for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
	{
		last_neuron = layer_it->last_neuron;
		bias_neuron = (layer_it - 1)->last_neuron - 1;

		for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
		{
			for(num_connect = neuron_it->first_con; num_connect < neuron_it->last_con;
				num_connect++)
			{
				if(bias_neuron == ann->connections[num_connect])
				{
					ann->weights[num_connect] = (fann_type) fann_rand(-scale_factor, scale_factor);
				}
				else
				{
					ann->weights[num_connect] = (fann_type) fann_rand(0, scale_factor);
				}
			}
		}
	}

	if(ann->prev_train_slopes != NULL)
	{
		fann_clear_train_arrays(ann);
	}
}
FANN_EXTERNAL struct fann *FANN_API fann_create_sparse_array(float connection_rate,
															 unsigned int num_layers,
															 const unsigned int *layers)
{
	struct fann_layer *layer_it, *last_layer, *prev_layer;
	struct fann *ann;
	struct fann_neuron *neuron_it, *last_neuron, *random_neuron, *bias_neuron;
#ifdef DEBUG
	unsigned int prev_layer_size;
#endif
	unsigned int num_neurons_in, num_neurons_out, i, j;
	unsigned int min_connections, max_connections, num_connections;
	unsigned int connections_per_neuron, allocated_connections;
	unsigned int random_number, found_connection, tmp_con;

#ifdef FIXEDFANN
	unsigned int decimal_point;
	unsigned int multiplier;
#endif
	if(connection_rate > 1)
	{
		connection_rate = 1;
	}

	/* seed random */
#ifndef FANN_NO_SEED
	fann_seed_rand();
#endif

	/* allocate the general structure */
	ann = fann_allocate_structure(num_layers);
	if(ann == NULL)
	{
		fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
		return NULL;
	}

	ann->connection_rate = connection_rate;
#ifdef FIXEDFANN
	decimal_point = ann->decimal_point;
	multiplier = ann->multiplier;
	fann_update_stepwise(ann);
#endif

	/* determine how many neurons there should be in each layer */
	i = 0;
	for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++)
	{
		/* we do not allocate room here, but we make sure that
		 * last_neuron - first_neuron is the number of neurons */
		layer_it->first_neuron = NULL;
		layer_it->last_neuron = layer_it->first_neuron + layers[i++] + 1;	/* +1 for bias */
		ann->total_neurons += layer_it->last_neuron - layer_it->first_neuron;
	}

	ann->num_output = (ann->last_layer - 1)->last_neuron - (ann->last_layer - 1)->first_neuron - 1;
	ann->num_input = ann->first_layer->last_neuron - ann->first_layer->first_neuron - 1;

	/* allocate room for the actual neurons */
	fann_allocate_neurons(ann);
	if(ann->errno_f == FANN_E_CANT_ALLOCATE_MEM)
	{
		fann_destroy(ann);
		return NULL;
	}

#ifdef DEBUG
	printf("creating network with connection rate %f\n", connection_rate);
	printf("input\n");
	printf("  layer       : %d neurons, 1 bias\n",
		   ann->first_layer->last_neuron - ann->first_layer->first_neuron - 1);
#endif

	num_neurons_in = ann->num_input;
	for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
	{
		num_neurons_out = layer_it->last_neuron - layer_it->first_neuron - 1;
		/*�if all neurons in each layer should be connected to at least one neuron
		 * in the previous layer, and one neuron in the next layer.
		 * and the bias node should be connected to the all neurons in the next layer.
		 * Then this is the minimum amount of neurons */
		min_connections = fann_max(num_neurons_in, num_neurons_out) + num_neurons_out;
		max_connections = num_neurons_in * num_neurons_out;	/* not calculating bias */
		num_connections = fann_max(min_connections,
								   (unsigned int) (0.5 + (connection_rate * max_connections)) +
								   num_neurons_out);

		connections_per_neuron = num_connections / num_neurons_out;
		allocated_connections = 0;
		/* Now split out the connections on the different neurons */
		for(i = 0; i != num_neurons_out; i++)
		{
			layer_it->first_neuron[i].first_con = ann->total_connections + allocated_connections;
			allocated_connections += connections_per_neuron;
			layer_it->first_neuron[i].last_con = ann->total_connections + allocated_connections;

			layer_it->first_neuron[i].activation_function = FANN_SIGMOID_STEPWISE;
#ifdef FIXEDFANN
			layer_it->first_neuron[i].activation_steepness = ann->multiplier / 2;
#else
			layer_it->first_neuron[i].activation_steepness = 0.5;
#endif

			if(allocated_connections < (num_connections * (i + 1)) / num_neurons_out)
			{
				layer_it->first_neuron[i].last_con++;
				allocated_connections++;
			}
		}

		/* bias neuron also gets stuff */
		layer_it->first_neuron[i].first_con = ann->total_connections + allocated_connections;
		layer_it->first_neuron[i].last_con = ann->total_connections + allocated_connections;

		ann->total_connections += num_connections;

		/* used in the next run of the loop */
		num_neurons_in = num_neurons_out;
	}

	fann_allocate_connections(ann);
	if(ann->errno_f == FANN_E_CANT_ALLOCATE_MEM)
	{
		fann_destroy(ann);
		return NULL;
	}

	if(connection_rate >= 1)
	{
#ifdef DEBUG
		prev_layer_size = ann->num_input + 1;
#endif
		prev_layer = ann->first_layer;
		last_layer = ann->last_layer;
		for(layer_it = ann->first_layer + 1; layer_it != last_layer; layer_it++)
		{
			last_neuron = layer_it->last_neuron - 1;
			for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
			{
				tmp_con = neuron_it->last_con - 1;
				for(i = neuron_it->first_con; i != tmp_con; i++)
				{
					ann->weights[i] = (fann_type) fann_random_weight();
					/* these connections are still initialized for fully connected networks, to allow
					 * operations to work, that are not optimized for fully connected networks.
					 */
					ann->connections[i] = prev_layer->first_neuron + (i - neuron_it->first_con);
				}

				/* bias weight */
				ann->weights[tmp_con] = (fann_type) fann_random_bias_weight();
				ann->connections[tmp_con] = prev_layer->first_neuron + (tmp_con - neuron_it->first_con);
			}
#ifdef DEBUG
			prev_layer_size = layer_it->last_neuron - layer_it->first_neuron;
#endif
			prev_layer = layer_it;
#ifdef DEBUG
			printf("  layer       : %d neurons, 1 bias\n", prev_layer_size - 1);
#endif
		}
	}
	else
	{
		/* make connections for a network, that are not fully connected */

		/* generally, what we do is first to connect all the input
		 * neurons to a output neuron, respecting the number of
		 * available input neurons for each output neuron. Then
		 * we go through all the output neurons, and connect the
		 * rest of the connections to input neurons, that they are
		 * not allready connected to.
		 */

		/* All the connections are cleared by calloc, because we want to
		 * be able to see which connections are allready connected */

		for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
		{

			num_neurons_out = layer_it->last_neuron - layer_it->first_neuron - 1;
			num_neurons_in = (layer_it - 1)->last_neuron - (layer_it - 1)->first_neuron - 1;

			/* first connect the bias neuron */
			bias_neuron = (layer_it - 1)->last_neuron - 1;
			last_neuron = layer_it->last_neuron - 1;
			for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
			{

				ann->connections[neuron_it->first_con] = bias_neuron;
				ann->weights[neuron_it->first_con] = (fann_type) fann_random_bias_weight();
			}

			/* then connect all neurons in the input layer */
			last_neuron = (layer_it - 1)->last_neuron - 1;
			for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++)
			{

				/* random neuron in the output layer that has space
				 * for more connections */
				do
				{
					random_number = (int) (0.5 + fann_rand(0, num_neurons_out - 1));
					random_neuron = layer_it->first_neuron + random_number;
					/* checks the last space in the connections array for room */
				}
				while(ann->connections[random_neuron->last_con - 1]);

				/* find an empty space in the connection array and connect */
				for(i = random_neuron->first_con; i < random_neuron->last_con; i++)
				{
					if(ann->connections[i] == NULL)
					{
						ann->connections[i] = neuron_it;
						ann->weights[i] = (fann_type) fann_random_weight();
						break;
					}
				}
			}

			/* then connect the rest of the unconnected neurons */
			last_neuron = layer_it->last_neuron - 1;
			for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
			{
				/* find empty space in the connection array and connect */
				for(i = neuron_it->first_con; i < neuron_it->last_con; i++)
				{
					/* continue if allready connected */
					if(ann->connections[i] != NULL)
						continue;

					do
					{
						found_connection = 0;
						random_number = (int) (0.5 + fann_rand(0, num_neurons_in - 1));
						random_neuron = (layer_it - 1)->first_neuron + random_number;

						/* check to see if this connection is allready there */
						for(j = neuron_it->first_con; j < i; j++)
						{
							if(random_neuron == ann->connections[j])
							{
								found_connection = 1;
								break;
							}
						}

					}
					while(found_connection);

					/* we have found a neuron that is not allready
					 * connected to us, connect it */
					ann->connections[i] = random_neuron;
					ann->weights[i] = (fann_type) fann_random_weight();
				}
			}

#ifdef DEBUG
			printf("  layer       : %d neurons, 1 bias\n", num_neurons_out);
#endif
		}

		/* TODO it would be nice to have the randomly created
		 * connections sorted for smoother memory access.
		 */
	}

#ifdef DEBUG
	printf("output\n");
#endif

	return ann;
}
Esempio n. 6
0
int fann_initialize_candidates(struct fann *ann)
{
	/* The candidates are allocated after the normal neurons and connections,
	 * but there is an empty place between the real neurons and the candidate neurons,
	 * so that it will be possible to make room when the chosen candidate are copied in
	 * on the desired place.
	 */
	unsigned int neurons_to_allocate, connections_to_allocate;
	unsigned int num_candidates = fann_get_cascade_num_candidates(ann);
	unsigned int num_neurons = ann->total_neurons + num_candidates + 1;
	unsigned int num_hidden_neurons = ann->total_neurons - ann->num_input - ann->num_output;
	unsigned int candidate_connections_in = ann->total_neurons - ann->num_output;
	unsigned int candidate_connections_out = ann->num_output;

	/* the number of connections going into a and out of a candidate is
	 * ann->total_neurons */
	unsigned int num_connections =
		ann->total_connections + (ann->total_neurons * (num_candidates + 1));
	unsigned int first_candidate_connection = ann->total_connections + ann->total_neurons;
	unsigned int first_candidate_neuron = ann->total_neurons + 1;
	unsigned int connection_it, i, j, k, candidate_index;
	struct fann_neuron *neurons;
	float scale_factor;
	
	/* First make sure that there is enough room, and if not then allocate a
	 * bit more so that we do not need to allocate more room each time.
	 */
	if(num_neurons > fann_get_total_neurons_allocated(ann))
	{
		/* Then we need to allocate more neurons
		 * Allocate half as many neurons as already exist (at least ten)
		 */
		neurons_to_allocate = num_neurons + num_neurons / 2;
		if(neurons_to_allocate < num_neurons + 10)
		{
			neurons_to_allocate = num_neurons + 10;
		}

		if(fann_reallocate_neurons(ann, neurons_to_allocate) == -1)
		{
			return -1;
		}
	}

	if(num_connections > fann_get_total_connections_allocated(ann))
	{
		/* Then we need to allocate more connections
		 * Allocate half as many connections as already exist
		 * (at least enough for ten neurons)
		 */
		connections_to_allocate = num_connections + num_connections / 2;
		if(connections_to_allocate < num_connections + ann->total_neurons * 10)
		{
			connections_to_allocate = num_connections + ann->total_neurons * 10;
		}

		if(fann_reallocate_connections(ann, connections_to_allocate) == -1)
		{
			return -1;
		}
	}

	/* Some test code to do semi Widrow + Nguyen initialization */
	scale_factor = (float) 2.0f*(pow((double) (0.7f * (double) num_hidden_neurons),
				                (double) (1.0f / (double) ann->num_input)));
	if(scale_factor > 8)
		scale_factor = 8;
	else if(scale_factor < 0.5)
		scale_factor = 0.5;

	/* Set the neurons.
	 */
	connection_it = first_candidate_connection;
	neurons = ann->first_layer->first_neuron;
	candidate_index = first_candidate_neuron;

	for(i = 0; i < fann_get_cascade_activation_functions_count(ann); i++)
	{
		for(j = 0; j < fann_get_cascade_activation_steepnesses_count(ann); j++)
		{
			for(k = 0; k < fann_get_cascade_num_candidate_groups(ann); k++)
			{
				/* TODO candidates should actually be created both in
				 * the last layer before the output layer, and in a new layer.
				 */
				neurons[candidate_index].value = 0;
				neurons[candidate_index].sum = 0;
				
				neurons[candidate_index].activation_function =
					fann_get_cascade_activation_functions(ann)[i];
				neurons[candidate_index].activation_steepness =
					fann_get_cascade_activation_steepnesses(ann)[j];
				
				neurons[candidate_index].first_con = connection_it;
				connection_it += candidate_connections_in;
				neurons[candidate_index].last_con = connection_it;
				/* We have no specific pointers to the output weights, but they are
				 * available after last_con */
				connection_it += candidate_connections_out;
				ann->training_params->train_errors[candidate_index] = 0;
				initialize_candidate_weights(ann, neurons[candidate_index].first_con, neurons[candidate_index].last_con+candidate_connections_out, scale_factor);
				candidate_index++;
			}
		}
	}

	
	/* Now randomize the weights and zero out the arrays that needs zeroing out.
	 */
#if 0
#ifdef CASCADE_DEBUG_FULL
	printf("random cand weight [%d ... %d]\n", first_candidate_connection, num_connections - 1);
#endif

	for(i = first_candidate_connection; i < num_connections; i++)
	{
		
		/*ann->weights[i] = fann_random_weight();*/
		ann->weights[i] = fann_rand(-2.0,2.0);
		ann->train_slopes[i] = 0;
		ann->prev_steps[i] = 0;
		ann->prev_train_slopes[i] = initial_slope;
	}
#endif

	return 0;
}