void initialize_candidate_weights(struct fann *ann, unsigned int first_con, unsigned int last_con, float scale_factor)
{
fann_type prev_step;
unsigned int i = 0;
unsigned int bias_weight = (unsigned int)(first_con + (ann->first_layer->last_neuron - ann->first_layer->first_neuron) - 1);
if(ann->training_algorithm == FANN_TRAIN_RPROP)
prev_step = ann->rprop_delta_zero;
else
prev_step = 0;
for(i = first_con; i < last_con; i++)
{
if(i == bias_weight)
ann->weights[i] = fann_rand(-scale_factor, scale_factor);
else
ann->weights[i] = fann_rand(0,scale_factor);
ann->train_slopes[i] = 0;
ann->prev_steps[i] = prev_step;
ann->prev_train_slopes[i] = 0;
}
}
FANN_EXTERNAL void FANN_API fann_randomize_weights(struct fann *ann, fann_type min_weight,
fann_type max_weight)
{
fann_type *last_weight;
fann_type *weights = ann->weights;
last_weight = weights + ann->total_connections;
for(; weights != last_weight; weights++)
{
*weights = (fann_type) (fann_rand(min_weight, max_weight));
}
#ifndef FIXEDFANN
if(ann->prev_train_slopes != NULL)
{
fann_clear_train_arrays(ann);
}
#endif
}
/* Allocates room inside the neuron for the connections.
* Creates a fully connected neuron
*/
FANN_EXTERNAL int FANN_API fann_sparse_neuron_constructor(struct fann *ann, struct fann_layer *layer,
struct fann_neuron *neuron, struct fann_neuron_descr * descr)
{
unsigned int i, j;
unsigned int min_connections, max_connections, num_connections;
unsigned int connections_per_output;
float connection_rate = * ((float* )descr->private_data);
struct fann_sparse_neuron_private_data* private_data;
struct fann_neuron_private_data_connected_any_any* generic_private_data;
fann_type *mask, *weights;
struct dice *dices;
#ifdef FIXEDFANN
fann_type multiplier = ann->fixed_params->multiplier;
neuron->activation_steepness = ann->fixed_params->multiplier / 2;
#else
neuron->activation_steepness = 0.5;
#endif
connection_rate = connection_rate > 1.0f ? 1.0f : connection_rate;
neuron->activation_function = FANN_SIGMOID_STEPWISE;
neuron->num_outputs=descr->num_outputs;
neuron->inputs=layer->inputs;
neuron->num_inputs=layer->num_inputs;
/* set the error array to null (lazy allocation) */
neuron->train_errors=NULL;
/* this is the number of actually allocated weights (some are unused) */
neuron->num_weights=neuron->num_outputs*neuron->num_inputs;
/* allocate the weights even for unused connections */
if ( (weights = neuron->weights = (fann_type*) calloc(neuron->num_weights, sizeof(fann_type))) == NULL)
return 1;
/* allocate space for the dot products results */
if ( (neuron->sums = (fann_type*) malloc(neuron->num_outputs*sizeof(fann_type))) == NULL)
return 1;
/* allocate private data */
if ( (private_data = neuron->private_data = (struct fann_sparse_neuron_private_data*) malloc(sizeof(struct fann_sparse_neuron_private_data))) == NULL)
return 1;
/* private data stores the connection mask, allocate it */
if ( (mask = private_data->mask = (fann_type*) calloc(neuron->num_weights, sizeof(fann_type))) == NULL)
return 1;
if ( (generic_private_data = private_data->generic = (struct fann_neuron_private_data_connected_any_any*) malloc (sizeof(struct fann_neuron_private_data_connected_any_any))) == NULL)
return 1;
generic_private_data->prev_steps=NULL;
generic_private_data->prev_weights_deltas=NULL;
/* alocate a set of dices to select rows */
if ( (dices = (struct dice*) malloc(neuron->num_inputs*sizeof(struct dice))) == NULL)
return 1;
for (i=0; i<neuron->num_inputs; i++)
{
dices[i].idx=i;
dices[i].value=0;
}
min_connections = fann_max(neuron->num_inputs, neuron->num_outputs);
max_connections = neuron->num_inputs * neuron->num_outputs;
num_connections = fann_max(min_connections,
(unsigned int) (0.5 + (connection_rate * max_connections)));
connections_per_output = num_connections / neuron->num_outputs;
/* Dice throw simulation: a float value is assigned to each input.
* The value decimal component is chosen randomly between 0 and 0.4 ("dice throw").
* The integer components is equal to the number of output neurons already
* connected to this input.
* For each output neuron ecah input gets a new "dice throw". Then the inputs and are
* sorted in ascending order according to the value.
* The first ones in the array had less output neurons attached to the
* and better luck in "dice thow". This ones are selected and theyr value is incremented.
*/
for (i=0; i<neuron->num_outputs; i++)
{
/* throw one dice per input */
for (j=0; j<neuron->num_inputs; j++)
dices[j].value= ((int)dices[j].value) + fann_rand(0, 0.4);
/* sort: smaller (dice value + num_connections) wins) */
qsort((void*) dices, neuron->num_inputs, sizeof(struct dice), dice_sorter);
/* assign connections to the output to the winner inputs */
for (j=0; j<connections_per_output; j++)
{
dices[j].value+=1;
mask[dices[j].idx] = (fann_type) 1.0f;
weights[dices[j].idx] = (fann_type) fann_random_weight();
}
weights += neuron->num_inputs;
}
free(dices);
/* set the function pointers */
neuron->destructor = fann_sparse_neuron_destructor;
neuron->run = fann_sparse_neuron_run;
neuron->backpropagate = fann_sparse_neuron_backprop;
neuron->update_weights = fann_sparse_neuron_update;
neuron->compute_error = fann_sparse_neuron_compute_MSE;
return 0;
}
/* Initialize the weights using Widrow + Nguyen's algorithm.
*/
FANN_EXTERNAL void FANN_API fann_init_weights(struct fann *ann, struct fann_train_data *train_data)
{
fann_type smallest_inp, largest_inp;
unsigned int dat = 0, elem, num_connect, num_hidden_neurons;
struct fann_layer *layer_it;
struct fann_neuron *neuron_it, *last_neuron, *bias_neuron;
#ifdef FIXEDFANN
unsigned int multiplier = ann->multiplier;
#endif
float scale_factor;
for(smallest_inp = largest_inp = train_data->input[0][0]; dat < train_data->num_data; dat++)
{
for(elem = 0; elem < train_data->num_input; elem++)
{
if(train_data->input[dat][elem] < smallest_inp)
smallest_inp = train_data->input[dat][elem];
if(train_data->input[dat][elem] > largest_inp)
largest_inp = train_data->input[dat][elem];
}
}
num_hidden_neurons =
ann->total_neurons - (ann->num_input + ann->num_output +
(unsigned int)(ann->last_layer - ann->first_layer));
scale_factor =
(float) (pow
((double) (0.7f * (double) num_hidden_neurons),
(double) (1.0f / (double) ann->num_input)) / (double) (largest_inp -
smallest_inp));
#ifdef DEBUG
printf("Initializing weights with scale factor %f\n", scale_factor);
#endif
bias_neuron = ann->first_layer->last_neuron - 1;
for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
{
last_neuron = layer_it->last_neuron;
bias_neuron = (layer_it - 1)->last_neuron - 1;
for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
{
for(num_connect = neuron_it->first_con; num_connect < neuron_it->last_con;
num_connect++)
{
if(bias_neuron == ann->connections[num_connect])
{
ann->weights[num_connect] = (fann_type) fann_rand(-scale_factor, scale_factor);
}
else
{
ann->weights[num_connect] = (fann_type) fann_rand(0, scale_factor);
}
}
}
}
if(ann->prev_train_slopes != NULL)
{
fann_clear_train_arrays(ann);
}
}
FANN_EXTERNAL struct fann *FANN_API fann_create_sparse_array(float connection_rate,
unsigned int num_layers,
const unsigned int *layers)
{
struct fann_layer *layer_it, *last_layer, *prev_layer;
struct fann *ann;
struct fann_neuron *neuron_it, *last_neuron, *random_neuron, *bias_neuron;
#ifdef DEBUG
unsigned int prev_layer_size;
#endif
unsigned int num_neurons_in, num_neurons_out, i, j;
unsigned int min_connections, max_connections, num_connections;
unsigned int connections_per_neuron, allocated_connections;
unsigned int random_number, found_connection, tmp_con;
#ifdef FIXEDFANN
unsigned int decimal_point;
unsigned int multiplier;
#endif
if(connection_rate > 1)
{
connection_rate = 1;
}
/* seed random */
#ifndef FANN_NO_SEED
fann_seed_rand();
#endif
/* allocate the general structure */
ann = fann_allocate_structure(num_layers);
if(ann == NULL)
{
fann_error(NULL, FANN_E_CANT_ALLOCATE_MEM);
return NULL;
}
ann->connection_rate = connection_rate;
#ifdef FIXEDFANN
decimal_point = ann->decimal_point;
multiplier = ann->multiplier;
fann_update_stepwise(ann);
#endif
/* determine how many neurons there should be in each layer */
i = 0;
for(layer_it = ann->first_layer; layer_it != ann->last_layer; layer_it++)
{
/* we do not allocate room here, but we make sure that
* last_neuron - first_neuron is the number of neurons */
layer_it->first_neuron = NULL;
layer_it->last_neuron = layer_it->first_neuron + layers[i++] + 1; /* +1 for bias */
ann->total_neurons += layer_it->last_neuron - layer_it->first_neuron;
}
ann->num_output = (ann->last_layer - 1)->last_neuron - (ann->last_layer - 1)->first_neuron - 1;
ann->num_input = ann->first_layer->last_neuron - ann->first_layer->first_neuron - 1;
/* allocate room for the actual neurons */
fann_allocate_neurons(ann);
if(ann->errno_f == FANN_E_CANT_ALLOCATE_MEM)
{
fann_destroy(ann);
return NULL;
}
#ifdef DEBUG
printf("creating network with connection rate %f\n", connection_rate);
printf("input\n");
printf(" layer : %d neurons, 1 bias\n",
ann->first_layer->last_neuron - ann->first_layer->first_neuron - 1);
#endif
num_neurons_in = ann->num_input;
for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
{
num_neurons_out = layer_it->last_neuron - layer_it->first_neuron - 1;
/*�if all neurons in each layer should be connected to at least one neuron
* in the previous layer, and one neuron in the next layer.
* and the bias node should be connected to the all neurons in the next layer.
* Then this is the minimum amount of neurons */
min_connections = fann_max(num_neurons_in, num_neurons_out) + num_neurons_out;
max_connections = num_neurons_in * num_neurons_out; /* not calculating bias */
num_connections = fann_max(min_connections,
(unsigned int) (0.5 + (connection_rate * max_connections)) +
num_neurons_out);
connections_per_neuron = num_connections / num_neurons_out;
allocated_connections = 0;
/* Now split out the connections on the different neurons */
for(i = 0; i != num_neurons_out; i++)
{
layer_it->first_neuron[i].first_con = ann->total_connections + allocated_connections;
allocated_connections += connections_per_neuron;
layer_it->first_neuron[i].last_con = ann->total_connections + allocated_connections;
layer_it->first_neuron[i].activation_function = FANN_SIGMOID_STEPWISE;
#ifdef FIXEDFANN
layer_it->first_neuron[i].activation_steepness = ann->multiplier / 2;
#else
layer_it->first_neuron[i].activation_steepness = 0.5;
#endif
if(allocated_connections < (num_connections * (i + 1)) / num_neurons_out)
{
layer_it->first_neuron[i].last_con++;
allocated_connections++;
}
}
/* bias neuron also gets stuff */
layer_it->first_neuron[i].first_con = ann->total_connections + allocated_connections;
layer_it->first_neuron[i].last_con = ann->total_connections + allocated_connections;
ann->total_connections += num_connections;
/* used in the next run of the loop */
num_neurons_in = num_neurons_out;
}
fann_allocate_connections(ann);
if(ann->errno_f == FANN_E_CANT_ALLOCATE_MEM)
{
fann_destroy(ann);
return NULL;
}
if(connection_rate >= 1)
{
#ifdef DEBUG
prev_layer_size = ann->num_input + 1;
#endif
prev_layer = ann->first_layer;
last_layer = ann->last_layer;
for(layer_it = ann->first_layer + 1; layer_it != last_layer; layer_it++)
{
last_neuron = layer_it->last_neuron - 1;
for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
{
tmp_con = neuron_it->last_con - 1;
for(i = neuron_it->first_con; i != tmp_con; i++)
{
ann->weights[i] = (fann_type) fann_random_weight();
/* these connections are still initialized for fully connected networks, to allow
* operations to work, that are not optimized for fully connected networks.
*/
ann->connections[i] = prev_layer->first_neuron + (i - neuron_it->first_con);
}
/* bias weight */
ann->weights[tmp_con] = (fann_type) fann_random_bias_weight();
ann->connections[tmp_con] = prev_layer->first_neuron + (tmp_con - neuron_it->first_con);
}
#ifdef DEBUG
prev_layer_size = layer_it->last_neuron - layer_it->first_neuron;
#endif
prev_layer = layer_it;
#ifdef DEBUG
printf(" layer : %d neurons, 1 bias\n", prev_layer_size - 1);
#endif
}
}
else
{
/* make connections for a network, that are not fully connected */
/* generally, what we do is first to connect all the input
* neurons to a output neuron, respecting the number of
* available input neurons for each output neuron. Then
* we go through all the output neurons, and connect the
* rest of the connections to input neurons, that they are
* not allready connected to.
*/
/* All the connections are cleared by calloc, because we want to
* be able to see which connections are allready connected */
for(layer_it = ann->first_layer + 1; layer_it != ann->last_layer; layer_it++)
{
num_neurons_out = layer_it->last_neuron - layer_it->first_neuron - 1;
num_neurons_in = (layer_it - 1)->last_neuron - (layer_it - 1)->first_neuron - 1;
/* first connect the bias neuron */
bias_neuron = (layer_it - 1)->last_neuron - 1;
last_neuron = layer_it->last_neuron - 1;
for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
{
ann->connections[neuron_it->first_con] = bias_neuron;
ann->weights[neuron_it->first_con] = (fann_type) fann_random_bias_weight();
}
/* then connect all neurons in the input layer */
last_neuron = (layer_it - 1)->last_neuron - 1;
for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++)
{
/* random neuron in the output layer that has space
* for more connections */
do
{
random_number = (int) (0.5 + fann_rand(0, num_neurons_out - 1));
random_neuron = layer_it->first_neuron + random_number;
/* checks the last space in the connections array for room */
}
while(ann->connections[random_neuron->last_con - 1]);
/* find an empty space in the connection array and connect */
for(i = random_neuron->first_con; i < random_neuron->last_con; i++)
{
if(ann->connections[i] == NULL)
{
ann->connections[i] = neuron_it;
ann->weights[i] = (fann_type) fann_random_weight();
break;
}
}
}
/* then connect the rest of the unconnected neurons */
last_neuron = layer_it->last_neuron - 1;
for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
{
/* find empty space in the connection array and connect */
for(i = neuron_it->first_con; i < neuron_it->last_con; i++)
{
/* continue if allready connected */
if(ann->connections[i] != NULL)
continue;
do
{
found_connection = 0;
random_number = (int) (0.5 + fann_rand(0, num_neurons_in - 1));
random_neuron = (layer_it - 1)->first_neuron + random_number;
/* check to see if this connection is allready there */
for(j = neuron_it->first_con; j < i; j++)
{
if(random_neuron == ann->connections[j])
{
found_connection = 1;
break;
}
}
}
while(found_connection);
/* we have found a neuron that is not allready
* connected to us, connect it */
ann->connections[i] = random_neuron;
ann->weights[i] = (fann_type) fann_random_weight();
}
}
#ifdef DEBUG
printf(" layer : %d neurons, 1 bias\n", num_neurons_out);
#endif
}
/* TODO it would be nice to have the randomly created
* connections sorted for smoother memory access.
*/
}
#ifdef DEBUG
printf("output\n");
#endif
return ann;
}
int fann_initialize_candidates(struct fann *ann)
{
/* The candidates are allocated after the normal neurons and connections,
* but there is an empty place between the real neurons and the candidate neurons,
* so that it will be possible to make room when the chosen candidate are copied in
* on the desired place.
*/
unsigned int neurons_to_allocate, connections_to_allocate;
unsigned int num_candidates = fann_get_cascade_num_candidates(ann);
unsigned int num_neurons = ann->total_neurons + num_candidates + 1;
unsigned int num_hidden_neurons = ann->total_neurons - ann->num_input - ann->num_output;
unsigned int candidate_connections_in = ann->total_neurons - ann->num_output;
unsigned int candidate_connections_out = ann->num_output;
/* the number of connections going into a and out of a candidate is
* ann->total_neurons */
unsigned int num_connections =
ann->total_connections + (ann->total_neurons * (num_candidates + 1));
unsigned int first_candidate_connection = ann->total_connections + ann->total_neurons;
unsigned int first_candidate_neuron = ann->total_neurons + 1;
unsigned int connection_it, i, j, k, candidate_index;
struct fann_neuron *neurons;
float scale_factor;
/* First make sure that there is enough room, and if not then allocate a
* bit more so that we do not need to allocate more room each time.
*/
if(num_neurons > fann_get_total_neurons_allocated(ann))
{
/* Then we need to allocate more neurons
* Allocate half as many neurons as already exist (at least ten)
*/
neurons_to_allocate = num_neurons + num_neurons / 2;
if(neurons_to_allocate < num_neurons + 10)
{
neurons_to_allocate = num_neurons + 10;
}
if(fann_reallocate_neurons(ann, neurons_to_allocate) == -1)
{
return -1;
}
}
if(num_connections > fann_get_total_connections_allocated(ann))
{
/* Then we need to allocate more connections
* Allocate half as many connections as already exist
* (at least enough for ten neurons)
*/
connections_to_allocate = num_connections + num_connections / 2;
if(connections_to_allocate < num_connections + ann->total_neurons * 10)
{
connections_to_allocate = num_connections + ann->total_neurons * 10;
}
if(fann_reallocate_connections(ann, connections_to_allocate) == -1)
{
return -1;
}
}
/* Some test code to do semi Widrow + Nguyen initialization */
scale_factor = (float) 2.0f*(pow((double) (0.7f * (double) num_hidden_neurons),
(double) (1.0f / (double) ann->num_input)));
if(scale_factor > 8)
scale_factor = 8;
else if(scale_factor < 0.5)
scale_factor = 0.5;
/* Set the neurons.
*/
connection_it = first_candidate_connection;
neurons = ann->first_layer->first_neuron;
candidate_index = first_candidate_neuron;
for(i = 0; i < fann_get_cascade_activation_functions_count(ann); i++)
{
for(j = 0; j < fann_get_cascade_activation_steepnesses_count(ann); j++)
{
for(k = 0; k < fann_get_cascade_num_candidate_groups(ann); k++)
{
/* TODO candidates should actually be created both in
* the last layer before the output layer, and in a new layer.
*/
neurons[candidate_index].value = 0;
neurons[candidate_index].sum = 0;
neurons[candidate_index].activation_function =
fann_get_cascade_activation_functions(ann)[i];
neurons[candidate_index].activation_steepness =
fann_get_cascade_activation_steepnesses(ann)[j];
neurons[candidate_index].first_con = connection_it;
connection_it += candidate_connections_in;
neurons[candidate_index].last_con = connection_it;
/* We have no specific pointers to the output weights, but they are
* available after last_con */
connection_it += candidate_connections_out;
ann->training_params->train_errors[candidate_index] = 0;
initialize_candidate_weights(ann, neurons[candidate_index].first_con, neurons[candidate_index].last_con+candidate_connections_out, scale_factor);
candidate_index++;
}
}
}
/* Now randomize the weights and zero out the arrays that needs zeroing out.
*/
#if 0
#ifdef CASCADE_DEBUG_FULL
printf("random cand weight [%d ... %d]\n", first_candidate_connection, num_connections - 1);
#endif
for(i = first_candidate_connection; i < num_connections; i++)
{
/*ann->weights[i] = fann_random_weight();*/
ann->weights[i] = fann_rand(-2.0,2.0);
ann->train_slopes[i] = 0;
ann->prev_steps[i] = 0;
ann->prev_train_slopes[i] = initial_slope;
}
#endif
return 0;
}