/* INTERNAL FUNCTION
compute the error at the network output
(usually, after forward propagation of a certain input vector, fann_run)
the error is a sum of squares for all the output units
also increments a counter because MSE is an average of such errors
After this train_errors in the output layer will be set to:
neuron_value_derived * (desired_output - neuron_value)
*/
void fann_compute_MSE(struct fann *ann, fann_type * desired_output)
{
fann_type neuron_value, neuron_diff, *error_it = 0, *error_begin = 0;
struct fann_neuron *last_layer_begin = (ann->last_layer - 1)->first_neuron;
const struct fann_neuron *last_layer_end = last_layer_begin + ann->num_output;
const struct fann_neuron *first_neuron = ann->first_layer->first_neuron;
/* if no room allocated for the error variabels, allocate it now */
if(ann->train_errors == NULL)
{
ann->train_errors = (fann_type *) calloc(ann->total_neurons, sizeof(fann_type));
if(ann->train_errors == NULL)
{
fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM);
return;
}
}
else
{
/* clear the error variabels */
memset(ann->train_errors, 0, (ann->total_neurons) * sizeof(fann_type));
}
error_begin = ann->train_errors;
#ifdef DEBUGTRAIN
printf("\ncalculate errors\n");
#endif
/* calculate the error and place it in the output layer */
error_it = error_begin + (last_layer_begin - first_neuron);
for(; last_layer_begin != last_layer_end; last_layer_begin++)
{
neuron_value = last_layer_begin->value;
neuron_diff = *desired_output - neuron_value;
neuron_diff = fann_update_MSE(ann, last_layer_begin, neuron_diff);
if(ann->train_error_function)
{ /* TODO make switch when more functions */
if(neuron_diff < -.9999999)
neuron_diff = -17.0;
else if(neuron_diff > .9999999)
neuron_diff = 17.0;
else
neuron_diff = (fann_type) log((1.0 + neuron_diff) / (1.0 - neuron_diff));
}
*error_it = fann_activation_derived(last_layer_begin->activation_function,
last_layer_begin->activation_steepness, neuron_value,
last_layer_begin->sum) * neuron_diff;
desired_output++;
error_it++;
ann->num_MSE++;
}
}
/* INTERNAL FUNCTION
Propagate the error backwards from the output layer.
After this the train_errors in the hidden layers will be:
neuron_value_derived * sum(outgoing_weights * connected_neuron)
*/
void fann_backpropagate_MSE(struct fann *ann)
{
fann_type tmp_error, max;
unsigned int i;
struct fann_layer *layer_it;
struct fann_neuron *neuron_it, *last_neuron;
struct fann_neuron **connections;
fann_type *error_begin = ann->train_errors;
fann_type *error_prev_layer;
fann_type *weights;
unsigned int *connections_to_weights;
const struct fann_neuron *first_neuron = ann->first_layer->first_neuron;
const struct fann_layer *second_layer = ann->first_layer + 1;
struct fann_layer *last_layer = ann->last_layer;
/* go through all the layers, from last to first.
* And propagate the error backwards */
for(layer_it = last_layer - 1; layer_it > second_layer; --layer_it)
{
last_neuron = layer_it->last_neuron;
/* for each connection in this layer, propagate the error backwards */
if(ann->connection_rate >= 1)
{
if(ann->network_type == FANN_NETTYPE_LAYER)
{
error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron);
}
else
{
error_prev_layer = error_begin;
}
for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
{
tmp_error = error_begin[neuron_it - first_neuron];
weights = ann->weights;
connections_to_weights = ann->connections_to_weights + neuron_it->first_con;
for(i = neuron_it->last_con - neuron_it->first_con; i--;)
{
/*printf("i = %d\n", i);
* printf("error_prev_layer[%d] = %f\n", i, error_prev_layer[i]);
* printf("weights[%d] = %f\n", i, weights[i]); */
error_prev_layer[i] += tmp_error * weights[connections_to_weights[i]];
}
}
}
else
{
for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++)
{
tmp_error = error_begin[neuron_it - first_neuron];
weights = ann->weights;
connections_to_weights = ann->connections_to_weights + neuron_it->first_con;
connections = ann->connections + neuron_it->first_con;
if (neuron_it->activation_function != FANN_MAXPOOLING){
for(i = neuron_it->last_con - neuron_it->first_con; i--;)
{
error_begin[connections[i] - first_neuron] += tmp_error * weights[connections_to_weights[i]];
}
}else{
tmp_error = error_begin[neuron_it - first_neuron];
weights = ann->weights;
connections_to_weights = ann->connections_to_weights + neuron_it->first_con;
connections = ann->connections + neuron_it->first_con;
max = connections[neuron_it->last_con - neuron_it->first_con]->value;
//find the maximum value from the previous layers
for(i = neuron_it->last_con - neuron_it->first_con; i--;)
{
max = fann_max(max, connections[i]->value);
}
for(i = neuron_it->last_con - neuron_it->first_con; i--;)
{
if (connections[i]->value == max){
error_begin[connections[i] - first_neuron] += tmp_error;
}
}
}
}
}
/* then calculate the actual errors in the previous layer */
error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron);
last_neuron = (layer_it - 1)->last_neuron;
for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++)
{
*error_prev_layer *= fann_activation_derived(neuron_it->activation_function,
neuron_it->activation_steepness, neuron_it->value, neuron_it->sum);
error_prev_layer++;
}
}
}
void fann_update_candidate_slopes(struct fann *ann)
{
struct fann_neuron *neurons = ann->first_layer->first_neuron;
struct fann_neuron *first_cand = neurons + ann->total_neurons + 1;
struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann);
struct fann_neuron *cand_it;
unsigned int i, j, num_connections;
unsigned int num_output = ann->num_output;
fann_type max_sum, cand_sum, activation, derived, error_value, diff, cand_score;
fann_type *weights, *cand_out_weights, *cand_slopes, *cand_out_slopes;
fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output);
for(cand_it = first_cand; cand_it < last_cand; cand_it++)
{
cand_score = ann->cascade_candidate_scores[cand_it - first_cand];
error_value = 0.0;
/* code more or less stolen from fann_run to fast forward pass
*/
cand_sum = 0.0;
num_connections = cand_it->last_con - cand_it->first_con;
weights = ann->weights + cand_it->first_con;
/* unrolled loop start */
i = num_connections & 3; /* same as modulo 4 */
switch (i)
{
case 3:
cand_sum += weights[2] * neurons[2].value;
case 2:
cand_sum += weights[1] * neurons[1].value;
case 1:
cand_sum += weights[0] * neurons[0].value;
case 0:
break;
}
for(; i != num_connections; i += 4)
{
cand_sum +=
weights[i] * neurons[i].value +
weights[i + 1] * neurons[i + 1].value +
weights[i + 2] * neurons[i + 2].value + weights[i + 3] * neurons[i + 3].value;
}
/*
* for(i = 0; i < num_connections; i++){
* cand_sum += weights[i] * neurons[i].value;
* }
*/
/* unrolled loop end */
max_sum = 150/cand_it->activation_steepness;
if(cand_sum > max_sum)
cand_sum = max_sum;
else if(cand_sum < -max_sum)
cand_sum = -max_sum;
activation =
fann_activation(ann, cand_it->activation_function, cand_it->activation_steepness,
cand_sum);
/* printf("%f = sigmoid(%f);\n", activation, cand_sum); */
cand_it->sum = cand_sum;
cand_it->value = activation;
derived = fann_activation_derived(cand_it->activation_function,
cand_it->activation_steepness, activation, cand_sum);
/* The output weights is located right after the input weights in
* the weight array.
*/
cand_out_weights = weights + num_connections;
cand_out_slopes = ann->train_slopes + cand_it->first_con + num_connections;
for(j = 0; j < num_output; j++)
{
diff = (activation * cand_out_weights[j]) - output_train_errors[j];
#ifdef CASCADE_DEBUG_FULL
/* printf("diff = %f = (%f * %f) - %f;\n", diff, activation, cand_out_weights[j], output_train_errors[j]); */
#endif
cand_out_slopes[j] -= 2.0f * diff * activation;
#ifdef CASCADE_DEBUG_FULL
/* printf("cand_out_slopes[%d] <= %f += %f * %f;\n", j, cand_out_slopes[j], diff, activation); */
#endif
error_value += diff * cand_out_weights[j];
cand_score -= (diff * diff);
#ifdef CASCADE_DEBUG_FULL
/* printf("cand_score[%d][%d] = %f -= (%f * %f)\n", cand_it - first_cand, j, cand_score, diff, diff); */
printf("cand[%d]: error=%f, activation=%f, diff=%f, slope=%f\n", cand_it - first_cand,
output_train_errors[j], (activation * cand_out_weights[j]), diff,
-2.0 * diff * activation);
#endif
}
ann->cascade_candidate_scores[cand_it - first_cand] = cand_score;
error_value *= derived;
cand_slopes = ann->train_slopes + cand_it->first_con;
for(i = 0; i < num_connections; i++)
{
cand_slopes[i] -= error_value * neurons[i].value;
}
}
}
