/* INTERNAL FUNCTION compute the error at the network output (usually, after forward propagation of a certain input vector, fann_run) the error is a sum of squares for all the output units also increments a counter because MSE is an average of such errors After this train_errors in the output layer will be set to: neuron_value_derived * (desired_output - neuron_value) */ void fann_compute_MSE(struct fann *ann, fann_type * desired_output) { fann_type neuron_value, neuron_diff, *error_it = 0, *error_begin = 0; struct fann_neuron *last_layer_begin = (ann->last_layer - 1)->first_neuron; const struct fann_neuron *last_layer_end = last_layer_begin + ann->num_output; const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; /* if no room allocated for the error variabels, allocate it now */ if(ann->train_errors == NULL) { ann->train_errors = (fann_type *) calloc(ann->total_neurons, sizeof(fann_type)); if(ann->train_errors == NULL) { fann_error((struct fann_error *) ann, FANN_E_CANT_ALLOCATE_MEM); return; } } else { /* clear the error variabels */ memset(ann->train_errors, 0, (ann->total_neurons) * sizeof(fann_type)); } error_begin = ann->train_errors; #ifdef DEBUGTRAIN printf("\ncalculate errors\n"); #endif /* calculate the error and place it in the output layer */ error_it = error_begin + (last_layer_begin - first_neuron); for(; last_layer_begin != last_layer_end; last_layer_begin++) { neuron_value = last_layer_begin->value; neuron_diff = *desired_output - neuron_value; neuron_diff = fann_update_MSE(ann, last_layer_begin, neuron_diff); if(ann->train_error_function) { /* TODO make switch when more functions */ if(neuron_diff < -.9999999) neuron_diff = -17.0; else if(neuron_diff > .9999999) neuron_diff = 17.0; else neuron_diff = (fann_type) log((1.0 + neuron_diff) / (1.0 - neuron_diff)); } *error_it = fann_activation_derived(last_layer_begin->activation_function, last_layer_begin->activation_steepness, neuron_value, last_layer_begin->sum) * neuron_diff; desired_output++; error_it++; ann->num_MSE++; } }
/* INTERNAL FUNCTION Propagate the error backwards from the output layer. After this the train_errors in the hidden layers will be: neuron_value_derived * sum(outgoing_weights * connected_neuron) */ void fann_backpropagate_MSE(struct fann *ann) { fann_type tmp_error, max; unsigned int i; struct fann_layer *layer_it; struct fann_neuron *neuron_it, *last_neuron; struct fann_neuron **connections; fann_type *error_begin = ann->train_errors; fann_type *error_prev_layer; fann_type *weights; unsigned int *connections_to_weights; const struct fann_neuron *first_neuron = ann->first_layer->first_neuron; const struct fann_layer *second_layer = ann->first_layer + 1; struct fann_layer *last_layer = ann->last_layer; /* go through all the layers, from last to first. * And propagate the error backwards */ for(layer_it = last_layer - 1; layer_it > second_layer; --layer_it) { last_neuron = layer_it->last_neuron; /* for each connection in this layer, propagate the error backwards */ if(ann->connection_rate >= 1) { if(ann->network_type == FANN_NETTYPE_LAYER) { error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron); } else { error_prev_layer = error_begin; } for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) { tmp_error = error_begin[neuron_it - first_neuron]; weights = ann->weights; connections_to_weights = ann->connections_to_weights + neuron_it->first_con; for(i = neuron_it->last_con - neuron_it->first_con; i--;) { /*printf("i = %d\n", i); * printf("error_prev_layer[%d] = %f\n", i, error_prev_layer[i]); * printf("weights[%d] = %f\n", i, weights[i]); */ error_prev_layer[i] += tmp_error * weights[connections_to_weights[i]]; } } } else { for(neuron_it = layer_it->first_neuron; neuron_it != last_neuron; neuron_it++) { tmp_error = error_begin[neuron_it - first_neuron]; weights = ann->weights; connections_to_weights = ann->connections_to_weights + neuron_it->first_con; connections = ann->connections + neuron_it->first_con; if (neuron_it->activation_function != FANN_MAXPOOLING){ for(i = neuron_it->last_con - neuron_it->first_con; i--;) { error_begin[connections[i] - first_neuron] += tmp_error * weights[connections_to_weights[i]]; } }else{ tmp_error = error_begin[neuron_it - first_neuron]; weights = ann->weights; connections_to_weights = ann->connections_to_weights + neuron_it->first_con; connections = ann->connections + neuron_it->first_con; max = connections[neuron_it->last_con - neuron_it->first_con]->value; //find the maximum value from the previous layers for(i = neuron_it->last_con - neuron_it->first_con; i--;) { max = fann_max(max, connections[i]->value); } for(i = neuron_it->last_con - neuron_it->first_con; i--;) { if (connections[i]->value == max){ error_begin[connections[i] - first_neuron] += tmp_error; } } } } } /* then calculate the actual errors in the previous layer */ error_prev_layer = error_begin + ((layer_it - 1)->first_neuron - first_neuron); last_neuron = (layer_it - 1)->last_neuron; for(neuron_it = (layer_it - 1)->first_neuron; neuron_it != last_neuron; neuron_it++) { *error_prev_layer *= fann_activation_derived(neuron_it->activation_function, neuron_it->activation_steepness, neuron_it->value, neuron_it->sum); error_prev_layer++; } } }
void fann_update_candidate_slopes(struct fann *ann) { struct fann_neuron *neurons = ann->first_layer->first_neuron; struct fann_neuron *first_cand = neurons + ann->total_neurons + 1; struct fann_neuron *last_cand = first_cand + fann_get_cascade_num_candidates(ann); struct fann_neuron *cand_it; unsigned int i, j, num_connections; unsigned int num_output = ann->num_output; fann_type max_sum, cand_sum, activation, derived, error_value, diff, cand_score; fann_type *weights, *cand_out_weights, *cand_slopes, *cand_out_slopes; fann_type *output_train_errors = ann->train_errors + (ann->total_neurons - ann->num_output); for(cand_it = first_cand; cand_it < last_cand; cand_it++) { cand_score = ann->cascade_candidate_scores[cand_it - first_cand]; error_value = 0.0; /* code more or less stolen from fann_run to fast forward pass */ cand_sum = 0.0; num_connections = cand_it->last_con - cand_it->first_con; weights = ann->weights + cand_it->first_con; /* unrolled loop start */ i = num_connections & 3; /* same as modulo 4 */ switch (i) { case 3: cand_sum += weights[2] * neurons[2].value; case 2: cand_sum += weights[1] * neurons[1].value; case 1: cand_sum += weights[0] * neurons[0].value; case 0: break; } for(; i != num_connections; i += 4) { cand_sum += weights[i] * neurons[i].value + weights[i + 1] * neurons[i + 1].value + weights[i + 2] * neurons[i + 2].value + weights[i + 3] * neurons[i + 3].value; } /* * for(i = 0; i < num_connections; i++){ * cand_sum += weights[i] * neurons[i].value; * } */ /* unrolled loop end */ max_sum = 150/cand_it->activation_steepness; if(cand_sum > max_sum) cand_sum = max_sum; else if(cand_sum < -max_sum) cand_sum = -max_sum; activation = fann_activation(ann, cand_it->activation_function, cand_it->activation_steepness, cand_sum); /* printf("%f = sigmoid(%f);\n", activation, cand_sum); */ cand_it->sum = cand_sum; cand_it->value = activation; derived = fann_activation_derived(cand_it->activation_function, cand_it->activation_steepness, activation, cand_sum); /* The output weights is located right after the input weights in * the weight array. */ cand_out_weights = weights + num_connections; cand_out_slopes = ann->train_slopes + cand_it->first_con + num_connections; for(j = 0; j < num_output; j++) { diff = (activation * cand_out_weights[j]) - output_train_errors[j]; #ifdef CASCADE_DEBUG_FULL /* printf("diff = %f = (%f * %f) - %f;\n", diff, activation, cand_out_weights[j], output_train_errors[j]); */ #endif cand_out_slopes[j] -= 2.0f * diff * activation; #ifdef CASCADE_DEBUG_FULL /* printf("cand_out_slopes[%d] <= %f += %f * %f;\n", j, cand_out_slopes[j], diff, activation); */ #endif error_value += diff * cand_out_weights[j]; cand_score -= (diff * diff); #ifdef CASCADE_DEBUG_FULL /* printf("cand_score[%d][%d] = %f -= (%f * %f)\n", cand_it - first_cand, j, cand_score, diff, diff); */ printf("cand[%d]: error=%f, activation=%f, diff=%f, slope=%f\n", cand_it - first_cand, output_train_errors[j], (activation * cand_out_weights[j]), diff, -2.0 * diff * activation); #endif } ann->cascade_candidate_scores[cand_it - first_cand] = cand_score; error_value *= derived; cand_slopes = ann->train_slopes + cand_it->first_con; for(i = 0; i < num_connections; i++) { cand_slopes[i] -= error_value * neurons[i].value; } } }