matrix network::output_layer_error(const matrix &A,const matrix &Y,const matrix &Zvector_L)
{
matrix Ga=A-Y;
matrix SigmaL_deriv=Sigma_derivatives_vector(Zvector_L,layers_count-1);
return hadamard_product(Ga,SigmaL_deriv);
}
matrix network::hidden_layer_error(int l, const matrix &delta_next,const matrix &Zvector_l )
{
matrix w_next=w[l+1];
matrix Sl_deriv=Sigma_derivatives_vector(Zvector_l,l);
w_next.transpose();
return hadamard_product( w_next*delta_next, Sl_deriv );
}
void Neural_network::train(const std::vector<double>& inputs, const std::vector<double>& targets)
{
// Feed forward
auto inputs_mat = Matrix::from_vector(inputs);
auto hidden_mat = Matrix::multiply(input_to_hidden_weights, inputs_mat);
if(is_bias_on)
hidden_mat.add(bias_hidden);
hidden_mat.apply_function(activation_function_hid);
auto output_mat = Matrix::multiply(hidden_to_output_weights, hidden_mat);
if(is_bias_on)
output_mat.add(bias_output);
output_mat.apply_function(activation_function_out);
auto targets_mat = Matrix::from_vector(targets); // Convert goals to a matrix of targets
// Output layer errors
auto output_errors = Matrix::subtract(targets_mat, output_mat); // Create an output error matrix
// Compute MSE
current_MSE += Matrix::compute_MSE(output_errors);
// Gradient - output
auto output_gradients = Matrix::apply_function(output_mat, activation_function_out_derivative);
output_gradients.hadamard_product(output_errors);
output_gradients.multiply(learning_rate);
// Deltas - output layer
auto hidden_transposed = Matrix::transpose(hidden_mat);
auto hidden_to_output_weights_deltas = Matrix::multiply(output_gradients, hidden_transposed);
auto bias_output_deltas = output_gradients; // In case of biases their deltas are just the gradients.
// Hidden_to_output_weights tweaking
hidden_to_output_weights_previous_deltas.multiply(momentum_coefficient);
hidden_to_output_weights_deltas.add(hidden_to_output_weights_previous_deltas);
hidden_to_output_weights_previous_deltas = hidden_to_output_weights_deltas;
hidden_to_output_weights.add(hidden_to_output_weights_deltas);
// Output bias tweaking
if(is_bias_on) {
bias_output_previous_deltas.multiply(momentum_coefficient);
bias_output_deltas.add(bias_output_previous_deltas);
bias_output_previous_deltas = bias_output_deltas;
bias_output.add(bias_output_deltas); // The actual tweaking.
}
// Hidden layer errors
auto hidden_to_output_weights_transposed = Matrix::transpose(hidden_to_output_weights);
auto hidden_errors = Matrix::multiply(hidden_to_output_weights_transposed, output_errors);
// Gradient - hidden
auto hidden_gradients = Matrix::apply_function(hidden_mat, activation_function_hid_derivative);
hidden_gradients.hadamard_product(hidden_errors);
hidden_gradients.multiply(learning_rate);
// Deltas - hidden layer
auto inputs_transposed = Matrix::transpose(inputs_mat);
auto input_to_hidden_weights_deltas = Matrix::multiply(hidden_gradients, inputs_transposed);
auto bias_hidden_deltas = hidden_gradients; // In case of biases their deltas are just the gradients.
// Input_to_hidden_weights tweaking
input_to_hidden_weights_previous_deltas.multiply(momentum_coefficient);
input_to_hidden_weights_deltas.add(input_to_hidden_weights_previous_deltas);
input_to_hidden_weights_previous_deltas = input_to_hidden_weights_deltas;
input_to_hidden_weights.add(input_to_hidden_weights_deltas);
// Hidden bias tweaking
if(is_bias_on) {
bias_hidden_previous_deltas.multiply(momentum_coefficient);
bias_hidden_deltas.add(bias_hidden_previous_deltas);
bias_hidden_previous_deltas = bias_hidden_deltas;
bias_hidden.add(bias_hidden_deltas); // The actual tweaking.
}
}
