void NumericalDifferentiationTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
Vector<double> x;
Vector<double> g;
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::ForwardDifferences);
x.set(2, 0.0);
g = nd.calculate_gradient(*this, &NumericalDifferentiationTest::f2, x);
assert_true(g.size() == 2, LOG);
assert_true(g == 1.0, LOG);
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::CentralDifferences);
x.set(2, 0.0);
g = nd.calculate_gradient(*this, &NumericalDifferentiationTest::f2, x);
assert_true(g.size() == 2, LOG);
assert_true(g == 1.0, LOG);
}
void ProbabilisticLayerTest::test_calculate_Jacobian(void)
{
message += "test_calculate_Jacobian\n";
NumericalDifferentiation nd;
ProbabilisticLayer pl;
Vector<double> inputs;
Matrix<double> Jacobian;
Matrix<double> numerical_Jacobian;
// Test
if(numerical_differentiation_tests)
{
pl.set_probabilistic_method(ProbabilisticLayer::Softmax);
pl.set(3);
inputs.set(3);
inputs.randomize_normal();
Jacobian = pl.calculate_Jacobian(inputs);
numerical_Jacobian = nd.calculate_Jacobian(pl, &ProbabilisticLayer::calculate_outputs, inputs);
assert_true((Jacobian-numerical_Jacobian).calculate_absolute_value() < 1.0e-3, LOG);
}
}
void NumericalDifferentiationTest::test_calculate_Hessian(void)
{
message += "test_calculate_Hessian\n";
NumericalDifferentiation nd;
Vector<double> x;
Matrix<double> H;
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::ForwardDifferences);
x.set(2, 0.0);
H = nd.calculate_Hessian(*this, &NumericalDifferentiationTest::f2, x);
assert_true(H.get_rows_number() == 2, LOG);
assert_true(H.get_columns_number() == 2, LOG);
assert_true(H == 0.0, LOG);
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::CentralDifferences);
x.set(2, 0.0);
H = nd.calculate_Hessian(*this, &NumericalDifferentiationTest::f2, x);
assert_true(H.get_rows_number() == 2, LOG);
assert_true(H.get_columns_number() == 2, LOG);
assert_true(H == 0.0, LOG);
}
void NumericalDifferentiationTest::test_calculate_Jacobian(void)
{
message += "test_calculate_Jacobian\n";
NumericalDifferentiation nd;
Vector<double> x;
Matrix<double> J;
Matrix<double> J_true;
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::ForwardDifferences);
x.set(2, 0.0);
J = nd.calculate_central_differences_Jacobian(*this, &NumericalDifferentiationTest::f3, x);
J_true.set_identity(2);
assert_true(J == J_true, LOG);
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::CentralDifferences);
x.set(2, 0.0);
J = nd.calculate_central_differences_Jacobian(*this, &NumericalDifferentiationTest::f3, x);
J_true.set_identity(2);
assert_true(J == J_true, LOG);
}
void NumericalDifferentiationTest::test_calculate_derivative(void)
{
message += "test_calculate_derivative\n";
NumericalDifferentiation nd;
// double x;
double d;
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::ForwardDifferences);
// x = 0.0;
d = nd.calculate_derivative(*this, &NumericalDifferentiationTest::f1, 0.0);
assert_true(d == 1.0, LOG);
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::CentralDifferences);
// x = 0.0;
d = nd.calculate_derivative(*this, &NumericalDifferentiationTest::f1, 0.0);
assert_true(d == 1.0, LOG);
}
void NumericalDifferentiationTest::test_calculate_second_derivative(void)
{
message += "test_calculate_second_derivative\n";
NumericalDifferentiation nd;
double x;
double d2;
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::ForwardDifferences);
x = 0.0;
d2 = nd.calculate_second_derivative(*this, &NumericalDifferentiationTest::f1, x);
assert_true(fabs(d2) <= 1.0e-6, LOG);
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::CentralDifferences);
x = 0.0;
d2 = nd.calculate_second_derivative(*this, &NumericalDifferentiationTest::f1, x);
assert_true(fabs(d2) <= 1.0e-6, LOG);
}
void NormalizedSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<double> network_parameters;
DataSet ds;
Matrix<double> data;
NormalizedSquaredError nse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> numerical_objective_gradient;
// Test
nn.set(1,1,1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 2);
data.set(2, 2);
data[0][0] = -1.0;
data[0][1] = -1.0;
data[1][0] = 1.0;
data[1][1] = 1.0;
ds.set_data(data);
objective_gradient = nse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(3, 4, 5);
nn.randomize_parameters_normal();
network_parameters = nn.arrange_parameters();
ds.set(3, 5, 2);
ds.randomize_data_normal();
objective_gradient = nse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(nse, &NormalizedSquaredError::calculate_performance, network_parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void RootMeanSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<double> network_parameters;
DataSet ds;
RootMeanSquaredError rmse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> numerical_objective_gradient;
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
ds.initialize_data(0.0);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(1.0);
network_parameters = nn.arrange_parameters();
ds.set(3, 2, 5);
ds.initialize_data(1.0);
objective_gradient = rmse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(1,1,1);
network_parameters = nn.arrange_parameters();
ds.set(1,1,1);
ds.initialize_data(1.0);
rmse.set_neural_network_pointer(&nn);
objective_gradient = rmse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void NumericalDifferentiationTest::test_calculate_central_differences_second_derivative(void)
{
message += "test_calculate_central_differences_second_derivative\n";
NumericalDifferentiation nd;
double x;
double d2;
// Test
x = 0.0;
d2 = nd.calculate_central_differences_second_derivative(*this, &NumericalDifferentiationTest::f1, x);
assert_true(fabs(d2) <= 1.0e-6, LOG);
}
void NumericalDifferentiationTest::test_calculate_central_differences_derivative(void)
{
message += "test_calculate_central_differences_derivative\n";
NumericalDifferentiation nd;
double x;
double d;
// Test
x = 0.0;
d = nd.calculate_central_differences_derivative(*this, &NumericalDifferentiationTest::f1, x);
assert_true(d == 1.0, LOG);
}
void NumericalDifferentiationTest::test_calculate_forward_differences_gradient(void)
{
message += "test_calculate_forward_differences_gradient\n";
NumericalDifferentiation nd;
Vector<double> x;
Vector<double> g;
// Test
x.set(2, 0.0);
g = nd.calculate_forward_differences_gradient(*this, &NumericalDifferentiationTest::f2, x);
assert_true(g.size() == 2, LOG);
assert_true(g == 1.0, LOG);
}
void NumericalDifferentiationTest::test_calculate_central_differences_Hessian_form(void)
{
message += "test_calculate_central_differences_Hessian_form\n";
NumericalDifferentiation nd;
Vector<double> x(2, 0.0);
Vector< Matrix<double> > Hessian = nd.calculate_central_differences_Hessian_form(*this, &NumericalDifferentiationTest::f3, x);
assert_true(Hessian.size() == 2, LOG);
assert_true(Hessian[0].get_rows_number() == 2, LOG);
assert_true(Hessian[0].get_columns_number() == 2, LOG);
assert_true(Hessian[0] == 0.0, LOG);
assert_true(Hessian[1].get_rows_number() == 2, LOG);
assert_true(Hessian[1].get_columns_number() == 2, LOG);
assert_true(Hessian[1] == 0.0, LOG);
}
void NumericalDifferentiationTest::test_calculate_forward_differences_Jacobian(void)
{
message += "test_calculate_forward_differences_Jacobian\n";
NumericalDifferentiation nd;
Vector<double> x;
Matrix<double> J;
Matrix<double> J_true;
// Test
x.set(2, 0.0);
J = nd.calculate_forward_differences_Jacobian(*this, &NumericalDifferentiationTest::f3, x);
J_true.set(2, 2);
J_true.initialize_identity();
assert_true(J == J_true, LOG);
}
void MeanSquaredErrorTest::test_calculate_Jacobian_terms(void)
{
message += "test_calculate_Jacobian_terms\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<unsigned> multilayer_perceptron_architecture;
Vector<double> parameters;
DataSet ds;
MeanSquaredError mse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> evaluation_terms;
Matrix<double> terms_Jacobian;
Matrix<double> numerical_Jacobian_terms;
// Test
nn.set(1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
terms_Jacobian = mse.calculate_terms_Jacobian();
assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
assert_true(terms_Jacobian == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
terms_Jacobian = mse.calculate_terms_Jacobian();
assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
assert_true(terms_Jacobian == 0.0, LOG);
// Test
multilayer_perceptron_architecture.set(3);
multilayer_perceptron_architecture[0] = 2;
multilayer_perceptron_architecture[1] = 1;
multilayer_perceptron_architecture[2] = 2;
nn.set(multilayer_perceptron_architecture);
nn.initialize_parameters(0.0);
ds.set(2, 2, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
terms_Jacobian = mse.calculate_terms_Jacobian();
assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
assert_true(terms_Jacobian == 0.0, LOG);
// Test
nn.set(1, 1, 1);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(1, 1, 1);
ds.randomize_data_normal();
terms_Jacobian = mse.calculate_terms_Jacobian();
numerical_Jacobian_terms = nd.calculate_Jacobian(mse, &MeanSquaredError::calculate_terms, parameters);
assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(2, 2, 2);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(2, 2, 2);
ds.randomize_data_normal();
terms_Jacobian = mse.calculate_terms_Jacobian();
numerical_Jacobian_terms = nd.calculate_Jacobian(mse, &MeanSquaredError::calculate_terms, parameters);
assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(2, 2, 2);
nn.randomize_parameters_normal();
ds.set(2, 2, 2);
ds.randomize_data_normal();
objective_gradient = mse.calculate_gradient();
evaluation_terms = mse.calculate_terms();
terms_Jacobian = mse.calculate_terms_Jacobian();
assert_true(((terms_Jacobian.calculate_transpose()).dot(evaluation_terms)*2.0 - objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void MeanSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<unsigned> multilayer_perceptron_architecture;
Vector<double> parameters;
DataSet ds;
MeanSquaredError mse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> numerical_objective_gradient;
Vector<double> numerical_differentiation_error;
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
multilayer_perceptron_architecture.set(3);
multilayer_perceptron_architecture[0] = 2;
multilayer_perceptron_architecture[1] = 1;
multilayer_perceptron_architecture[2] = 3;
nn.set(multilayer_perceptron_architecture);
nn.initialize_parameters(0.0);
ds.set(2, 3, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(1, 1);
nn.initialize_parameters(1.0);
parameters = nn.arrange_parameters();
ds.set(1, 1, 2);
ds.initialize_data(1.0);
objective_gradient = mse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(mse, &MeanSquaredError::calculate_performance, parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void NeuralParametersNormTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
NeuralParametersNorm npn(&nn);
Vector<size_t> architecture;
Vector<double> parameters;
Vector<double> gradient;
Vector<double> numerical_gradient;
Vector<double> error;
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 5;
architecture[1] = 1;
architecture[2] = 2;
nn.set(architecture);
nn.initialize_parameters(0.0);
npn.set_neural_network_pointer(&nn);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
npn.set_neural_network_pointer(&nn);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.initialize_parameters(1.0);
parameters = nn.arrange_parameters();
gradient = npn.calculate_gradient();
numerical_gradient = nd.calculate_gradient(npn, &NeuralParametersNorm::calculate_regularization, parameters);
error = (gradient - numerical_gradient).calculate_absolute_value();
assert_true(error < 1.0e-3, LOG);
}
void NormalizedSquaredErrorTest::test_calculate_Jacobian_terms(void)
{
message += "test_calculate_Jacobian_terms\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<int> hidden_layers_size;
Vector<double> network_parameters;
DataSet ds;
NormalizedSquaredError nse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> evaluation_terms;
Matrix<double> terms_Jacobian;
Matrix<double> numerical_Jacobian_terms;
// Test
nn.set(1, 1);
nn.randomize_parameters_normal();
network_parameters = nn.arrange_parameters();
ds.set(1, 1, 2);
ds.randomize_data_normal();
terms_Jacobian = nse.calculate_terms_Jacobian();
numerical_Jacobian_terms = nd.calculate_Jacobian(nse, &NormalizedSquaredError::calculate_terms, network_parameters);
assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(2, 2, 2);
nn.randomize_parameters_normal();
network_parameters = nn.arrange_parameters();
ds.set(2, 2, 2);
ds.randomize_data_normal();
terms_Jacobian = nse.calculate_terms_Jacobian();
numerical_Jacobian_terms = nd.calculate_Jacobian(nse, &NormalizedSquaredError::calculate_terms, network_parameters);
assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(2,2,2);
nn.randomize_parameters_normal();
ds.set(2,2,2);
ds.randomize_data_normal();
objective_gradient = nse.calculate_gradient();
evaluation_terms = nse.calculate_terms();
terms_Jacobian = nse.calculate_terms_Jacobian();
assert_true(((terms_Jacobian.calculate_transpose()).dot(evaluation_terms)*2.0 - objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void UnscalingLayerTest::test_calculate_derivatives(void)
{
message += "test_calculate_derivatives\n";
NumericalDifferentiation nd;
UnscalingLayer ul;
ul.set_display(false);
Vector<double> inputs;
Vector<double> derivative;
Vector<double> numerical_derivative;
// Test
ul.set(1);
ul.set_unscaling_method(UnscalingLayer::MinimumMaximum);
inputs.set(1, 0.0);
derivative = ul.calculate_derivatives(inputs);
assert_true(derivative == 1.0, LOG);
// Test
ul.set(1);
ul.set_unscaling_method(UnscalingLayer::MeanStandardDeviation);
inputs.set(1, 0.0);
derivative = ul.calculate_derivatives(inputs);
assert_true(derivative == 1.0, LOG);
// Test
if(numerical_differentiation_tests)
{
ul.set(3);
ul.initialize_random();
ul.set_unscaling_method(UnscalingLayer::MinimumMaximum);
inputs.set(3);
inputs.randomize_normal();
derivative = ul.calculate_derivatives(inputs);
numerical_derivative = nd.calculate_derivative(ul, &UnscalingLayer::calculate_outputs, inputs);
assert_true((derivative-numerical_derivative).calculate_absolute_value() < 1.0e-3, LOG);
}
// Test
if(numerical_differentiation_tests)
{
ul.set(3);
ul.initialize_random();
ul.set_unscaling_method(UnscalingLayer::MeanStandardDeviation);
inputs.set(3);
inputs.randomize_normal();
derivative = ul.calculate_derivatives(inputs);
numerical_derivative = nd.calculate_derivative(ul, &UnscalingLayer::calculate_outputs, inputs);
assert_true((derivative-numerical_derivative).calculate_absolute_value() < 1.0e-3, LOG);
}
}
void MinkowskiErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<size_t> architecture;
Vector<double> parameters;
DataSet ds;
MinkowskiError me(&nn, &ds);
Vector<double> gradient;
Vector<double> numerical_gradient;
// Test
nn.set(1,1,1);
nn.initialize_parameters(0.0);
ds.set(1,1,1);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3,4,2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 2;
architecture[1] = 1;
architecture[2] = 3;
nn.set(architecture);
nn.initialize_parameters(0.0);
ds.set(2, 3, 5);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(1,1,1);
nn.initialize_parameters(0.0);
ds.set(1,1,1);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3,4,2);
nn.initialize_parameters(0.0);
ds.set(3,2,5);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 2;
architecture[1] = 1;
architecture[2] = 2;
nn.set(architecture);
nn.initialize_parameters(0.0);
ds.set(2,2,3);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(4, 1);
nn.set(architecture);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(1,1,1);
ds.randomize_data_normal();
gradient = me.calculate_gradient();
numerical_gradient = nd.calculate_gradient(me, &MinkowskiError::calculate_error, parameters);
assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(5,4,3);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(2,5,3);
ds.randomize_data_normal();
me.set_Minkowski_parameter(1.75);
gradient = me.calculate_gradient();
numerical_gradient = nd.calculate_gradient(me, &MinkowskiError::calculate_error, parameters);
assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void SumSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
DataSet ds;
NeuralNetwork nn;
SumSquaredError sse(&nn, &ds);
Vector<size_t> architecture;
Vector<double> parameters;
Vector<double> gradient;
Vector<double> numerical_gradient;
Vector<double> error;
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 5;
architecture[1] = 1;
architecture[2] = 2;
nn.set(architecture);
nn.initialize_parameters(0.0);
ds.set(5, 5, 2);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 3, 2);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(2, 3, 4);
nn.initialize_parameters(0.0);
ds.set(2, 4, 5);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
for(unsigned i = 0; i < 100; i++)
{
ds.initialize_data(1.0);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
gradient.clear();
gradient = sse.calculate_gradient();
numerical_gradient = nd.calculate_gradient(sse, &SumSquaredError::calculate_error, parameters);
error = (gradient - numerical_gradient).calculate_absolute_value();
assert_true(error < 1.0e-3, LOG);
}
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(1.0);
parameters = nn.arrange_parameters();
ds.set(1, 1, 1);
ds.initialize_data(1.0);
gradient.clear();
gradient = sse.calculate_gradient();
numerical_gradient = nd.calculate_gradient(sse, &SumSquaredError::calculate_error, parameters);
assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG);
// Test
architecture.set(1000, 1);
nn.set(architecture);
nn.randomize_parameters_normal();
ds.set(10, 1, 1);
ds.randomize_data_normal();
sse.set(&nn, &ds);
gradient.clear();
gradient = sse.calculate_gradient();
}
void NeuralNetworkTest::test_calculate_Jacobian(void) {
message += "test_calculate_Jacobian\n";
// One layer
NeuralNetwork nn;
Vector<unsigned> multilayer_perceptron_architecture;
Vector<double> inputs;
Matrix<double> Jacobian;
// Vector<double> inputs_minimum;
// Vector<double> inputs_maximum;
// Vector<double> inputs_mean;
// Vector<double> inputs_standard_deviation;
// Vector<double> outputs_minimum;
// Vector<double> outputs_maximum;
// Vector<double> outputs_mean;
// Vector<double> outputs_standard_deviation;
// mmlp.set_display(false);
NumericalDifferentiation nd;
Matrix<double> numerical_Jacobian;
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
inputs.set(1, 0.0);
Jacobian = nn.calculate_Jacobian(inputs);
assert_true(Jacobian == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
inputs.set(3, 0.0);
Jacobian = nn.calculate_Jacobian(inputs);
assert_true(Jacobian == 0.0, LOG);
// Test
if (numerical_differentiation_tests) {
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
inputs.set(3, 0.0);
Jacobian = nn.calculate_Jacobian(inputs);
numerical_Jacobian =
nd.calculate_Jacobian(nn, &NeuralNetwork::calculate_outputs, inputs);
assert_true(
(Jacobian - numerical_Jacobian).calculate_absolute_value() < 1.0e-3,
LOG);
}
// Test
multilayer_perceptron_architecture.set(3, 1);
nn.set(multilayer_perceptron_architecture);
nn.initialize_parameters(0.0);
inputs.set(1, 0.0);
Jacobian = nn.calculate_Jacobian(inputs);
assert_true(Jacobian == 0.0, LOG);
// Test
multilayer_perceptron_architecture.set(3);
multilayer_perceptron_architecture[0] = 3;
multilayer_perceptron_architecture[1] = 4;
multilayer_perceptron_architecture[2] = 1;
nn.set(multilayer_perceptron_architecture);
nn.initialize_parameters(0.0);
inputs.set(3, 0.0);
Jacobian = nn.calculate_Jacobian(inputs);
assert_true(Jacobian == 0.0, LOG);
// Test
if (numerical_differentiation_tests) {
multilayer_perceptron_architecture.set(3);
multilayer_perceptron_architecture[0] = 3;
multilayer_perceptron_architecture[1] = 4;
multilayer_perceptron_architecture[2] = 1;
nn.set(multilayer_perceptron_architecture);
inputs.set(3);
inputs[0] = 0.0;
inputs[1] = 1.0;
inputs[2] = 2.0;
Jacobian = nn.calculate_Jacobian(inputs);
numerical_Jacobian =
nd.calculate_Jacobian(nn, &NeuralNetwork::calculate_outputs, inputs);
assert_true(
(Jacobian - numerical_Jacobian).calculate_absolute_value() < 1.0e-3,
LOG);
}
// Scaling and unscaling test
// if(numerical_differentiation_tests)
// {
// nn.set(2, 3);
// nn.set_variables_scaling_method(NeuralNetwork::MinimumMaximum);
// nn.set_input_minimum(0, -0.3);
// nn.set_input_minimum(1, -0.2);
// nn.set_input_maximum(0, 0.0);
// nn.set_input_maximum(1, 0.1);
// nn.set_output_minimum(0, -1.0);
// nn.set_output_minimum(1, -4.1);
// nn.set_output_minimum(2, -8.2);
// nn.set_output_maximum(0, 1.0);
// nn.set_output_maximum(1, 7.2);
// nn.set_output_maximum(2, 6.0);
// inputs.set(2);
// inputs.randomize_normal();
// Jacobian = nn.calculate_Jacobian(inputs);
// numerical_Jacobian = nd.calculate_Jacobian(nn,
// &NeuralNetwork::calculate_outputs, inputs);
// assert_true((Jacobian-numerical_Jacobian).calculate_absolute_value() <
// 1.0e-3, LOG);
// }
// Scaling and unscaling test
// if(numerical_differentiation_tests)
// {
// nn.set(2, 3);
// nn.set_variables_scaling_method(NeuralNetwork::MeanStandardDeviation);
// nn.set_input_mean(0, -0.3);
// nn.set_input_mean(1, -0.2);
// nn.set_input_standard_deviation(0, 0.2);
// nn.set_input_standard_deviation(1, 0.1);
// nn.set_output_mean(0, -1.0);
// nn.set_output_mean(1, -4.1);
// nn.set_output_mean(2, -8.2);
// nn.set_output_standard_deviation(0, 1.0);
// nn.set_output_standard_deviation(1, 7.2);
// nn.set_output_standard_deviation(2, 6.0);
// inputs.set(2);
// inputs.randomize_normal();
// Jacobian = nn.calculate_Jacobian(inputs);
// numerical_Jacobian = nd.calculate_Jacobian(nn,
// &NeuralNetwork::calculate_outputs, inputs);
// assert_true((Jacobian-numerical_Jacobian).calculate_absolute_value() <
// 1.0e-3, LOG);
// }
// Conditions test
// mmlp.set(1, 1, 1);
// mmlp.initialize_parameters(0.0);
// inputs.set(1, 0.0);
// Jacobian = mmlp.calculate_Jacobian(inputs);
// assert_true(Jacobian == 0.0, LOG);
// Conditions test
// Lower and upper bounds test
// Probabilistic postprocessing test
}
void SumSquaredErrorTest::test_calculate_terms_Jacobian(void)
{
message += "test_calculate_terms_Jacobian\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<size_t> architecture;
Vector<double> parameters;
DataSet ds;
SumSquaredError sse(&nn, &ds);
Vector<double> gradient;
Vector<double> terms;
Matrix<double> terms_Jacobian;
Matrix<double> numerical_Jacobian_terms;
// Test
nn.set(1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
terms_Jacobian = sse.calculate_terms_Jacobian();
assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().get_instances_number(), LOG);
assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
assert_true(terms_Jacobian == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
terms_Jacobian = sse.calculate_terms_Jacobian();
assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
assert_true(terms_Jacobian == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 5;
architecture[1] = 1;
architecture[2] = 2;
nn.set(architecture);
nn.initialize_parameters(0.0);
ds.set(5, 2, 3);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
terms_Jacobian = sse.calculate_terms_Jacobian();
assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG);
assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG);
assert_true(terms_Jacobian == 0.0, LOG);
// Test
nn.set(1, 1, 1);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(1, 1, 1);
ds.randomize_data_normal();
terms_Jacobian = sse.calculate_terms_Jacobian();
numerical_Jacobian_terms = nd.calculate_Jacobian(sse, &SumSquaredError::calculate_terms, parameters);
assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(2, 2, 2);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(2, 2, 2);
ds.randomize_data_normal();
terms_Jacobian = sse.calculate_terms_Jacobian();
numerical_Jacobian_terms = nd.calculate_Jacobian(sse, &SumSquaredError::calculate_terms, parameters);
assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(2, 2, 2);
nn.randomize_parameters_normal();
ds.set(2, 2, 2);
ds.randomize_data_normal();
gradient = sse.calculate_gradient();
terms = sse.calculate_terms();
terms_Jacobian = sse.calculate_terms_Jacobian();
assert_true(((terms_Jacobian.calculate_transpose()).dot(terms)*2.0 - gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void LevenbergMarquardtAlgorithmTest::test_calculate_Hessian_approximation(void)
{
message += "test_calculate_Hessian_approximation\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
size_t parameters_number;
Vector<double> parameters;
DataSet ds;
PerformanceFunctional pf(&nn, &ds);
pf.set_error_type(PerformanceFunctional::SUM_SQUARED_ERROR);
Matrix<double> terms_Jacobian;
Matrix<double> Hessian;
Matrix<double> numerical_Hessian;
Matrix<double> Hessian_approximation;
LevenbergMarquardtAlgorithm lma(&pf);
// Test
nn.set(1, 2);
nn.initialize_parameters(0.0);
parameters_number = nn.count_parameters_number();
ds.set(1,2,2);
ds.initialize_data(0.0);
terms_Jacobian = pf.calculate_terms_Jacobian();
Hessian_approximation = lma.calculate_Hessian_approximation(terms_Jacobian);
assert_true(Hessian_approximation.get_rows_number() == parameters_number, LOG);
assert_true(Hessian_approximation.get_columns_number() == parameters_number, LOG);
assert_true(Hessian_approximation.is_symmetric(), LOG);
// Test
pf.set_error_type(PerformanceFunctional::NORMALIZED_SQUARED_ERROR);
nn.set(1,1,2);
nn.randomize_parameters_normal();
parameters_number = nn.count_parameters_number();
ds.set(1,2,3);
ds.randomize_data_normal();
terms_Jacobian = pf.calculate_terms_Jacobian();
Hessian_approximation = lma.calculate_Hessian_approximation(terms_Jacobian);
assert_true(Hessian_approximation.get_rows_number() == parameters_number, LOG);
assert_true(Hessian_approximation.get_columns_number() == parameters_number, LOG);
assert_true(Hessian_approximation.is_symmetric(), LOG);
// Test
nn.set(2);
nn.randomize_parameters_normal();
MockErrorTerm* mptp = new MockErrorTerm(&nn);
pf.set_user_error_pointer(mptp);
terms_Jacobian = pf.calculate_terms_Jacobian();
Hessian = pf.calculate_Hessian();
lma.set_damping_parameter(0.0);
assert_true((lma.calculate_Hessian_approximation(terms_Jacobian) - Hessian).calculate_absolute_value() < 1.0e-3, LOG);
// Test
pf.set_error_type(PerformanceFunctional::SUM_SQUARED_ERROR);
ds.set(1, 1, 1);
ds.randomize_data_normal();
nn.set(1, 1);
parameters = nn.arrange_parameters();
nn.randomize_parameters_normal();
numerical_Hessian = nd.calculate_Hessian(pf, &PerformanceFunctional::calculate_performance, parameters);
terms_Jacobian = pf.calculate_terms_Jacobian();
Hessian_approximation = lma.calculate_Hessian_approximation(terms_Jacobian);
assert_true((numerical_Hessian - Hessian_approximation).calculate_absolute_value() >= 0.0, LOG);
}
