void NumericalDifferentiationTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
Vector<double> x;
Vector<double> g;
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::ForwardDifferences);
x.set(2, 0.0);
g = nd.calculate_gradient(*this, &NumericalDifferentiationTest::f2, x);
assert_true(g.size() == 2, LOG);
assert_true(g == 1.0, LOG);
// Test
nd.set_numerical_differentiation_method(NumericalDifferentiation::CentralDifferences);
x.set(2, 0.0);
g = nd.calculate_gradient(*this, &NumericalDifferentiationTest::f2, x);
assert_true(g.size() == 2, LOG);
assert_true(g == 1.0, LOG);
}
void RootMeanSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<double> network_parameters;
DataSet ds;
RootMeanSquaredError rmse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> numerical_objective_gradient;
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
ds.initialize_data(0.0);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(1.0);
network_parameters = nn.arrange_parameters();
ds.set(3, 2, 5);
ds.initialize_data(1.0);
objective_gradient = rmse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(1,1,1);
network_parameters = nn.arrange_parameters();
ds.set(1,1,1);
ds.initialize_data(1.0);
rmse.set_neural_network_pointer(&nn);
objective_gradient = rmse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void NormalizedSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<double> network_parameters;
DataSet ds;
Matrix<double> data;
NormalizedSquaredError nse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> numerical_objective_gradient;
// Test
nn.set(1,1,1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 2);
data.set(2, 2);
data[0][0] = -1.0;
data[0][1] = -1.0;
data[1][0] = 1.0;
data[1][1] = 1.0;
ds.set_data(data);
objective_gradient = nse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(3, 4, 5);
nn.randomize_parameters_normal();
network_parameters = nn.arrange_parameters();
ds.set(3, 5, 2);
ds.randomize_data_normal();
objective_gradient = nse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(nse, &NormalizedSquaredError::calculate_performance, network_parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void SumSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
DataSet ds;
NeuralNetwork nn;
SumSquaredError sse(&nn, &ds);
Vector<size_t> architecture;
Vector<double> parameters;
Vector<double> gradient;
Vector<double> numerical_gradient;
Vector<double> error;
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 5;
architecture[1] = 1;
architecture[2] = 2;
nn.set(architecture);
nn.initialize_parameters(0.0);
ds.set(5, 5, 2);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 3, 2);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(2, 3, 4);
nn.initialize_parameters(0.0);
ds.set(2, 4, 5);
sse.set(&nn, &ds);
ds.initialize_data(0.0);
gradient.clear();
gradient = sse.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
for(unsigned i = 0; i < 100; i++)
{
ds.initialize_data(1.0);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
gradient.clear();
gradient = sse.calculate_gradient();
numerical_gradient = nd.calculate_gradient(sse, &SumSquaredError::calculate_error, parameters);
error = (gradient - numerical_gradient).calculate_absolute_value();
assert_true(error < 1.0e-3, LOG);
}
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(1.0);
parameters = nn.arrange_parameters();
ds.set(1, 1, 1);
ds.initialize_data(1.0);
gradient.clear();
gradient = sse.calculate_gradient();
numerical_gradient = nd.calculate_gradient(sse, &SumSquaredError::calculate_error, parameters);
assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG);
// Test
architecture.set(1000, 1);
nn.set(architecture);
nn.randomize_parameters_normal();
ds.set(10, 1, 1);
ds.randomize_data_normal();
sse.set(&nn, &ds);
gradient.clear();
gradient = sse.calculate_gradient();
}
void MeanSquaredErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<unsigned> multilayer_perceptron_architecture;
Vector<double> parameters;
DataSet ds;
MeanSquaredError mse(&nn, &ds);
Vector<double> objective_gradient;
Vector<double> numerical_objective_gradient;
Vector<double> numerical_differentiation_error;
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
multilayer_perceptron_architecture.set(3);
multilayer_perceptron_architecture[0] = 2;
multilayer_perceptron_architecture[1] = 1;
multilayer_perceptron_architecture[2] = 3;
nn.set(multilayer_perceptron_architecture);
nn.initialize_parameters(0.0);
ds.set(2, 3, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
ds.set(1, 1, 1);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
mse.set(&nn, &ds);
ds.initialize_data(0.0);
objective_gradient = mse.calculate_gradient();
assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG);
assert_true(objective_gradient == 0.0, LOG);
// Test
nn.set(1, 1);
nn.initialize_parameters(1.0);
parameters = nn.arrange_parameters();
ds.set(1, 1, 2);
ds.initialize_data(1.0);
objective_gradient = mse.calculate_gradient();
numerical_objective_gradient = nd.calculate_gradient(mse, &MeanSquaredError::calculate_performance, parameters);
assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
void NeuralParametersNormTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
NeuralParametersNorm npn(&nn);
Vector<size_t> architecture;
Vector<double> parameters;
Vector<double> gradient;
Vector<double> numerical_gradient;
Vector<double> error;
// Test
nn.set(1, 1, 1);
nn.initialize_parameters(0.0);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 5;
architecture[1] = 1;
architecture[2] = 2;
nn.set(architecture);
nn.initialize_parameters(0.0);
npn.set_neural_network_pointer(&nn);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3, 4, 2);
nn.initialize_parameters(0.0);
npn.set_neural_network_pointer(&nn);
gradient = npn.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.initialize_parameters(1.0);
parameters = nn.arrange_parameters();
gradient = npn.calculate_gradient();
numerical_gradient = nd.calculate_gradient(npn, &NeuralParametersNorm::calculate_regularization, parameters);
error = (gradient - numerical_gradient).calculate_absolute_value();
assert_true(error < 1.0e-3, LOG);
}
void MinkowskiErrorTest::test_calculate_gradient(void)
{
message += "test_calculate_gradient\n";
NumericalDifferentiation nd;
NeuralNetwork nn;
Vector<size_t> architecture;
Vector<double> parameters;
DataSet ds;
MinkowskiError me(&nn, &ds);
Vector<double> gradient;
Vector<double> numerical_gradient;
// Test
nn.set(1,1,1);
nn.initialize_parameters(0.0);
ds.set(1,1,1);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3,4,2);
nn.initialize_parameters(0.0);
ds.set(3, 2, 5);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 2;
architecture[1] = 1;
architecture[2] = 3;
nn.set(architecture);
nn.initialize_parameters(0.0);
ds.set(2, 3, 5);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(1,1,1);
nn.initialize_parameters(0.0);
ds.set(1,1,1);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
nn.set(3,4,2);
nn.initialize_parameters(0.0);
ds.set(3,2,5);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(3);
architecture[0] = 2;
architecture[1] = 1;
architecture[2] = 2;
nn.set(architecture);
nn.initialize_parameters(0.0);
ds.set(2,2,3);
me.set(&nn, &ds);
ds.initialize_data(0.0);
gradient = me.calculate_gradient();
assert_true(gradient.size() == nn.count_parameters_number(), LOG);
assert_true(gradient == 0.0, LOG);
// Test
architecture.set(4, 1);
nn.set(architecture);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(1,1,1);
ds.randomize_data_normal();
gradient = me.calculate_gradient();
numerical_gradient = nd.calculate_gradient(me, &MinkowskiError::calculate_error, parameters);
assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG);
// Test
nn.set(5,4,3);
nn.randomize_parameters_normal();
parameters = nn.arrange_parameters();
ds.set(2,5,3);
ds.randomize_data_normal();
me.set_Minkowski_parameter(1.75);
gradient = me.calculate_gradient();
numerical_gradient = nd.calculate_gradient(me, &MinkowskiError::calculate_error, parameters);
assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG);
}
