void RootMeanSquaredErrorTest::test_calculate_gradient(void) { message += "test_calculate_gradient\n"; NumericalDifferentiation nd; NeuralNetwork nn; Vector<double> network_parameters; DataSet ds; RootMeanSquaredError rmse(&nn, &ds); Vector<double> objective_gradient; Vector<double> numerical_objective_gradient; // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); ds.set(3, 2, 5); ds.initialize_data(0.0); // Test nn.set(3, 4, 2); nn.initialize_parameters(1.0); network_parameters = nn.arrange_parameters(); ds.set(3, 2, 5); ds.initialize_data(1.0); objective_gradient = rmse.calculate_gradient(); numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters); assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(1,1,1); network_parameters = nn.arrange_parameters(); ds.set(1,1,1); ds.initialize_data(1.0); rmse.set_neural_network_pointer(&nn); objective_gradient = rmse.calculate_gradient(); numerical_objective_gradient = nd.calculate_gradient(rmse, &RootMeanSquaredError::calculate_performance, network_parameters); assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG); }
void NeuralParametersNormTest::test_calculate_performance(void) { message += "test_calculate_performance\n"; NeuralNetwork nn; Vector<double> neural_parameters; NeuralParametersNorm npn(&nn); Vector<double> parameters; double performance; // Test nn.set(1, 1); nn.initialize_parameters(0.0); performance = npn.calculate_regularization(); assert_true(performance == 0.0, LOG); // Test nn.set(1, 1); nn.initialize_parameters(3.1415927); parameters = nn.arrange_parameters(); assert_true(npn.calculate_regularization() == npn.calculate_regularization(parameters), LOG); }
void NeuralNetworkTest::test_arrange_parameters(void) { message += "test_arrange_parameters\n"; NeuralNetwork nn; Vector<double> parameters; IndependentParameters* ip; // Test nn.set(); parameters = nn.arrange_parameters(); assert_true(parameters.size() == 0, LOG); // Test nn.set(1, 1, 1); ip = new IndependentParameters(1); nn.set_independent_parameters_pointer(ip); nn.initialize_parameters(0.0); parameters = nn.arrange_parameters(); assert_true(parameters.size() == 5, LOG); assert_true(parameters == 0.0, LOG); // Test nn.set(); ip = new IndependentParameters(1); nn.set_independent_parameters_pointer(ip); nn.initialize_parameters(0.0); parameters = nn.arrange_parameters(); assert_true(parameters.size() == 1, LOG); assert_true(parameters == 0.0, LOG); // Test nn.set(1, 1, 1); ip = new IndependentParameters(1); nn.set_independent_parameters_pointer(ip); nn.initialize_parameters(0.0); parameters = nn.arrange_parameters(); assert_true(parameters.size() == 5, LOG); assert_true(parameters == 0.0, LOG); }
void NormalizedSquaredErrorTest::test_calculate_gradient(void) { message += "test_calculate_gradient\n"; NumericalDifferentiation nd; NeuralNetwork nn; Vector<double> network_parameters; DataSet ds; Matrix<double> data; NormalizedSquaredError nse(&nn, &ds); Vector<double> objective_gradient; Vector<double> numerical_objective_gradient; // Test nn.set(1,1,1); nn.initialize_parameters(0.0); ds.set(1, 1, 2); data.set(2, 2); data[0][0] = -1.0; data[0][1] = -1.0; data[1][0] = 1.0; data[1][1] = 1.0; ds.set_data(data); objective_gradient = nse.calculate_gradient(); assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG); assert_true(objective_gradient == 0.0, LOG); // Test nn.set(3, 4, 5); nn.randomize_parameters_normal(); network_parameters = nn.arrange_parameters(); ds.set(3, 5, 2); ds.randomize_data_normal(); objective_gradient = nse.calculate_gradient(); numerical_objective_gradient = nd.calculate_gradient(nse, &NormalizedSquaredError::calculate_performance, network_parameters); assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG); }
void NeuralNetworkTest::test_randomize_parameters_normal(void) { message += "test_randomize_parameters_normal\n"; NeuralNetwork nn; Vector<double> network_parameters; // Test nn.set(1, 1, 1); nn.randomize_parameters_normal(1.0, 0.0); network_parameters = nn.arrange_parameters(); assert_true(network_parameters == 1.0, LOG); }
void NeuralNetworkTest::test_randomize_parameters_uniform(void) { message += "test_randomize_parameters_uniform\n"; NeuralNetwork nn; Vector<double> parameters; // Test nn.set(1, 1, 1); nn.randomize_parameters_uniform(); parameters = nn.arrange_parameters(); assert_true(parameters >= -1.0, LOG); assert_true(parameters <= 1.0, LOG); }
void NeuralNetworkTest::test_set_parameters(void) { message += "test_set_parameters\n"; Vector<unsigned> multilayer_perceptron_architecture; NeuralNetwork nn; unsigned parameters_number; Vector<double> parameters; // Test nn.set_parameters(parameters); parameters = nn.arrange_parameters(); assert_true(parameters.size() == 0, LOG); // Test multilayer_perceptron_architecture.set(2, 2); nn.set(multilayer_perceptron_architecture); nn.construct_independent_parameters(); nn.get_independent_parameters_pointer()->set_parameters_number(2); parameters_number = nn.count_parameters_number(); parameters.set(0.0, 1.0, parameters_number - 1); nn.set_parameters(parameters); parameters = nn.arrange_parameters(); assert_true(parameters.size() == parameters_number, LOG); assert_true(parameters[0] == 0.0, LOG); assert_true(parameters[parameters_number - 1] == parameters_number - 1.0, LOG); }
void PerformanceFunctionalTest::test_calculate_gradient(void) { message += "test_calculate_gradient\n"; NeuralNetwork nn; size_t parameters_number; Vector<double> parameters; PerformanceFunctional pf(&nn); pf.destruct_all_terms(); pf.set_regularization_type(PerformanceFunctional::NEURAL_PARAMETERS_NORM_REGULARIZATION); Vector<double> gradient; // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); parameters = nn.arrange_parameters(); gradient = pf.calculate_gradient(parameters); assert_true(gradient == 0.0, LOG); // Test parameters_number = nn.count_parameters_number(); nn.initialize_parameters(0.0); MockPerformanceTerm* mptp = new MockPerformanceTerm(&nn); pf.set_user_objective_pointer(mptp); gradient = pf.calculate_gradient(); assert_true(gradient.size() == parameters_number, LOG); assert_true(gradient == 0.0, LOG); }
void NeuralNetworkTest::test_initialize_parameters(void) { message += "test_initialize_parameters\n"; NeuralNetwork nn; Vector<double> parameters; IndependentParameters* ip; // Test nn.set(1, 1, 1); nn.construct_independent_parameters(); ip = nn.get_independent_parameters_pointer(); ip->set_parameters_number(1); nn.randomize_parameters_normal(1.0, 0.0); parameters = nn.arrange_parameters(); assert_true(parameters == 1.0, LOG); }
void MeanSquaredErrorTest::test_calculate_Jacobian_terms(void) { message += "test_calculate_Jacobian_terms\n"; NumericalDifferentiation nd; NeuralNetwork nn; Vector<unsigned> multilayer_perceptron_architecture; Vector<double> parameters; DataSet ds; MeanSquaredError mse(&nn, &ds); Vector<double> objective_gradient; Vector<double> evaluation_terms; Matrix<double> terms_Jacobian; Matrix<double> numerical_Jacobian_terms; // Test nn.set(1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); terms_Jacobian = mse.calculate_terms_Jacobian(); assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG); assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG); assert_true(terms_Jacobian == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); ds.set(3, 2, 5); mse.set(&nn, &ds); ds.initialize_data(0.0); terms_Jacobian = mse.calculate_terms_Jacobian(); assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG); assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG); assert_true(terms_Jacobian == 0.0, LOG); // Test multilayer_perceptron_architecture.set(3); multilayer_perceptron_architecture[0] = 2; multilayer_perceptron_architecture[1] = 1; multilayer_perceptron_architecture[2] = 2; nn.set(multilayer_perceptron_architecture); nn.initialize_parameters(0.0); ds.set(2, 2, 5); mse.set(&nn, &ds); ds.initialize_data(0.0); terms_Jacobian = mse.calculate_terms_Jacobian(); assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG); assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG); assert_true(terms_Jacobian == 0.0, LOG); // Test nn.set(1, 1, 1); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.randomize_data_normal(); terms_Jacobian = mse.calculate_terms_Jacobian(); numerical_Jacobian_terms = nd.calculate_Jacobian(mse, &MeanSquaredError::calculate_terms, parameters); assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(2, 2, 2); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(2, 2, 2); ds.randomize_data_normal(); terms_Jacobian = mse.calculate_terms_Jacobian(); numerical_Jacobian_terms = nd.calculate_Jacobian(mse, &MeanSquaredError::calculate_terms, parameters); assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(2, 2, 2); nn.randomize_parameters_normal(); ds.set(2, 2, 2); ds.randomize_data_normal(); objective_gradient = mse.calculate_gradient(); evaluation_terms = mse.calculate_terms(); terms_Jacobian = mse.calculate_terms_Jacobian(); assert_true(((terms_Jacobian.calculate_transpose()).dot(evaluation_terms)*2.0 - objective_gradient).calculate_absolute_value() < 1.0e-3, LOG); }
void MeanSquaredErrorTest::test_calculate_gradient(void) { message += "test_calculate_gradient\n"; NumericalDifferentiation nd; NeuralNetwork nn; Vector<unsigned> multilayer_perceptron_architecture; Vector<double> parameters; DataSet ds; MeanSquaredError mse(&nn, &ds); Vector<double> objective_gradient; Vector<double> numerical_objective_gradient; Vector<double> numerical_differentiation_error; // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); objective_gradient = mse.calculate_gradient(); assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG); assert_true(objective_gradient == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); ds.set(3, 2, 5); mse.set(&nn, &ds); ds.initialize_data(0.0); objective_gradient = mse.calculate_gradient(); assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG); assert_true(objective_gradient == 0.0, LOG); // Test multilayer_perceptron_architecture.set(3); multilayer_perceptron_architecture[0] = 2; multilayer_perceptron_architecture[1] = 1; multilayer_perceptron_architecture[2] = 3; nn.set(multilayer_perceptron_architecture); nn.initialize_parameters(0.0); ds.set(2, 3, 5); mse.set(&nn, &ds); ds.initialize_data(0.0); objective_gradient = mse.calculate_gradient(); assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG); assert_true(objective_gradient == 0.0, LOG); // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); objective_gradient = mse.calculate_gradient(); assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG); assert_true(objective_gradient == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); ds.set(3, 2, 5); mse.set(&nn, &ds); ds.initialize_data(0.0); objective_gradient = mse.calculate_gradient(); assert_true(objective_gradient.size() == nn.count_parameters_number(), LOG); assert_true(objective_gradient == 0.0, LOG); // Test nn.set(1, 1); nn.initialize_parameters(1.0); parameters = nn.arrange_parameters(); ds.set(1, 1, 2); ds.initialize_data(1.0); objective_gradient = mse.calculate_gradient(); numerical_objective_gradient = nd.calculate_gradient(mse, &MeanSquaredError::calculate_performance, parameters); assert_true((objective_gradient - numerical_objective_gradient).calculate_absolute_value() < 1.0e-3, LOG); }
void NeuralNetworkTest::test_calculate_outputs(void) { message += "test_calculate_outputs\n"; NeuralNetwork nn; unsigned inputs_number; unsigned outputs_number; Vector<unsigned> architecture; Vector<double> inputs; Vector<double> outputs; unsigned parameters_number; Vector<double> parameters; // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); inputs.set(3, 0.0); outputs = nn.calculate_outputs(inputs); assert_true(outputs == 0.0, LOG); // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); inputs.set(1, 0.0); outputs = nn.calculate_outputs(inputs); assert_true(outputs == 0.0, LOG); // Test nn.set(1, 1); inputs.set(1); inputs.randomize_normal(); parameters = nn.arrange_parameters(); assert_true( nn.calculate_outputs(inputs) == nn.calculate_outputs(inputs, parameters), LOG); // Test nn.set(4, 3, 5); inputs.set(4, 0.0); parameters_number = nn.count_parameters_number(); parameters.set(parameters_number, 0.0); outputs = nn.calculate_outputs(inputs, parameters); assert_true(outputs.size() == 5, LOG); assert_true(outputs == 0.0, LOG); // Test architecture.set(5); architecture.randomize_uniform(5, 10); nn.set(architecture); inputs_number = nn.get_inputs_pointer()->get_inputs_number(); outputs_number = nn.get_outputs_pointer()->get_outputs_number(); inputs.set(inputs_number, 0.0); parameters_number = nn.count_parameters_number(); parameters.set(parameters_number, 0.0); outputs = nn.calculate_outputs(inputs, parameters); assert_true(outputs.size() == outputs_number, LOG); assert_true(outputs == 0.0, LOG); }
void PerformanceFunctionalTest::test_calculate_performance(void) { message += "test_calculate_performance\n"; DataSet ds; NeuralNetwork nn; Vector<double> parameters; PerformanceFunctional pf(&nn); double performance; Vector<double> direction; double rate; // Test pf.destruct_all_terms(); pf.set_regularization_type(PerformanceFunctional::NEURAL_PARAMETERS_NORM_REGULARIZATION); NeuralParametersNorm* neural_parameters_norm = pf.get_neural_parameters_norm_regularization_pointer(); double neural_parameters_norm_weight = neural_parameters_norm->get_neural_parameters_norm_weight(); nn.set(1, 1); nn.initialize_parameters(1.0); parameters = nn.arrange_parameters(); assert_true(fabs(pf.calculate_performance() - neural_parameters_norm_weight*sqrt(2.0)) < 1.0e-3, LOG); assert_true(fabs(pf.calculate_performance() - pf.calculate_performance(parameters)) < 1.0e-3, LOG); // Test parameters = nn.arrange_parameters(); assert_true(pf.calculate_performance() != pf.calculate_performance(parameters*2.0), LOG); // Test direction.set(2, -0.5); rate = 2.0; assert_true(pf.calculate_performance(direction, rate) == 0.0, LOG); // Test parameters = nn.arrange_parameters(); direction.set(2, -1.5); rate = 2.3; assert_true(pf.calculate_performance(direction, rate) == pf.calculate_performance(parameters + direction*rate), LOG); // Test ds.set(1, 1, 1); ds.randomize_data_normal(); pf.set_data_set_pointer(&ds); pf.destruct_all_terms(); pf.set_objective_type(PerformanceFunctional::SUM_SQUARED_ERROR_OBJECTIVE); nn.set(1, 1); nn.initialize_parameters(1.0); parameters = nn.arrange_parameters(); assert_true(fabs(pf.calculate_performance() - pf.calculate_performance(parameters)) < 1.0e-3, LOG); // Test parameters = nn.arrange_parameters(); assert_true(pf.calculate_performance() != pf.calculate_performance(parameters*2.0), LOG); // Test parameters = nn.arrange_parameters(); direction.set(2, -1.5); rate = 2.3; assert_true(pf.calculate_performance(direction, rate) == pf.calculate_performance(parameters + direction*rate), LOG); // Test nn.initialize_parameters(0.0); MockPerformanceTerm* mptp = new MockPerformanceTerm(&nn); pf.set_user_objective_pointer(mptp); performance = pf.calculate_performance(); assert_true(performance == 0.0, LOG); }
void SumSquaredErrorTest::test_calculate_terms_Jacobian(void) { message += "test_calculate_terms_Jacobian\n"; NumericalDifferentiation nd; NeuralNetwork nn; Vector<size_t> architecture; Vector<double> parameters; DataSet ds; SumSquaredError sse(&nn, &ds); Vector<double> gradient; Vector<double> terms; Matrix<double> terms_Jacobian; Matrix<double> numerical_Jacobian_terms; // Test nn.set(1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); terms_Jacobian = sse.calculate_terms_Jacobian(); assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().get_instances_number(), LOG); assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG); assert_true(terms_Jacobian == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); ds.set(3, 2, 5); sse.set(&nn, &ds); ds.initialize_data(0.0); terms_Jacobian = sse.calculate_terms_Jacobian(); assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG); assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG); assert_true(terms_Jacobian == 0.0, LOG); // Test architecture.set(3); architecture[0] = 5; architecture[1] = 1; architecture[2] = 2; nn.set(architecture); nn.initialize_parameters(0.0); ds.set(5, 2, 3); sse.set(&nn, &ds); ds.initialize_data(0.0); terms_Jacobian = sse.calculate_terms_Jacobian(); assert_true(terms_Jacobian.get_rows_number() == ds.get_instances().count_training_instances_number(), LOG); assert_true(terms_Jacobian.get_columns_number() == nn.count_parameters_number(), LOG); assert_true(terms_Jacobian == 0.0, LOG); // Test nn.set(1, 1, 1); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.randomize_data_normal(); terms_Jacobian = sse.calculate_terms_Jacobian(); numerical_Jacobian_terms = nd.calculate_Jacobian(sse, &SumSquaredError::calculate_terms, parameters); assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(2, 2, 2); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(2, 2, 2); ds.randomize_data_normal(); terms_Jacobian = sse.calculate_terms_Jacobian(); numerical_Jacobian_terms = nd.calculate_Jacobian(sse, &SumSquaredError::calculate_terms, parameters); assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(2, 2, 2); nn.randomize_parameters_normal(); ds.set(2, 2, 2); ds.randomize_data_normal(); gradient = sse.calculate_gradient(); terms = sse.calculate_terms(); terms_Jacobian = sse.calculate_terms_Jacobian(); assert_true(((terms_Jacobian.calculate_transpose()).dot(terms)*2.0 - gradient).calculate_absolute_value() < 1.0e-3, LOG); }
void SumSquaredErrorTest::test_calculate_performance(void) { message += "test_calculate_performance\n"; NeuralNetwork nn; Vector<double> parameters; DataSet ds; Matrix<double> data; SumSquaredError sse(&nn, &ds); double performance; // Test nn.set(1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); performance = sse.calculate_performance(); assert_true(performance == 0.0, LOG); // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(1.0); performance = sse.calculate_performance(); assert_true(performance == 1.0, LOG); // Test nn.set(1, 1); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.randomize_data_normal(); assert_true( sse.calculate_performance() == sse.calculate_performance(parameters), LOG); // Test nn.set(1, 1); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.randomize_data_normal(); assert_true(sse.calculate_performance() != sse.calculate_performance(parameters * 2.0), LOG); }
void TrainingRateAlgorithmTest::test_calculate_fixed_directional_point(void) { message += "test_calculate_fixed_directional_point\n"; NeuralNetwork nn; Vector<double> parameters; PerformanceFunctional pf(&nn); pf.destruct_all_terms(); pf.set_regularization_type( PerformanceFunctional::NEURAL_PARAMETERS_NORM_REGULARIZATION); double performance; Vector<double> gradient; TrainingRateAlgorithm tra(&pf); Vector<double> training_direction; double training_rate; Vector<double> directional_point; // Test nn.set(1, 1); nn.initialize_parameters(1.0); performance = pf.calculate_performance(); gradient = pf.calculate_gradient(); training_direction = gradient * (-1.0); training_rate = 0.001; directional_point = tra.calculate_fixed_directional_point( performance, training_direction, training_rate); assert_true(directional_point.size() == 2, LOG); assert_true(directional_point[1] < performance, LOG); assert_true(directional_point[1] == pf.calculate_performance(training_direction, training_rate), LOG); parameters = nn.arrange_parameters(); nn.set_parameters(parameters + training_direction * training_rate); assert_true(directional_point[1] == pf.calculate_performance(), LOG); // Test nn.set(1, 1); nn.initialize_parameters(1.0); training_direction.set(2, -1.0); training_rate = 1.0; directional_point = tra.calculate_fixed_directional_point( 3.14, training_direction, training_rate); assert_true(directional_point.size() == 2, LOG); assert_true(directional_point[0] == 1.0, LOG); assert_true(directional_point[1] == 0.0, LOG); }
void LevenbergMarquardtAlgorithmTest::test_calculate_Hessian_approximation(void) { message += "test_calculate_Hessian_approximation\n"; NumericalDifferentiation nd; NeuralNetwork nn; size_t parameters_number; Vector<double> parameters; DataSet ds; PerformanceFunctional pf(&nn, &ds); pf.set_error_type(PerformanceFunctional::SUM_SQUARED_ERROR); Matrix<double> terms_Jacobian; Matrix<double> Hessian; Matrix<double> numerical_Hessian; Matrix<double> Hessian_approximation; LevenbergMarquardtAlgorithm lma(&pf); // Test nn.set(1, 2); nn.initialize_parameters(0.0); parameters_number = nn.count_parameters_number(); ds.set(1,2,2); ds.initialize_data(0.0); terms_Jacobian = pf.calculate_terms_Jacobian(); Hessian_approximation = lma.calculate_Hessian_approximation(terms_Jacobian); assert_true(Hessian_approximation.get_rows_number() == parameters_number, LOG); assert_true(Hessian_approximation.get_columns_number() == parameters_number, LOG); assert_true(Hessian_approximation.is_symmetric(), LOG); // Test pf.set_error_type(PerformanceFunctional::NORMALIZED_SQUARED_ERROR); nn.set(1,1,2); nn.randomize_parameters_normal(); parameters_number = nn.count_parameters_number(); ds.set(1,2,3); ds.randomize_data_normal(); terms_Jacobian = pf.calculate_terms_Jacobian(); Hessian_approximation = lma.calculate_Hessian_approximation(terms_Jacobian); assert_true(Hessian_approximation.get_rows_number() == parameters_number, LOG); assert_true(Hessian_approximation.get_columns_number() == parameters_number, LOG); assert_true(Hessian_approximation.is_symmetric(), LOG); // Test nn.set(2); nn.randomize_parameters_normal(); MockErrorTerm* mptp = new MockErrorTerm(&nn); pf.set_user_error_pointer(mptp); terms_Jacobian = pf.calculate_terms_Jacobian(); Hessian = pf.calculate_Hessian(); lma.set_damping_parameter(0.0); assert_true((lma.calculate_Hessian_approximation(terms_Jacobian) - Hessian).calculate_absolute_value() < 1.0e-3, LOG); // Test pf.set_error_type(PerformanceFunctional::SUM_SQUARED_ERROR); ds.set(1, 1, 1); ds.randomize_data_normal(); nn.set(1, 1); parameters = nn.arrange_parameters(); nn.randomize_parameters_normal(); numerical_Hessian = nd.calculate_Hessian(pf, &PerformanceFunctional::calculate_performance, parameters); terms_Jacobian = pf.calculate_terms_Jacobian(); Hessian_approximation = lma.calculate_Hessian_approximation(terms_Jacobian); assert_true((numerical_Hessian - Hessian_approximation).calculate_absolute_value() >= 0.0, LOG); }
void NeuralParametersNormTest::test_calculate_gradient(void) { message += "test_calculate_gradient\n"; NumericalDifferentiation nd; NeuralNetwork nn; NeuralParametersNorm npn(&nn); Vector<size_t> architecture; Vector<double> parameters; Vector<double> gradient; Vector<double> numerical_gradient; Vector<double> error; // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); gradient = npn.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); gradient = npn.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test architecture.set(3); architecture[0] = 5; architecture[1] = 1; architecture[2] = 2; nn.set(architecture); nn.initialize_parameters(0.0); npn.set_neural_network_pointer(&nn); gradient = npn.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); npn.set_neural_network_pointer(&nn); gradient = npn.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.initialize_parameters(1.0); parameters = nn.arrange_parameters(); gradient = npn.calculate_gradient(); numerical_gradient = nd.calculate_gradient(npn, &NeuralParametersNorm::calculate_regularization, parameters); error = (gradient - numerical_gradient).calculate_absolute_value(); assert_true(error < 1.0e-3, LOG); }
void MinkowskiErrorTest::test_calculate_gradient(void) { message += "test_calculate_gradient\n"; NumericalDifferentiation nd; NeuralNetwork nn; Vector<size_t> architecture; Vector<double> parameters; DataSet ds; MinkowskiError me(&nn, &ds); Vector<double> gradient; Vector<double> numerical_gradient; // Test nn.set(1,1,1); nn.initialize_parameters(0.0); ds.set(1,1,1); ds.initialize_data(0.0); gradient = me.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(3,4,2); nn.initialize_parameters(0.0); ds.set(3, 2, 5); me.set(&nn, &ds); ds.initialize_data(0.0); gradient = me.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test architecture.set(3); architecture[0] = 2; architecture[1] = 1; architecture[2] = 3; nn.set(architecture); nn.initialize_parameters(0.0); ds.set(2, 3, 5); me.set(&nn, &ds); ds.initialize_data(0.0); gradient = me.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(1,1,1); nn.initialize_parameters(0.0); ds.set(1,1,1); ds.initialize_data(0.0); gradient = me.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(3,4,2); nn.initialize_parameters(0.0); ds.set(3,2,5); me.set(&nn, &ds); ds.initialize_data(0.0); gradient = me.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test architecture.set(3); architecture[0] = 2; architecture[1] = 1; architecture[2] = 2; nn.set(architecture); nn.initialize_parameters(0.0); ds.set(2,2,3); me.set(&nn, &ds); ds.initialize_data(0.0); gradient = me.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test architecture.set(4, 1); nn.set(architecture); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1,1,1); ds.randomize_data_normal(); gradient = me.calculate_gradient(); numerical_gradient = nd.calculate_gradient(me, &MinkowskiError::calculate_error, parameters); assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(5,4,3); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(2,5,3); ds.randomize_data_normal(); me.set_Minkowski_parameter(1.75); gradient = me.calculate_gradient(); numerical_gradient = nd.calculate_gradient(me, &MinkowskiError::calculate_error, parameters); assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG); }
void SumSquaredErrorTest::test_calculate_gradient(void) { message += "test_calculate_gradient\n"; NumericalDifferentiation nd; DataSet ds; NeuralNetwork nn; SumSquaredError sse(&nn, &ds); Vector<size_t> architecture; Vector<double> parameters; Vector<double> gradient; Vector<double> numerical_gradient; Vector<double> error; // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); gradient = sse.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); ds.set(3, 2, 5); sse.set(&nn, &ds); ds.initialize_data(0.0); gradient.clear(); gradient = sse.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test architecture.set(3); architecture[0] = 5; architecture[1] = 1; architecture[2] = 2; nn.set(architecture); nn.initialize_parameters(0.0); ds.set(5, 5, 2); sse.set(&nn, &ds); ds.initialize_data(0.0); gradient.clear(); gradient = sse.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); gradient.clear(); gradient = sse.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(3, 4, 2); nn.initialize_parameters(0.0); ds.set(3, 3, 2); sse.set(&nn, &ds); ds.initialize_data(0.0); gradient.clear(); gradient = sse.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test nn.set(2, 3, 4); nn.initialize_parameters(0.0); ds.set(2, 4, 5); sse.set(&nn, &ds); ds.initialize_data(0.0); gradient.clear(); gradient = sse.calculate_gradient(); assert_true(gradient.size() == nn.count_parameters_number(), LOG); assert_true(gradient == 0.0, LOG); // Test for(unsigned i = 0; i < 100; i++) { ds.initialize_data(1.0); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); gradient.clear(); gradient = sse.calculate_gradient(); numerical_gradient = nd.calculate_gradient(sse, &SumSquaredError::calculate_error, parameters); error = (gradient - numerical_gradient).calculate_absolute_value(); assert_true(error < 1.0e-3, LOG); } // Test nn.set(1, 1, 1); nn.initialize_parameters(1.0); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.initialize_data(1.0); gradient.clear(); gradient = sse.calculate_gradient(); numerical_gradient = nd.calculate_gradient(sse, &SumSquaredError::calculate_error, parameters); assert_true((gradient - numerical_gradient).calculate_absolute_value() < 1.0e-3, LOG); // Test architecture.set(1000, 1); nn.set(architecture); nn.randomize_parameters_normal(); ds.set(10, 1, 1); ds.randomize_data_normal(); sse.set(&nn, &ds); gradient.clear(); gradient = sse.calculate_gradient(); }
void NormalizedSquaredErrorTest::test_calculate_Jacobian_terms(void) { message += "test_calculate_Jacobian_terms\n"; NumericalDifferentiation nd; NeuralNetwork nn; Vector<int> hidden_layers_size; Vector<double> network_parameters; DataSet ds; NormalizedSquaredError nse(&nn, &ds); Vector<double> objective_gradient; Vector<double> evaluation_terms; Matrix<double> terms_Jacobian; Matrix<double> numerical_Jacobian_terms; // Test nn.set(1, 1); nn.randomize_parameters_normal(); network_parameters = nn.arrange_parameters(); ds.set(1, 1, 2); ds.randomize_data_normal(); terms_Jacobian = nse.calculate_terms_Jacobian(); numerical_Jacobian_terms = nd.calculate_Jacobian(nse, &NormalizedSquaredError::calculate_terms, network_parameters); assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(2, 2, 2); nn.randomize_parameters_normal(); network_parameters = nn.arrange_parameters(); ds.set(2, 2, 2); ds.randomize_data_normal(); terms_Jacobian = nse.calculate_terms_Jacobian(); numerical_Jacobian_terms = nd.calculate_Jacobian(nse, &NormalizedSquaredError::calculate_terms, network_parameters); assert_true((terms_Jacobian-numerical_Jacobian_terms).calculate_absolute_value() < 1.0e-3, LOG); // Test nn.set(2,2,2); nn.randomize_parameters_normal(); ds.set(2,2,2); ds.randomize_data_normal(); objective_gradient = nse.calculate_gradient(); evaluation_terms = nse.calculate_terms(); terms_Jacobian = nse.calculate_terms_Jacobian(); assert_true(((terms_Jacobian.calculate_transpose()).dot(evaluation_terms)*2.0 - objective_gradient).calculate_absolute_value() < 1.0e-3, LOG); }
void SumSquaredErrorTest::test_calculate_loss(void) { message += "test_calculate_loss\n"; NeuralNetwork nn; Vector<double> parameters; DataSet ds; Matrix<double> data; MissingValues* missing_values_pointer = ds.get_missing_values_pointer(); SumSquaredError sse(&nn, &ds); double loss; // Test nn.set(1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(0.0); loss = sse.calculate_error(); assert_true(loss == 0.0, LOG); // Test nn.set(1, 1, 1); nn.initialize_parameters(0.0); ds.set(1, 1, 1); ds.initialize_data(1.0); loss = sse.calculate_error(); assert_true(loss == 1.0, LOG); // Test nn.set(1, 1); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.randomize_data_normal(); assert_true(sse.calculate_error() == sse.calculate_error(parameters), LOG); // Test nn.set(1, 1); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.randomize_data_normal(); assert_true(sse.calculate_error() != sse.calculate_error(parameters*2.0), LOG); // Test nn.set(1, 1); nn.randomize_parameters_normal(); parameters = nn.arrange_parameters(); ds.set(1, 1, 1); ds.randomize_data_normal(); missing_values_pointer->append(0, 0); // assert_true(sse.calculate_loss() == 0.0, LOG); }
void PerformanceFunctionalTest::test_calculate_Hessian(void) { message += "test_calculate_Hessian\n"; NeuralNetwork nn; unsigned parameters_number; Vector<double> parameters; PerformanceFunctional pf(&nn); pf.destruct_all_terms(); pf.set_regularization_type( PerformanceFunctional::NEURAL_PARAMETERS_NORM_REGULARIZATION); Matrix<double> Hessian; nn.set(1, 1, 1); nn.initialize_parameters(0.0); parameters_number = nn.count_parameters_number(); parameters = nn.arrange_parameters(); Hessian = pf.calculate_Hessian(parameters); assert_true(Hessian.get_rows_number() == parameters_number, LOG); assert_true(Hessian.get_columns_number() == parameters_number, LOG); nn.set(); nn.initialize_parameters(0.0); parameters_number = nn.count_parameters_number(); parameters = nn.arrange_parameters(); Hessian = pf.calculate_Hessian(parameters); assert_true(Hessian.get_rows_number() == parameters_number, LOG); assert_true(Hessian.get_columns_number() == parameters_number, LOG); nn.set(1, 1); nn.initialize_parameters(0.0); parameters_number = nn.count_parameters_number(); parameters = nn.arrange_parameters(); Hessian = pf.calculate_Hessian(parameters); assert_true(Hessian.get_rows_number() == parameters_number, LOG); assert_true(Hessian.get_columns_number() == parameters_number, LOG); // Test parameters_number = nn.count_parameters_number(); nn.initialize_parameters(0.0); MockPerformanceTerm* mptp = new MockPerformanceTerm(&nn); pf.set_user_objective_pointer(mptp); Hessian = pf.calculate_Hessian(); assert_true(Hessian.get_rows_number() == parameters_number, LOG); assert_true(Hessian.get_columns_number() == parameters_number, LOG); }