GoldenSectionOrder::GoldenSectionOrderResults* GoldenSectionOrder::perform_order_selection(void)
{
GoldenSectionOrderResults* results = new GoldenSectionOrderResults();
NeuralNetwork* neural_network_pointer = training_strategy_pointer->get_performance_functional_pointer()->get_neural_network_pointer();
MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
Vector<double> mu_performance(2);
Vector<double> ln_performance(2);
Vector<double> a_parameters;
Vector<double> ln_parameters;
Vector<double> mu_parameters;
Vector<double> b_parameters;
bool end = false;
Vector<double> minimums(4);
double minimum;
size_t iterations = 0;
double current_training_performance, current_generalization_performance;
time_t beginning_time, current_time;
double elapsed_time;
size_t a = minimum_order;
size_t b = maximum_order;
size_t ln = (int)(a+(1.-0.618)*(b-a));
size_t mu = (int)(a+0.618*(b-a));
if (display)
std::cout << "Performing order selection with golden section method..." << std::endl;
time(&beginning_time);
mu_performance = calculate_performances(mu);
current_training_performance = mu_performance[0];
current_generalization_performance = mu_performance[1];
mu_parameters = get_parameters_order(mu);
results->order_data.push_back(mu);
if (reserve_performance_data)
{
results->performance_data.push_back(current_training_performance);
}
if (reserve_generalization_performance_data)
{
results->generalization_performance_data.push_back(current_generalization_performance);
}
if (reserve_parameters_data)
{
results->parameters_data.push_back(mu_parameters);
}
ln_performance = calculate_performances(ln);
current_training_performance = ln_performance[0];
current_generalization_performance = ln_performance[1];
ln_parameters = get_parameters_order(ln);
results->order_data.push_back(ln);
if (reserve_performance_data)
{
results->performance_data.push_back(current_training_performance);
}
if (reserve_generalization_performance_data)
{
results->generalization_performance_data.push_back(current_generalization_performance);
}
if (reserve_parameters_data)
{
results->parameters_data.push_back(ln_parameters);
}
time(¤t_time);
elapsed_time = difftime(current_time, beginning_time);
if (display)
{
std::cout << "Initial values : " << std::endl;
std::cout << "a = " << a << " ln = " << ln << " mu = " << mu << " b = " << b << std::endl;
std::cout << "ln final training performance : " << ln_performance[0] << std::endl;
std::cout << "ln final generalization performance : " << ln_performance[1] << std::endl;
std::cout << "mu final training performance : " << mu_performance[0] << std::endl;
std::cout << "mu final generalization performance : " << mu_performance[1] << std::endl;
std::cout << "Elapsed time : " << elapsed_time << std::endl;
}
if ((ln == mu) || (ln > mu) || (mu < ln)){
end = true;
if (display)
std::cout << "Algorithm finished " << std::endl;
results->stopping_condition = GoldenSectionOrder::AlgorithmFinished;
}
while(!end){
if (ln_performance[1] < mu_performance[1]
|| fabs(ln_performance[1] - mu_performance[1]) < tolerance)
{
b = mu;
mu = ln;
mu_performance = ln_performance;
ln = (int)(a+(1.-0.618)*(b-a));
ln_performance = calculate_performances(ln);
current_training_performance = ln_performance[0];
current_generalization_performance = ln_performance[1];
ln_parameters = get_parameters_order(ln);
results->order_data.push_back(ln);
if (reserve_performance_data)
{
results->performance_data.push_back(current_training_performance);
}
if (reserve_generalization_performance_data)
{
results->generalization_performance_data.push_back(current_generalization_performance);
}
if (reserve_parameters_data)
{
results->parameters_data.push_back(ln_parameters);
}
}else
{
a = ln;
ln = mu;
ln_performance = mu_performance;
mu = (int)(a+0.618*(b-a));
mu_performance = calculate_performances(mu);
current_training_performance = mu_performance[0];
current_generalization_performance = mu_performance[1];
mu_parameters = get_parameters_order(mu);
results->order_data.push_back(mu);
if (reserve_performance_data)
{
results->performance_data.push_back(current_training_performance);
}
if (reserve_generalization_performance_data)
{
results->generalization_performance_data.push_back(current_generalization_performance);
}
if (reserve_parameters_data)
{
results->parameters_data.push_back(mu_parameters);
}
}
time(¤t_time);
elapsed_time = difftime(current_time, beginning_time);
iterations++;
// Stopping criteria
if ((ln == mu) || (ln > mu)){
end = true;
if (display)
std::cout << "Algorithm finished " << std::endl;
results->stopping_condition = GoldenSectionOrder::AlgorithmFinished;
}else if (elapsed_time > maximum_time)
{
end = true;
if (display)
std::cout << "Maximum time reached." << std::endl;
results->stopping_condition = GoldenSectionOrder::MaximumTime;
}else if (fmin(ln_performance[1],mu_performance[1]) < selection_performance_goal)
{
end = true;
if (display)
std::cout << "Generalization performance reached." << std::endl;
results->stopping_condition = GoldenSectionOrder::SelectionPerformanceGoal;
}else if (iterations > maximum_iterations_number)
{
end = true;
if (display)
std::cout << "Maximum number of iterations reached." << std::endl;
results->stopping_condition = GoldenSectionOrder::MaximumIterations;
}
if (display && !end)
{
std::cout << "Iteration : " << iterations << std::endl;
std::cout << "a = " << a << " ln = " << ln << " mu = " << mu << " b = " << b << std::endl;
std::cout << "ln final training performance : " << ln_performance[0] << std::endl;
std::cout << "ln final generalization performance : " << ln_performance[1] << std::endl;
std::cout << "mu final training performance : " << mu_performance[0] << std::endl;
std::cout << "mu final generalization performance : " << mu_performance[1] << std::endl;
std::cout << "Elapsed time : " << elapsed_time << std::endl;
}
}
minimums[0] = calculate_performances(a)[1];
a_parameters = get_parameters_order(a);
minimums[1] = calculate_performances(ln)[1];
ln_parameters = get_parameters_order(ln);
minimums[2] = calculate_performances(mu)[1];
mu_parameters = get_parameters_order(mu);
minimums[3] = calculate_performances(b)[1];
b_parameters = get_parameters_order(b);
time(¤t_time);
elapsed_time = difftime(current_time, beginning_time);
if (display)
{
std::cout << "Iteration : " << iterations << std::endl;
std::cout << "a = " << a << " ln = " << ln << " mu = " << mu << " b = " << b << std::endl;
std::cout << "a final training performance : " << calculate_performances(a)[0] << std::endl;
std::cout << "a final generalization performance : " << calculate_performances(a)[1] << std::endl;
std::cout << "ln final training performance : " << ln_performance[0] << std::endl;
std::cout << "ln final generalization performance : " << ln_performance[1] << std::endl;
std::cout << "mu final training performance : " << mu_performance[0] << std::endl;
std::cout << "mu final generalization performance : " << mu_performance[1] << std::endl;
std::cout << "b final training performance : " << calculate_performances(b)[0] << std::endl;
std::cout << "b final generalization performance : " << calculate_performances(b)[1] << std::endl;
std::cout << "Elapsed time : " << elapsed_time << std::endl;
}
minimum = minimums.calculate_minimum();
if (fabs(minimums[0] - minimum) < tolerance)
{
if (display)
std::cout << "Optimal order : " << a << std::endl;
multilayer_perceptron_pointer->set(inputs_number, a, outputs_number);
multilayer_perceptron_pointer->set_parameters(a_parameters);
if (reserve_minimal_parameters)
results->minimal_parameters = a_parameters;
results->optimal_order = a;
results->final_generalization_performance = minimums[0];
results->final_performance = calculate_performances(a)[0];
}else if (fabs(minimums[1] - minimum) < tolerance)
{
if (display)
std::cout << "Optimal order : " << ln << std::endl;
multilayer_perceptron_pointer->set(inputs_number, ln, outputs_number);
multilayer_perceptron_pointer->set_parameters(ln_parameters);
if (reserve_minimal_parameters)
results->minimal_parameters = ln_parameters;
results->optimal_order = ln;
results->final_generalization_performance = minimums[1];
results->final_performance = calculate_performances(ln)[0];
}else if(fabs(minimums[2] - minimum) < tolerance)
{
if (display)
std::cout << "Optimal order : " << mu << std::endl;
multilayer_perceptron_pointer->set(inputs_number, mu, outputs_number);
multilayer_perceptron_pointer->set_parameters(mu_parameters);
if (reserve_minimal_parameters)
results->minimal_parameters = mu_parameters;
results->optimal_order = mu;
results->final_generalization_performance = minimums[2];
results->final_performance = calculate_performances(mu)[0];
}else
{
if (display)
std::cout << "Optimal order : " << b << std::endl;
multilayer_perceptron_pointer->set(inputs_number, b, outputs_number);
multilayer_perceptron_pointer->set_parameters(b_parameters);
if (reserve_minimal_parameters)
results->minimal_parameters = b_parameters;
results->optimal_order = b;
results->final_generalization_performance = minimums[3];
results->final_performance = calculate_performances(b)[0];
}
results->elapsed_time = elapsed_time;
results->iterations_number = iterations;
return(results);
}
IncrementalOrder::IncrementalOrderResults* IncrementalOrder::perform_order_selection(void)
{
IncrementalOrderResults* results = new IncrementalOrderResults();
NeuralNetwork* neural_network_pointer = training_strategy_pointer->get_performance_functional_pointer()->get_neural_network_pointer();
MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();
Vector<double> performance(2);
double prev_generalization_performance = 1.0e99;
size_t optimal_order;
Vector<double> optimum_parameters;
double optimum_generalization_performance;
Vector<double> parameters_history_row;
double current_training_performance, current_generalization_performance;
size_t order = minimum_order;
size_t iterations = 0;
size_t selection_failures = 0;
bool end = false;
time_t beginning_time, current_time;
double elapsed_time;
if (display)
std::cout << "Performing Incremental order selection..." << std::endl;
time(&beginning_time);
while (!end)
{
performance = calculate_performances(order);
current_training_performance = performance[0];
current_generalization_performance = performance[1];
time(¤t_time);
elapsed_time = difftime(current_time, beginning_time);
results->order_data.push_back(order);
if (reserve_performance_data)
{
results->performance_data.push_back(current_training_performance);
}
if (reserve_generalization_performance_data)
{
results->generalization_performance_data.push_back(current_generalization_performance);
}
if (reserve_parameters_data)
{
parameters_history_row = get_parameters_order(order);
results->parameters_data.push_back(parameters_history_row);
}
if (iterations == 0
|| (optimum_generalization_performance > current_generalization_performance
&& fabs(optimum_generalization_performance - current_generalization_performance) > tolerance))
{
optimal_order = order;
optimum_generalization_performance = current_generalization_performance;
optimum_parameters = get_parameters_order(optimal_order);
}else if (prev_generalization_performance < current_generalization_performance)
selection_failures++;
prev_generalization_performance = current_generalization_performance;
iterations++;
// Stopping criteria
if (elapsed_time > maximum_time)
{
end = true;
if (display)
std::cout << "Maximum time reached." << std::endl;
results->stopping_condition = IncrementalOrder::MaximumTime;
}else if (performance[1] < selection_performance_goal)
{
end = true;
if (display)
std::cout << "Generalization performance reached." << std::endl;
results->stopping_condition = IncrementalOrder::SelectionPerformanceGoal;
}else if (iterations > maximum_iterations_number)
{
end = true;
if (display)
std::cout << "Maximum number of iterations reached." << std::endl;
results->stopping_condition = IncrementalOrder::MaximumIterations;
}else if (selection_failures >= maximum_selection_failures)
{
end = true;
if (display)
std::cout << "Maximum generalization performance failures("<<selection_failures<<") reached." << std::endl;
results->stopping_condition = IncrementalOrder::MaximumSelectionFailures;
}else if (order == maximum_order)
{
end = true;
if (display)
std::cout << "Algorithm finished" << std::endl;
results->stopping_condition = IncrementalOrder::AlgorithmFinished;
}
if (display)
{
std::cout << "Iteration : " << iterations << std::endl;
std::cout << "Hidden Perceptron Number : " << order << std::endl;
std::cout << "Final Training Performance : " << performance[0] << std::endl;
std::cout << "Final Generalization Performance : " << performance[1] << std::endl;
std::cout << "Elapsed time : " << elapsed_time << std::endl;
}
if (!end)
order = std::min(maximum_order, order+step);
}
if (display)
std::cout << "Optimal order : " << optimal_order << std:: endl;
multilayer_perceptron_pointer->set(inputs_number, optimal_order, outputs_number);
multilayer_perceptron_pointer->set_parameters(optimum_parameters);
if (reserve_minimal_parameters)
results->minimal_parameters = optimum_parameters;
results->optimal_order = optimal_order;
results->final_generalization_performance = optimum_generalization_performance;
results->final_performance = calculate_performances(optimal_order)[0];
results->iterations_number = iterations;
results->elapsed_time = elapsed_time;
return(results);
}
SimulatedAnnealingOrder::SimulatedAnnealingOrderResults* SimulatedAnnealingOrder::perform_order_selection(void)
{
SimulatedAnnealingOrderResults* results = new SimulatedAnnealingOrderResults();
NeuralNetwork* neural_network_pointer = training_strategy_pointer->get_performance_functional_pointer()->get_neural_network_pointer();
MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
size_t optimal_order, current_order;
Vector<double> optimum_performance(2);
Vector<double> current_order_performance(2);
Vector<double> optimum_parameters, current_parameters;
double current_training_performance, current_selection_performance;
bool end = false;
size_t iterations = 0;
size_t selection_failures = 0;
size_t random_failures = 0;
size_t upper_bound;
size_t lower_bound;
time_t beginning_time, current_time;
double elapsed_time;
double temperature;
double boltzmann_probability;
double random_uniform;
if (display)
{
std::cout << "Performing order selection with simulated annealing method..." << std::endl;
std::cout.flush();
}
time(&beginning_time);
optimal_order = (size_t)(minimum_order +
calculate_random_uniform(0.,1.)*(maximum_order - minimum_order));
optimum_performance = calculate_performances(optimal_order);
optimum_parameters = get_parameters_order(optimal_order);
current_training_performance = optimum_performance[0];
current_selection_performance = optimum_performance[1];
temperature = current_selection_performance;
results->order_data.push_back(optimal_order);
if (reserve_performance_data)
{
results->performance_data.push_back(current_training_performance);
}
if (reserve_selection_performance_data)
{
results->selection_performance_data.push_back(current_selection_performance);
}
if (reserve_parameters_data)
{
results->parameters_data.push_back(optimum_parameters);
}
time(¤t_time);
elapsed_time = difftime(current_time, beginning_time);
if (display)
{
std::cout << "Initial values : " << std::endl;
std::cout << "Hidden perceptrons : " << optimal_order << std::endl;
std::cout << "Final Training Performance : " << optimum_performance[0] << std::endl;
std::cout << "Final selection performance : " << optimum_performance[1] << std::endl;
std::cout << "Temperature : " << temperature << std::endl;
std::cout << "Elapsed time : " << elapsed_time << std::endl;
}
while (!end){
upper_bound = std::min(maximum_order, optimal_order + (maximum_order-minimum_order)/3);
if (optimal_order <= (maximum_order-minimum_order)/3)
lower_bound = minimum_order;
else
lower_bound = optimal_order - (maximum_order-minimum_order)/3;
current_order = (size_t)(lower_bound + calculate_random_uniform(0.,1.)*(upper_bound - lower_bound));
while (current_order == optimal_order)
{
current_order = (size_t)(lower_bound + calculate_random_uniform(0.,1.)*(upper_bound - lower_bound));
random_failures++;
if (random_failures >= 5 && optimal_order != minimum_order)
current_order = optimal_order - 1;
else if (random_failures >= 5 && optimal_order != maximum_order)
current_order = optimal_order + 1;
}
random_failures = 0;
current_order_performance = calculate_performances(current_order);
current_training_performance = current_order_performance[0];
current_selection_performance = current_order_performance[1];
current_parameters = get_parameters_order(current_order);
boltzmann_probability = fmin(1, exp(-(current_selection_performance-optimum_performance[1])/temperature));
random_uniform = calculate_random_uniform(0.,1.);
if ((boltzmann_probability <= random_uniform)
|| (fabs(optimum_performance[1]-current_selection_performance) <= tolerance
&& current_order >= optimal_order))
// Selection failures
{
selection_failures++;
}else
// Selection success
{
optimal_order = current_order;
optimum_performance = current_order_performance;
optimum_parameters = get_parameters_order(optimal_order);
}
time(¤t_time);
elapsed_time = difftime(current_time, beginning_time);
results->order_data.push_back(current_order);
if (reserve_performance_data)
{
results->performance_data.push_back(current_training_performance);
}
if (reserve_selection_performance_data)
{
results->selection_performance_data.push_back(current_selection_performance);
}
if (reserve_parameters_data)
{
results->parameters_data.push_back(current_parameters);
}
temperature = cooling_rate*temperature;
iterations++;
// Stopping criteria
if (temperature <= minimum_temperature)
{
end = true;
if (display)
std::cout << "Minimum temperature reached." << std::endl;
results->stopping_condition = SimulatedAnnealingOrder::MinimumTemperature;
}else if (elapsed_time > maximum_time)
{
end = true;
if (display)
std::cout << "Maximum time reached." << std::endl;
results->stopping_condition = SimulatedAnnealingOrder::MaximumTime;
}else if (optimum_performance[1] <= selection_performance_goal)
{
end = true;
if (display)
std::cout << "Selection performance reached." << std::endl;
results->stopping_condition = SimulatedAnnealingOrder::SelectionPerformanceGoal;
}else if (selection_failures >= maximum_selection_failures)
{
end = true;
if (display)
std::cout << "Maximum selection performance failures("<<selection_failures<<") reached." << std::endl;
results->stopping_condition = SimulatedAnnealingOrder::MaximumSelectionFailures;
}else if (iterations >= maximum_iterations_number)
{
end = true;
if (display)
std::cout << "Maximum number of iterations reached." << std::endl;
results->stopping_condition = SimulatedAnnealingOrder::MaximumIterations;
}
if (display)
{
std::cout << "Iteration : " << iterations << std::endl;
std::cout << "Hidden neurons number : " << optimal_order << std::endl;
std::cout << "Training performance : " << optimum_performance[0] << std::endl;
std::cout << "Selection performance : " << optimum_performance[1] << std::endl;
std::cout << "Current temperature : " << temperature << std::endl;
std::cout << "Elapsed time : " << elapsed_time << std::endl;
}
}
size_t optimal_index = get_optimal_selection_performance_index();
optimal_order = order_history[optimal_index] ;
optimum_performance[0] = performance_history[optimal_index];
optimum_performance[1] = selection_performance_history[optimal_index];
optimum_parameters = get_parameters_order(optimal_order);
if (display)
{
std::cout << "Optimal order : " << optimal_order << std::endl;
std::cout << "Optimum training Performance : " << optimum_performance[0] << std::endl;
std::cout << "Optimum selection performance : " << optimum_performance[1] << std::endl;
}
const size_t last_hidden_layer = multilayer_perceptron_pointer->get_layers_number()-2;
const size_t perceptrons_number = multilayer_perceptron_pointer->get_layer_pointer(last_hidden_layer)->get_perceptrons_number();
if (optimal_order > perceptrons_number)
{
multilayer_perceptron_pointer->grow_layer_perceptron(last_hidden_layer,optimal_order-perceptrons_number);
}else
{
for (size_t i = 0; i < (perceptrons_number-optimal_order); i++)
multilayer_perceptron_pointer->prune_layer_perceptron(last_hidden_layer,0);
}
multilayer_perceptron_pointer->set_parameters(optimum_parameters);
if (reserve_minimal_parameters)
results->minimal_parameters = optimum_parameters;
results->optimal_order = optimal_order;
results->final_performance = optimum_performance[0];
results->final_selection_performance = optimum_performance[1];
results->elapsed_time = elapsed_time;
results->iterations_number = iterations;
return(results);
}