Vector<double> TestingAnalysis::calculate_binary_classification_tests(void) const
{
// Control sentence (if debug)
#ifndef NDEBUG
const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();
const unsigned inputs_number = multilayer_perceptron_pointer->get_inputs_number();
if(!data_set_pointer)
{
std::ostringstream buffer;
buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
<< "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
<< "Data set is NULL." << std::endl;
throw std::logic_error(buffer.str());
}
const Variables& variables = data_set_pointer->get_variables();
const unsigned targets_number = variables.count_targets_number();
const unsigned outputs_number = multilayer_perceptron_pointer->get_outputs_number();
// Control sentence
if(inputs_number != variables.count_inputs_number())
{
std::ostringstream buffer;
buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
<< "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
<< "Number of inputs in neural network is not equal to number of inputs in data set." << std::endl;
throw std::logic_error(buffer.str());
}
else if(outputs_number != 1)
{
std::ostringstream buffer;
buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
<< "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
<< "Number of outputs in neural network must be one." << std::endl;
throw std::logic_error(buffer.str());
}
else if(targets_number != 1)
{
std::ostringstream buffer;
buffer << "OpenNN Exception: TestingAnalysis class." << std::endl
<< "Vector<double> calculate_binary_classification_tests(void) const." << std::endl
<< "Number of targets in data set must be one." << std::endl;
throw std::logic_error(buffer.str());
}
#endif
// Confusion matrix
const Matrix<unsigned> confusion = calculate_confusion();
const unsigned true_positive = confusion[0][0];
const unsigned false_positive = confusion[0][1];
const unsigned false_negative = confusion[1][0];
const unsigned true_negative = confusion[1][1];
// Classification accuracy
double classification_accuracy;
if(true_positive + true_negative + false_positive + false_negative == 0)
{
classification_accuracy = 0.0;
}
else
{
classification_accuracy = (double)(true_positive + true_negative)/double(true_positive + true_negative + false_positive + false_negative);
}
// Error rate
double error_rate;
if(true_positive + true_negative + false_positive + false_negative == 0)
{
error_rate = 0.0;
}
else
{
error_rate = (double)(false_positive + false_negative)/(double)(true_positive + true_negative + false_positive + false_negative);
}
// Sensitivity
double sensitivity;
if(true_positive + false_negative == 0)
{
sensitivity = 0.0;
}
else
{
sensitivity = (double)true_positive/(double)(true_positive + false_negative);
}
// Specifity
double specifity;
if(true_negative + false_positive == 0)
{
specifity = 0.0;
}
else
{
specifity = (double)true_negative/(double)(true_negative + false_positive);
}
// Positive likelihood
double positive_likelihood;
if(classification_accuracy == 1.0)
{
positive_likelihood = 1.0;
}
else if(1.0 - specifity == 0.0)
{
positive_likelihood = 0.0;
}
else
{
positive_likelihood = sensitivity/(1.0 - specifity);
}
// Negative likelihood
double negative_likelihood;
if(classification_accuracy == 1.0)
{
negative_likelihood = 1.0;
}
else if(1.0 - sensitivity == 0.0)
{
negative_likelihood = 0.0;
}
else
{
negative_likelihood = specifity/(1.0 - sensitivity);
}
// Arrange vector
Vector<double> binary_classification_test(6);
binary_classification_test[0] = classification_accuracy;
binary_classification_test[1] = error_rate;
binary_classification_test[2] = sensitivity;
binary_classification_test[3] = specifity;
binary_classification_test[4] = positive_likelihood;
binary_classification_test[5] = negative_likelihood;
return(binary_classification_test);
}
KappaCoefficientOptimizationThreshold::KappaCoefficientOptimizationThresholdResults* KappaCoefficientOptimizationThreshold::perform_threshold_selection(void)
{
#ifdef __OPENNN_DEBUG__
check();
#endif
KappaCoefficientOptimizationThresholdResults* results = new KappaCoefficientOptimizationThresholdResults();
const LossIndex* loss_index_pointer = training_strategy_pointer->get_loss_index_pointer();
NeuralNetwork* neural_network_pointer = loss_index_pointer->get_neural_network_pointer();
double current_threshold = minimum_threshold;
Matrix<size_t> current_confusion;
Vector<double> current_binary_classification_test;
double current_kappa_coefficient;
double po, pe, prA, prB;
const size_t instances_number = loss_index_pointer->get_data_set_pointer()->get_instances_pointer()->count_selection_instances_number();
double optimum_threshold;
Vector<double> optimal_binary_classification_test(15,1);
double optimum_kappa_coefficient = 0.0;
size_t iterations = 0;
bool end = false;
while (!end)
{
current_confusion = calculate_confusion(current_threshold);
current_binary_classification_test = calculate_binary_classification_test(current_confusion);
po = (current_confusion(0,0) + current_confusion(1,1))/(double)instances_number;
prA = (current_confusion(0,0) + current_confusion(0,1))/(double)instances_number;
prB = (current_confusion(0,0) + current_confusion(1,0))/(double)instances_number;
pe = prA*prB + (1-prA)*(1-prB);
current_kappa_coefficient = (po-pe)/(1-pe);
results->threshold_data.push_back(current_threshold);
if(reserve_binary_classification_tests_data)
{
results->binary_classification_test_data.push_back(current_binary_classification_test);
}
if(reserve_function_data)
{
results->function_data.push_back(current_kappa_coefficient);
}
if (current_kappa_coefficient > optimum_kappa_coefficient ||
(current_kappa_coefficient == optimum_kappa_coefficient && current_binary_classification_test[1] < optimal_binary_classification_test[1]))
{
optimum_kappa_coefficient = current_kappa_coefficient;
optimum_threshold = current_threshold;
optimal_binary_classification_test.set(current_binary_classification_test);
}
iterations++;
if (current_confusion(0,1) == 0 && current_confusion(1,0) == 0)
{
end = true;
if(display)
{
std::cout << "Perfect confusion matrix reached." << std::endl;
}
results->stopping_condition = ThresholdSelectionAlgorithm::PerfectConfusionMatrix;
}
else if (current_threshold == maximum_threshold)
{
end = true;
if(display)
{
std::cout << "Algorithm finished." << std::endl;
}
results->stopping_condition = ThresholdSelectionAlgorithm::AlgorithmFinished;
}
if (display)
{
std::cout << "Iteration: " << iterations << std::endl;
std::cout << "Current threshold: " << current_threshold << std::endl;
std::cout << "Current error: " << current_binary_classification_test[1] << std::endl;
std::cout << "Current sensitivity: " << current_binary_classification_test[2] << std::endl;
std::cout << "Current specifity: " << current_binary_classification_test[3] << std::endl;
std::cout << "Current Kappa coefficient: " << current_kappa_coefficient << std::endl;
std::cout << "Confusion matrix: " << std::endl
<< current_confusion << std::endl;
std::cout << std::endl;
}
current_threshold = fmin(maximum_threshold, current_threshold + step);
}
if (display)
{
std::cout << "Optimum threshold: " << optimum_threshold << std::endl;
std::cout << "Optimal error: " << optimal_binary_classification_test[1] << std::endl;
}
results->iterations_number = iterations;
results->final_threshold = optimum_threshold;
results->final_function_value = optimum_kappa_coefficient;
neural_network_pointer->get_probabilistic_layer_pointer()->set_decision_threshold(optimum_threshold);
return(results);
}
